本文整理汇总了Python中swift.common.utils.closing_if_possible函数的典型用法代码示例。如果您正苦于以下问题:Python closing_if_possible函数的具体用法?Python closing_if_possible怎么用?Python closing_if_possible使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了closing_if_possible函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: call_app
def call_app(self, req, app=None):
if app is None:
app = self.app
self.authorized = []
def authorize(req):
self.authorized.append(req)
if 'swift.authorize' not in req.environ:
req.environ['swift.authorize'] = authorize
req.headers.setdefault("User-Agent", "Melted Cheddar")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
with utils.closing_if_possible(body_iter):
body = b''.join(body_iter)
return status[0], headers[0], body
开发者ID:fvennetier,项目名称:oio-swift,代码行数:26,代码来源:test_regexcontainer.py
示例2: _get_container_listing
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
'''
:param version: whatever
:param account: native
:param container: native
:param prefix: native
:param marker: native
'''
con_req = make_subrequest(
req.environ,
path=wsgi_quote('/'.join([
'', str_to_wsgi(version),
str_to_wsgi(account), str_to_wsgi(container)])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
if req.method == 'HEAD':
con_resp.body = b''
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(b''.join(con_resp.app_iter))
开发者ID:mahak,项目名称:swift,代码行数:28,代码来源:dlo.py
示例3: _fetch_sub_slo_segments
def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
"""
Fetch the submanifest, parse it, and return it.
Raise exception on failures.
"""
sub_req = req.copy_get()
sub_req.range = None
sub_req.environ['PATH_INFO'] = '/'.join(['', version, acc, con, obj])
sub_req.environ['swift.source'] = 'SLO'
sub_req.user_agent = "%s SLO MultipartGET" % sub_req.user_agent
sub_resp = sub_req.get_response(self.slo.app)
if not is_success(sub_resp.status_int):
raise ListingIterError(
'ERROR: while fetching %s, GET of submanifest %s '
'failed with status %d' % (req.path, sub_req.path,
sub_resp.status_int))
try:
with closing_if_possible(sub_resp.app_iter):
return json.loads(''.join(sub_resp.app_iter))
except ValueError as err:
raise ListingIterError(
'ERROR: while fetching %s, JSON-decoding of submanifest %s '
'failed with %s' % (req.path, sub_req.path, err))
开发者ID:nbhavana,项目名称:swift,代码行数:25,代码来源:slo.py
示例4: _fetch_sub_slo_segments
def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
"""
Fetch the submanifest, parse it, and return it.
Raise exception on failures.
"""
sub_req = make_subrequest(
req.environ,
path="/".join(["", version, acc, con, obj]),
method="GET",
headers={"x-auth-token": req.headers.get("x-auth-token")},
agent=("%(orig)s " + "SLO MultipartGET"),
swift_source="SLO",
)
sub_resp = sub_req.get_response(self.slo.app)
if not is_success(sub_resp.status_int):
close_if_possible(sub_resp.app_iter)
raise ListingIterError(
"ERROR: while fetching %s, GET of submanifest %s "
"failed with status %d" % (req.path, sub_req.path, sub_resp.status_int)
)
try:
with closing_if_possible(sub_resp.app_iter):
return json.loads("".join(sub_resp.app_iter))
except ValueError as err:
raise ListingIterError(
"ERROR: while fetching %s, JSON-decoding of submanifest %s "
"failed with %s" % (req.path, sub_req.path, err)
)
开发者ID:pchng,项目名称:swift,代码行数:30,代码来源:slo.py
示例5: get_or_head_response
def get_or_head_response(self, req, resp_headers, resp_iter):
with closing_if_possible(resp_iter):
resp_body = "".join(resp_iter)
try:
segments = json.loads(resp_body)
except ValueError:
segments = []
etag = md5()
content_length = 0
for seg_dict in segments:
if seg_dict.get("range"):
etag.update("%s:%s;" % (seg_dict["hash"], seg_dict["range"]))
else:
etag.update(seg_dict["hash"])
if config_true_value(seg_dict.get("sub_slo")):
override_bytes_from_content_type(seg_dict, logger=self.slo.logger)
content_length += self._segment_length(seg_dict)
response_headers = [(h, v) for h, v in resp_headers if h.lower() not in ("etag", "content-length")]
response_headers.append(("Content-Length", str(content_length)))
response_headers.append(("Etag", '"%s"' % etag.hexdigest()))
if req.method == "HEAD":
return self._manifest_head_response(req, response_headers)
else:
return self._manifest_get_response(req, content_length, response_headers, segments)
开发者ID:pchng,项目名称:swift,代码行数:28,代码来源:slo.py
示例6: call_dlo
def call_dlo(self, req, app=None, expect_exception=False):
if app is None:
app = self.dlo
req.headers.setdefault("User-Agent", "Soap Opera")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
body = ''
caught_exc = None
try:
# appease the close-checker
with closing_if_possible(body_iter):
for chunk in body_iter:
body += chunk
except Exception as exc:
if expect_exception:
caught_exc = exc
else:
raise
if expect_exception:
return status[0], headers[0], body, caught_exc
else:
return status[0], headers[0], body
开发者ID:bigdig,项目名称:swift,代码行数:31,代码来源:test_dlo.py
示例7: PUT
def PUT(self, req):
"""HTTP PUT request handler."""
container_info = self.container_info(
self.account_name, self.container_name, req)
req.acl = container_info['write_acl']
req.environ['swift_sync_key'] = container_info['sync_key']
# is request authorized
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
old_slo_manifest = None
# If versioning is disabled, we must check if the object exists.
# If it's a SLO, we will have to delete the parts if the current
# operation is a success.
if (self.app.delete_slo_parts and
not container_info['sysmeta'].get('versions-location', None)):
try:
dest_info = get_object_info(req.environ, self.app)
if 'slo-size' in dest_info['sysmeta']:
manifest_env = req.environ.copy()
manifest_env['QUERY_STRING'] = 'multipart-manifest=get'
manifest_req = make_subrequest(manifest_env, 'GET')
manifest_resp = manifest_req.get_response(self.app)
old_slo_manifest = json.loads(manifest_resp.body)
except Exception as exc:
self.app.logger.warn(('Failed to check existence of %s. If '
'overwriting a SLO, old parts may '
'remain. Error was: %s') %
(req.path, exc))
self._update_content_type(req)
self._update_x_timestamp(req)
# check constraints on object name and request headers
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
if req.headers.get('Oio-Copy-From'):
return self._link_object(req)
data_source = req.environ['wsgi.input']
if req.content_length:
data_source = ExpectedSizeReader(data_source, req.content_length)
headers = self._prepare_headers(req)
with closing_if_possible(data_source):
resp = self._store_object(req, data_source, headers)
if old_slo_manifest and resp.is_success:
self.app.logger.debug(
'Previous object %s was a SLO, deleting parts',
req.path)
self._delete_slo_parts(req, old_slo_manifest)
return resp
开发者ID:fvennetier,项目名称:oio-swift,代码行数:60,代码来源:obj.py
示例8: _fetch_sub_slo_segments
def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
"""
Fetch the submanifest, parse it, and return it.
Raise exception on failures.
"""
sub_req = make_subrequest(
req.environ, path='/'.join(['', version, acc, con, obj]),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO')
sub_resp = sub_req.get_response(self.slo.app)
if not is_success(sub_resp.status_int):
close_if_possible(sub_resp.app_iter)
raise ListingIterError(
'ERROR: while fetching %s, GET of submanifest %s '
'failed with status %d' % (req.path, sub_req.path,
sub_resp.status_int))
try:
with closing_if_possible(sub_resp.app_iter):
return json.loads(''.join(sub_resp.app_iter))
except ValueError as err:
raise ListingIterError(
'ERROR: while fetching %s, JSON-decoding of submanifest %s '
'failed with %s' % (req.path, sub_req.path, err))
开发者ID:iloveyou416068,项目名称:swift-1,代码行数:26,代码来源:slo.py
示例9: multipart_response_iter
def multipart_response_iter(self, resp, boundary, body_key, crypto_meta):
"""
Decrypts a multipart mime doc response body.
:param resp: application response
:param boundary: multipart boundary string
:param body_key: decryption key for the response body
:param crypto_meta: crypto_meta for the response body
:return: generator for decrypted response body
"""
with closing_if_possible(resp):
parts_iter = multipart_byteranges_to_document_iters(
FileLikeIter(resp), boundary)
for first_byte, last_byte, length, headers, body in parts_iter:
yield b"--" + boundary + b"\r\n"
for header, value in headers:
yield b"%s: %s\r\n" % (wsgi_to_bytes(header),
wsgi_to_bytes(value))
yield b"\r\n"
decrypt_ctxt = self.crypto.create_decryption_ctxt(
body_key, crypto_meta['iv'], first_byte)
for chunk in iter(lambda: body.read(DECRYPT_CHUNK_SIZE), b''):
yield decrypt_ctxt.update(chunk)
yield b"\r\n"
yield b"--" + boundary + b"--"
开发者ID:mahak,项目名称:swift,代码行数:30,代码来源:decrypter.py
示例10: get_or_head_response
def get_or_head_response(self, req, resp_headers, resp_iter):
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
try:
segments = json.loads(resp_body)
except ValueError:
segments = []
etag = md5()
content_length = 0
for seg_dict in segments:
if seg_dict.get('range'):
etag.update('%s:%s;' % (seg_dict['hash'], seg_dict['range']))
else:
etag.update(seg_dict['hash'])
if config_true_value(seg_dict.get('sub_slo')):
override_bytes_from_content_type(
seg_dict, logger=self.slo.logger)
content_length += self._segment_length(seg_dict)
response_headers = [(h, v) for h, v in resp_headers
if h.lower() not in ('etag', 'content-length')]
response_headers.append(('Content-Length', str(content_length)))
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
if req.method == 'HEAD':
return self._manifest_head_response(req, response_headers)
else:
return self._manifest_get_response(
req, content_length, response_headers, segments)
开发者ID:iloveyou416068,项目名称:swift-1,代码行数:31,代码来源:slo.py
示例11: _get_manifest_read
def _get_manifest_read(self, resp_iter):
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
try:
segments = json.loads(resp_body)
except ValueError:
segments = []
return segments
开发者ID:HoratiusTang,项目名称:swift,代码行数:9,代码来源:slo.py
示例12: process_json_resp
def process_json_resp(self, req, resp_iter):
"""
Parses json body listing and decrypt encrypted entries. Updates
Content-Length header with new body length and return a body iter.
"""
with closing_if_possible(resp_iter):
resp_body = b''.join(resp_iter)
body_json = json.loads(resp_body)
new_body = json.dumps([self.decrypt_obj_dict(req, obj_dict)
for obj_dict in body_json]).encode('ascii')
self.update_content_length(len(new_body))
return [new_body]
开发者ID:mahak,项目名称:swift,代码行数:12,代码来源:decrypter.py
示例13: _process_json_resp
def _process_json_resp(self, resp_iter, req):
"""
Iterate through json body looking for symlinks and modify its content
:return: modified json body
"""
with closing_if_possible(resp_iter):
resp_body = b''.join(resp_iter)
body_json = json.loads(resp_body)
swift_version, account, _junk = split_path(req.path, 2, 3, True)
new_body = json.dumps(
[self._extract_symlink_path_json(obj_dict, swift_version, account)
for obj_dict in body_json]).encode('ascii')
self.update_content_length(len(new_body))
return [new_body]
开发者ID:jgmerritt,项目名称:swift,代码行数:14,代码来源:symlink.py
示例14: response_iter
def response_iter(self, resp, body_key, crypto_meta, offset):
"""
Decrypts a response body.
:param resp: application response
:param body_key: decryption key for the response body
:param crypto_meta: crypto_meta for the response body
:param offset: offset into object content at which response body starts
:return: generator for decrypted response body
"""
decrypt_ctxt = self.crypto.create_decryption_ctxt(
body_key, crypto_meta['iv'], offset)
with closing_if_possible(resp):
for chunk in resp:
yield decrypt_ctxt.update(chunk)
开发者ID:mahak,项目名称:swift,代码行数:15,代码来源:decrypter.py
示例15: _get_container_listing
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
con_req = make_subrequest(
req.environ, path='/'.join(['', version, account, container]),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'format=json&prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(''.join(con_resp.app_iter))
开发者ID:nautilusnemo,项目名称:swift,代码行数:16,代码来源:dlo.py
示例16: _list_objects
def _list_objects(self, env, account, ct_parts, header_cb,
prefix='', limit=DEFAULT_LIMIT,
marker=None, force_master=False):
"""
returns items
"""
sub_path = quote(self.DELIMITER.join(
('', 'v1', account, self.ENCODED_DELIMITER.join(ct_parts))))
LOG.debug("%s: listing objects from '%s' "
"(limit=%d, prefix=%s, marker=%s)",
self.SWIFT_SOURCE, sub_path, limit, prefix, marker)
sub_req = make_subrequest(env.copy(), method='GET', path=sub_path,
body='',
swift_source=self.SWIFT_SOURCE)
params = sub_req.params
params.pop('delimiter', None) # allow list-multipart-uploads
params['limit'] = str(limit) # FIXME: why is it str?
params['prefix'] = prefix
params['format'] = 'json'
if marker:
params['marker'] = marker
else:
params.pop('marker', None)
if force_master:
sub_req.environ.setdefault('oio.query', {})
sub_req.environ['oio.query']['force_master'] = True
sub_req.params = params
resp = sub_req.get_response(self.app)
obj_prefix = ''
if len(ct_parts) > 1:
obj_prefix = self.DELIMITER.join(ct_parts[1:] + ['', ])
if not resp.is_success or resp.content_length == 0:
LOG.warn("%s: Failed to list %s",
self.SWIFT_SOURCE, sub_path)
return
with closing_if_possible(resp.app_iter):
items = json.loads(resp.body)
if header_cb:
header_cb(resp.headers)
for obj in items:
if 'name' in obj:
obj['name'] = obj_prefix.decode('utf-8') + obj['name']
yield obj
开发者ID:jfsmig,项目名称:oio-swift,代码行数:47,代码来源:container_hierarchy.py
示例17: process_xml_resp
def process_xml_resp(self, key, resp_iter):
"""
Parses xml body listing and decrypt encrypted entries. Updates
Content-Length header with new body length and return a body iter.
"""
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
tree = ElementTree.fromstring(resp_body)
for elem in tree.iter('hash'):
ciphertext = elem.text.encode('utf8')
plain = self.decrypt_value_with_meta(ciphertext, key)
elem.text = plain.decode('utf8')
new_body = ElementTree.tostring(tree, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
self.update_content_length(len(new_body))
return [new_body]
开发者ID:SmartInfrastructures,项目名称:swift,代码行数:17,代码来源:decrypter.py
示例18: call_dlo
def call_dlo(self, req, app=None):
if app is None:
app = self.dlo
req.headers.setdefault("User-Agent", "Soap Opera")
status = [None]
headers = [None]
def start_response(s, h, ei=None):
status[0] = s
headers[0] = h
body_iter = app(req.environ, start_response)
body = b''
# appease the close-checker
with closing_if_possible(body_iter):
for chunk in body_iter:
body += chunk
return status[0], headers[0], body
开发者ID:mahak,项目名称:swift,代码行数:20,代码来源:test_dlo.py
示例19: make_request
def make_request(
self, method, path, headers, acceptable_statuses, body_file=None,
params=None):
"""Makes a request to Swift with retries.
:param method: HTTP method of request.
:param path: Path of request.
:param headers: Headers to be sent with request.
:param acceptable_statuses: List of acceptable statuses for request.
:param body_file: Body file to be passed along with request,
defaults to None.
:param params: A dict of params to be set in request query string,
defaults to None.
:returns: Response object on success.
:raises UnexpectedResponse: Exception raised when make_request() fails
to get a response with an acceptable status
:raises Exception: Exception is raised when code fails in an
unexpected way.
"""
headers = dict(headers)
headers['user-agent'] = self.user_agent
for attempt in range(self.request_tries):
resp = exc_type = exc_value = exc_traceback = None
req = Request.blank(
path, environ={'REQUEST_METHOD': method}, headers=headers)
if body_file is not None:
if hasattr(body_file, 'seek'):
body_file.seek(0)
req.body_file = body_file
if params:
req.params = params
try:
resp = req.get_response(self.app)
except (Exception, Timeout):
exc_type, exc_value, exc_traceback = exc_info()
else:
if resp.status_int in acceptable_statuses or \
resp.status_int // 100 in acceptable_statuses:
return resp
elif not is_server_error(resp.status_int):
# No sense retrying when we expect the same result
break
# sleep only between tries, not after each one
if attempt < self.request_tries - 1:
if resp:
# always close any resp.app_iter before we discard it
with closing_if_possible(resp.app_iter):
# for non 2XX requests it's safe and useful to drain
# the response body so we log the correct status code
if resp.status_int // 100 != 2:
for iter_body in resp.app_iter:
pass
sleep(2 ** (attempt + 1))
if resp:
msg = 'Unexpected response: %s' % resp.status
if resp.status_int // 100 != 2 and resp.body:
# provide additional context (and drain the response body) for
# non 2XX responses
msg += ' (%s)' % resp.body
raise UnexpectedResponse(msg, resp)
if exc_type:
# To make pep8 tool happy, in place of raise t, v, tb:
six.reraise(exc_type, exc_value, exc_traceback)
开发者ID:jgmerritt,项目名称:swift,代码行数:66,代码来源:internal_client.py
示例20: __call__
def __call__(self, env, start_response):
# a lot of this is cribbed from listing_formats / swob.Request
if env['REQUEST_METHOD'] != 'GET':
# Nothing to translate
return self.app(env, start_response)
try:
v, a, c = split_path(env.get('SCRIPT_NAME', '') +
env['PATH_INFO'], 3, 3)
if not valid_api_version(v):
raise ValueError
except ValueError:
# not a container request; pass through
return self.app(env, start_response)
ctx = WSGIContext(self.app)
resp_iter = ctx._app_call(env)
content_type = content_length = cl_index = None
for index, (header, value) in enumerate(ctx._response_headers):
header = header.lower()
if header == 'content-type':
content_type = value.split(';', 1)[0].strip()
if content_length:
break
elif header == 'content-length':
cl_index = index
try:
content_length = int(value)
except ValueError:
pass # ignore -- we'll bail later
if content_type:
break
if content_type != 'application/json' or content_length is None or \
content_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return resp_iter
# We've done our sanity checks, slurp the response into memory
with closing_if_possible(resp_iter):
body = b''.join(resp_iter)
try:
listing = json.loads(body)
for item in listing:
if 'subdir' in item:
continue
value, params = parse_header(item['hash'])
if 's3_etag' in params:
item['s3_etag'] = '"%s"' % params.pop('s3_etag')
item['hash'] = value + ''.join(
'; %s=%s' % kv for kv in params.items())
except (TypeError, KeyError, ValueError):
# If anything goes wrong above, drop back to original response
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return [body]
body = json.dumps(listing).encode('ascii')
ctx._response_headers[cl_index] = (
ctx._response_headers[cl_index][0],
str(len(body)),
)
start_response(ctx._response_status, ctx._response_headers,
ctx._response_exc_info)
return [body]
开发者ID:mahak,项目名称:swift,代码行数:68,代码来源:s3api.py
注:本文中的swift.common.utils.closing_if_possible函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论