def _clean_outgoing_headers(self, headers):
        """
        Removes any headers as per the middleware configuration for
        outgoing responses.

        :param headers: A WSGI start_response style list of headers,
                        [('header1', 'value), ('header2', 'value),
                         ...]
        :returns: The same headers list, but with some headers
                  removed as per the middlware configuration for
                  outgoing responses.
        """
        headers = HeaderKeyDict(headers)
        for h in headers.keys():
            if h in self.outgoing_allow_headers:
                continue
            for p in self.outgoing_allow_headers_startswith:
                if h.startswith(p):
                    break
            else:
                if h in self.outgoing_remove_headers:
                    del headers[h]
                    continue
                for p in self.outgoing_remove_headers_startswith:
                    if h.startswith(p):
                        del headers[h]
                        break
        return headers.items()

    def test_get_undersize_segment(self):
        # If we send a Content-Length header to the client, it's based on the
        # container listing. If a segment gets smaller by the time we get to
        # it (like if a client uploads a smaller segment w/the same name), we
        # need to raise an exception so that the connection will be closed by
        # the WSGI server. Otherwise, the WSGI server will be waiting for the
        # next request, the client will still be waiting for the rest of the
        # response, and nobody will be happy.

        # Shrink it by a single byte
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
            'cccc')

        req = swob.Request.blank(
            '/v1/AUTH_test/mancon/manifest',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body, exc = self.call_dlo(req, expect_exception=True)
        headers = HeaderKeyDict(headers)

        self.assertEqual(status, '200 OK')  # sanity check
        self.assertEqual(headers.get('Content-Length'), '25')  # sanity check
        self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
        self.assertTrue(isinstance(exc, exceptions.SegmentError))

    def test_get_oversize_segment(self):
        # If we send a Content-Length header to the client, it's based on the
        # container listing. If a segment gets bigger by the time we get to it
        # (like if a client uploads a bigger segment w/the same name), we need
        # to not send anything beyond the length we promised. Also, we should
        # probably raise an exception.

        # This is now longer than the original seg_03+seg_04+seg_05 combined
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '20', 'Etag': 'seg03-etag'},
            'cccccccccccccccccccc')

        req = swob.Request.blank(
            '/v1/AUTH_test/mancon/manifest',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body = self.call_dlo(req)
        headers = HeaderKeyDict(headers)

        self.assertEqual(status, '200 OK')  # sanity check
        self.assertEqual(headers.get('Content-Length'), '25')  # sanity check
        self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
        self.assertEqual(
            self.app.calls,
            [('GET', '/v1/AUTH_test/mancon/manifest'),
             ('GET', '/v1/AUTH_test/c?prefix=seg'),
             ('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
             ('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
             ('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])

 def test_keys(self):
     headers = HeaderKeyDict()
     headers['content-length'] = 20
     headers['cOnTent-tYpe'] = 'text/plain'
     headers['SomeThing-eLse'] = 'somevalue'
     self.assertEqual(
         set(headers.keys()),
         set(('Content-Length', 'Content-Type', 'Something-Else')))

 def test_pop(self):
     headers = HeaderKeyDict()
     headers['content-length'] = 20
     headers['cOntent-tYpe'] = 'text/plain'
     self.assertEqual(headers.pop('content-Length'), '20')
     self.assertEqual(headers.pop('Content-type'), 'text/plain')
     self.assertEqual(headers.pop('Something-Else', 'somevalue'),
                      'somevalue')

    def process_object_update(self, update_path, device, policy):
        """
        Process the object information to be updated and update.

        :param update_path: path to pickled object update file
        :param device: path to device
        :param policy: storage policy of object update
        """
        try:
            update = pickle.load(open(update_path, 'rb'))
        except Exception:
            self.logger.exception(
                _('ERROR Pickle problem, quarantining %s'), update_path)
            self.stats.quarantines += 1
            self.logger.increment('quarantines')
            target_path = os.path.join(device, 'quarantined', 'objects',
                                       os.path.basename(update_path))
            renamer(update_path, target_path, fsync=False)
            return
        successes = update.get('successes', [])
        part, nodes = self.get_container_ring().get_nodes(
            update['account'], update['container'])
        obj = '/%s/%s/%s' % \
              (update['account'], update['container'], update['obj'])
        headers_out = HeaderKeyDict(update['headers'])
        headers_out['user-agent'] = 'object-updater %s' % os.getpid()
        headers_out.setdefault('X-Backend-Storage-Policy-Index',
                               str(int(policy)))
        events = [spawn(self.object_update,
                        node, part, update['op'], obj, headers_out)
                  for node in nodes if node['id'] not in successes]
        success = True
        new_successes = False
        for event in events:
            event_success, node_id = event.wait()
            if event_success is True:
                successes.append(node_id)
                new_successes = True
            else:
                success = False
        if success:
            self.stats.successes += 1
            self.logger.increment('successes')
            self.logger.debug('Update sent for %(obj)s %(path)s',
                              {'obj': obj, 'path': update_path})
            self.stats.unlinks += 1
            self.logger.increment('unlinks')
            os.unlink(update_path)
        else:
            self.stats.failures += 1
            self.logger.increment('failures')
            self.logger.debug('Update failed for %(obj)s %(path)s',
                              {'obj': obj, 'path': update_path})
            if new_successes:
                update['successes'] = successes
                write_pickle(update, update_path, os.path.join(
                    device, get_tmp_dir(policy)))

    def test_head_large_object_too_many_segments(self):
        req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                                 environ={'REQUEST_METHOD': 'HEAD'})
        with mock.patch(LIMIT, 3):
            status, headers, body = self.call_dlo(req)
        headers = HeaderKeyDict(headers)

        # etag is manifest's etag
        self.assertEqual(headers["Etag"], "etag-manyseg")
        self.assertIsNone(headers.get("Content-Length"))

 def test_get_suffix_range_many_segments(self):
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=-5'})
     with mock.patch(LIMIT, 3):
         status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "200 OK")
     self.assertIsNone(headers.get("Content-Length"))
     self.assertIsNone(headers.get("Content-Range"))
     self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")

 def test_get_range_many_segments_satisfiability_unknown(self):
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=10-22'})
     with mock.patch(LIMIT, 3):
         status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "200 OK")
     # this requires multiple pages of container listing, so we can't send
     # a Content-Length header
     self.assertIsNone(headers.get("Content-Length"))
     self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")

    def test_get_response_headers_with_legacy_data(self):
        broker = backend.AccountBroker(':memory:', account='a')
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            broker.initialize(Timestamp(now).internal)
        # add some container data
        ts = (Timestamp(t).internal for t in itertools.count(int(now)))
        total_containers = 0
        total_objects = 0
        total_bytes = 0
        for policy in POLICIES:
            delete_timestamp = next(ts)
            put_timestamp = next(ts)
            object_count = int(policy)
            bytes_used = int(policy) * 10
            broker.put_container('c-%s' % policy.name, put_timestamp,
                                 delete_timestamp, object_count, bytes_used,
                                 int(policy))
            total_containers += 1
            total_objects += object_count
            total_bytes += bytes_used
        expected = HeaderKeyDict({
            'X-Account-Container-Count': total_containers,
            'X-Account-Object-Count': total_objects,
            'X-Account-Bytes-Used': total_bytes,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        })
        for policy in POLICIES:
            prefix = 'X-Account-Storage-Policy-%s-' % policy.name
            expected[prefix + 'Object-Count'] = int(policy)
            expected[prefix + 'Bytes-Used'] = int(policy) * 10
        orig_policy_stats = broker.get_policy_stats

        def stub_policy_stats(*args, **kwargs):
            policy_stats = orig_policy_stats(*args, **kwargs)
            for stats in policy_stats.values():
                # legacy db's won't return container_count
                del stats['container_count']
            return policy_stats
        broker.get_policy_stats = stub_policy_stats
        resp_headers = utils.get_response_headers(broker)
        per_policy_container_headers = [
            h for h in resp_headers if
            h.lower().startswith('x-account-storage-policy-') and
            h.lower().endswith('-container-count')]
        self.assertFalse(per_policy_container_headers)
        for key, value in resp_headers.items():
            expected_value = expected.pop(key)
            self.assertEqual(expected_value, str(value),
                             'value for %r was %r not %r' % (
                                 key, value, expected_value))
        self.assertFalse(expected)

    def handle_get_head_symlink(self, req):
        """
        Handle get/head request when client sent parameter ?symlink=get

        :param req: HTTP GET or HEAD object request with param ?symlink=get
        :returns: Response Iterator
        """
        resp = self._app_call(req.environ)
        response_header_dict = HeaderKeyDict(self._response_headers)
        symlink_sysmeta_to_usermeta(response_header_dict)
        self._response_headers = response_header_dict.items()
        return resp

 def test_get_multi_range(self):
     # DLO doesn't support multi-range GETs. The way that you express that
     # in HTTP is to return a 200 response containing the whole entity.
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=5-9,15-19'})
     with mock.patch(LIMIT, 3):
         status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "200 OK")
     self.assertIsNone(headers.get("Content-Length"))
     self.assertIsNone(headers.get("Content-Range"))
     self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')

 def test_get_range(self):
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=8-17'})
     status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "206 Partial Content")
     self.assertEqual(headers["Content-Length"], "10")
     self.assertEqual(body, b'bbcccccddd')
     expected_etag = '"%s"' % md5hex(
         md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
         md5hex("ddddd") + md5hex("eeeee"))
     self.assertEqual(headers.get("Etag"), expected_etag)

    def handle(self, req, start_response):
        app_resp = self._app_call(req.environ)

        if is_success(self._get_status_int()):
            # only decrypt body of 2xx responses
            headers = HeaderKeyDict(self._response_headers)
            content_type = headers.get('content-type', '').split(';', 1)[0]
            if content_type == 'application/json':
                app_resp = self.process_json_resp(req, app_resp)

        start_response(self._response_status,
                       self._response_headers,
                       self._response_exc_info)

        return app_resp

    def test_get_undersize_segment_range(self):
        # Shrink it by a single byte
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
            'cccc')

        req = swob.Request.blank(
            '/v1/AUTH_test/mancon/manifest',
            environ={'REQUEST_METHOD': 'GET'},
            headers={'Range': 'bytes=0-14'})
        status, headers, body = self.call_dlo(req)
        headers = HeaderKeyDict(headers)

        self.assertEqual(status, '206 Partial Content')  # sanity check
        self.assertEqual(headers.get('Content-Length'), '15')  # sanity check
        self.assertEqual(body, b'aaaaabbbbbcccc')

class FakeConn(object):

    def __init__(self, status, headers=None, body='', **kwargs):
        self.status = status
        try:
            self.reason = RESPONSE_REASONS[self.status][0]
        except Exception:
            self.reason = 'Fake'
        self.body = body
        self.resp_headers = HeaderKeyDict()
        if headers:
            self.resp_headers.update(headers)
        self.etag = None

    def _update_raw_call_args(self, *args, **kwargs):
        capture_attrs = ('host', 'port', 'method', 'path', 'req_headers',
                         'query_string')
        for attr, value in zip(capture_attrs, args[:len(capture_attrs)]):
            setattr(self, attr, value)
        return self

    def getresponse(self):
        if self.etag:
            self.resp_headers['etag'] = str(self.etag.hexdigest())
        if isinstance(self.status, Exception):
            raise self.status
        return self

    def getheader(self, header, default=None):
        return self.resp_headers.get(header, default)

    def getheaders(self):
        return self.resp_headers.items()

    def read(self, amt=None):
        if isinstance(self.body, six.StringIO):
            return self.body.read(amt)
        elif amt is None:
            return self.body
        else:
            return Exception('Not a StringIO entry')

    def send(self, data):
        if not self.etag:
            self.etag = md5()
        self.etag.update(data)

 def __init__(self, status, body='', headers=None, frag_index=None):
     self.status = status
     self.body = body
     self.readable = BytesIO(body)
     self.headers = HeaderKeyDict(headers)
     if frag_index is not None:
         self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index
     fake_reason = ('Fake', 'This response is a lie.')
     self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]

    def __call__(self, env, start_response):
        method = env['REQUEST_METHOD']
        path = env['PATH_INFO']
        _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
                                       rest_with_last=True)
        if env.get('QUERY_STRING'):
            path += '?' + env['QUERY_STRING']

        if 'swift.authorize' in env:
            resp = env['swift.authorize'](swob.Request(env))
            if resp:
                return resp(env, start_response)

        req_headers = swob.Request(env).headers
        self.swift_sources.append(env.get('swift.source'))
        self.txn_ids.append(env.get('swift.trans_id'))

        try:
            resp_class, raw_headers, body = self._find_response(method, path)
            headers = HeaderKeyDict(raw_headers)
        except KeyError:
            if (env.get('QUERY_STRING')
                    and (method, env['PATH_INFO']) in self._responses):
                resp_class, raw_headers, body = self._find_response(
                    method, env['PATH_INFO'])
                headers = HeaderKeyDict(raw_headers)
            elif method == 'HEAD' and ('GET', path) in self._responses:
                resp_class, raw_headers, body = self._find_response(
                    'GET', path)
                body = None
                headers = HeaderKeyDict(raw_headers)
            elif method == 'GET' and obj and path in self.uploaded:
                resp_class = swob.HTTPOk
                headers, body = self.uploaded[path]
            else:
                raise KeyError("Didn't find %r in allowed responses" % (
                    (method, path),))

        self._calls.append((method, path, req_headers))

        # simulate object PUT
        if method == 'PUT' and obj:
            input = env['wsgi.input'].read()
            etag = md5(input).hexdigest()
            headers.setdefault('Etag', etag)
            headers.setdefault('Content-Length', len(input))

            # keep it for subsequent GET requests later
            self.uploaded[path] = (deepcopy(headers), input)
            if "CONTENT_TYPE" in env:
                self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]

        # range requests ought to work, which require conditional_response=True
        req = swob.Request(env)
        resp = resp_class(req=req, headers=headers, body=body,
                          conditional_response=req.method in ('GET', 'HEAD'))
        wsgi_iter = resp(env, start_response)
        self.mark_opened(path)
        return LeakTrackingIter(wsgi_iter, self, path)

 def __init__(self, status, headers=None, body='', **kwargs):
     self.status = status
     try:
         self.reason = RESPONSE_REASONS[self.status][0]
     except Exception:
         self.reason = 'Fake'
     self.body = body
     self.resp_headers = HeaderKeyDict()
     if headers:
         self.resp_headers.update(headers)
     self.etag = None

    def __call__(self, env, start_response):
        method = env["REQUEST_METHOD"]
        if method not in self.ALLOWED_METHODS:
            raise HTTPNotImplemented()

        path = env["PATH_INFO"]
        _, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True)
        if env.get("QUERY_STRING"):
            path += "?" + env["QUERY_STRING"]

        if "swift.authorize" in env:
            resp = env["swift.authorize"](swob.Request(env))
            if resp:
                return resp(env, start_response)

        req_headers = swob.Request(env).headers
        self.swift_sources.append(env.get("swift.source"))
        self.txn_ids.append(env.get("swift.trans_id"))

        try:
            resp_class, raw_headers, body = self._find_response(method, path)
            headers = HeaderKeyDict(raw_headers)
        except KeyError:
            if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses:
                resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"])
                headers = HeaderKeyDict(raw_headers)
            elif method == "HEAD" and ("GET", path) in self._responses:
                resp_class, raw_headers, body = self._find_response("GET", path)
                body = None
                headers = HeaderKeyDict(raw_headers)
            elif method == "GET" and obj and path in self.uploaded:
                resp_class = swob.HTTPOk
                headers, body = self.uploaded[path]
            else:
                raise KeyError("Didn't find %r in allowed responses" % ((method, path),))

        self._calls.append((method, path, req_headers))

        # simulate object PUT
        if method == "PUT" and obj:
            input = env["wsgi.input"].read()
            etag = md5(input).hexdigest()
            headers.setdefault("Etag", etag)
            headers.setdefault("Content-Length", len(input))

            # keep it for subsequent GET requests later
            self.uploaded[path] = (deepcopy(headers), input)
            if "CONTENT_TYPE" in env:
                self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"]

        # range requests ought to work, which require conditional_response=True
        req = swob.Request(env)
        resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD"))
        wsgi_iter = resp(env, start_response)
        self.mark_opened(path)
        return LeakTrackingIter(wsgi_iter, self, path)