def get_checked_field(document, name, value_type):
    """Validates and retrieves a typed field from a document.

    This function attempts to look up doc[name], and raises
    appropriate HTTP errors if the field is missing or not an
    instance of the given type.

    :param document: dict-like object
    :param name: field name
    :param value_type: expected value type, or '*' to accept any type
    :raises: HTTPBadRequest if the field is missing or not an
        instance of value_type
    :returns: value obtained from doc[name]
    """

    try:
        value = document[name]
    except KeyError:
        description = _(u'Missing "{name}" field.').format(name=name)
        raise errors.HTTPBadRequestBody(description)

    if value_type == '*' or isinstance(value, value_type):
        return value

    description = _(u'The value of the "{name}" field must be a {vtype}.')
    description = description.format(name=name, vtype=value_type.__name__)
    raise errors.HTTPBadRequestBody(description)

    def on_patch(self, req, resp, project_id, queue_name, claim_id):
        LOG.debug(_(u'Claim Item PATCH - claim: %(claim_id)s, '
                    u'queue: %(queue_name)s, project:%(project_id)s') %
                  {'queue_name': queue_name,
                   'project_id': project_id,
                   'claim_id': claim_id})

        # Read claim metadata (e.g., TTL) and raise appropriate
        # HTTP errors as needed.
        metadata, = wsgi_utils.filter_stream(req.stream, req.content_length,
                                             CLAIM_PATCH_SPEC)

        try:
            self._validate.claim_updating(metadata)
            self.claim_controller.update(queue_name,
                                         claim_id=claim_id,
                                         metadata=metadata,
                                         project=project_id)

            resp.status = falcon.HTTP_204

        except validation.ValidationFailed as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))

        except storage_errors.DoesNotExist as ex:
            LOG.debug(ex)
            raise falcon.HTTPNotFound()

        except Exception as ex:
            LOG.exception(ex)
            description = _(u'Claim could not be updated.')
            raise wsgi_errors.HTTPServiceUnavailable(description)

    def claim_creation(self, metadata, limit=None):
        """Restrictions on the claim parameters upon creation.

        :param metadata: The claim metadata
        :param limit: The number of messages to claim
        :raises: ValidationFailed if either TTL or grace is out of range,
            or the expected number of messages exceed the limit.
        """

        self.claim_updating(metadata)

        uplimit = self._limits_conf.max_messages_per_claim
        if limit is not None and not (0 < limit <= uplimit):
            msg = _(u'Limit must be at least 1 and may not '
                    'be greater than {0}.')

            raise ValidationFailed(
                msg, self._limits_conf.max_messages_per_claim)

        grace = metadata['grace']

        if not (MIN_CLAIM_GRACE <= grace <= self._limits_conf.max_claim_grace):
            msg = _(u'The grace for a claim may not exceed {0} seconds, and '
                    'must be at least {1} seconds long.')

            raise ValidationFailed(
                msg, self._limits_conf.max_claim_grace, MIN_CLAIM_GRACE)

    def on_get(self, req, resp, project_id, queue_name, message_id):
        LOG.debug(_(u'Messages item GET - message: %(message)s, '
                    u'queue: %(queue)s, project: %(project)s'),
                  {'message': message_id,
                   'queue': queue_name,
                   'project': project_id})
        try:
            message = self.message_controller.get(
                queue_name,
                message_id,
                project=project_id)

        except storage_errors.DoesNotExist as ex:
            LOG.debug(ex)
            raise falcon.HTTPNotFound()

        except Exception as ex:
            LOG.exception(ex)
            description = _(u'Message could not be retrieved.')
            raise wsgi_errors.HTTPServiceUnavailable(description)

        # Prepare response
        message['href'] = req.path
        del message['id']

        resp.content_location = req.relative_uri
        resp.body = utils.to_json(message)

    def __enter__(self):
        basedir = os.path.dirname(self.fname)

        if not os.path.exists(basedir):
            fileutils.ensure_tree(basedir)
            LOG.info(_('Created lock path: %s'), basedir)

        self.lockfile = open(self.fname, 'w')

        while True:
            try:
                # Using non-blocking locks since green threads are not
                # patched to deal with blocking locking calls.
                # Also upon reading the MSDN docs for locking(), it seems
                # to have a laughable 10 attempts "blocking" mechanism.
                self.trylock()
                LOG.debug(_('Got file lock "%s"'), self.fname)
                return self
            except IOError as e:
                if e.errno in (errno.EACCES, errno.EAGAIN):
                    # external locks synchronise things like iptables
                    # updates - give it some time to prevent busy spinning
                    time.sleep(0.01)
                else:
                    raise

    def wrapper(self, *args, **kwargs):
        # TODO(kgriffs): Figure out a way to not have to rely on the
        # Note(prashanthr_) : Try to reuse this utility. Violates DRY
        # Can pass config parameters into the decorator and create a
        # storage level utility.

        max_attemps = self.driver.redis_conf.max_reconnect_attempts
        sleep_sec = self.driver.redis_conf.reconnect_sleep

        for attempt in range(max_attemps):
            try:
                self.init_connection()
                return func(self, *args, **kwargs)
                break

            except redis.exceptions.ConnectionError as ex:
                LOG.warn(_(u'Caught AutoReconnect, retrying the '
                           'call to {0}').format(func))

                time.sleep(sleep_sec * (2 ** attempt))
        else:
            LOG.error(_(u'Caught AutoReconnect, maximum attempts '
                        'to {0} exceeded.').format(func))

            raise ex

    def on_delete(self, req, resp, project_id, queue_name, message_id):
        LOG.debug(_(u'Messages item DELETE - message: %(message)s, '
                    u'queue: %(queue)s, project: %(project)s'),
                  {'message': message_id,
                   'queue': queue_name,
                   'project': project_id})
        try:
            self.message_controller.delete(
                queue_name,
                message_id=message_id,
                project=project_id,
                claim=req.get_param('claim_id'))

        except storage_errors.NotPermitted as ex:
            LOG.exception(ex)
            title = _(u'Unable to delete')
            description = _(u'This message is claimed; it cannot be '
                            u'deleted without a valid claim_id.')
            raise falcon.HTTPForbidden(title, description)

        except Exception as ex:
            LOG.exception(ex)
            description = _(u'Message could not be deleted.')
            raise wsgi_errors.HTTPServiceUnavailable(description)

        # Alles guete
        resp.status = falcon.HTTP_204

 def __exit__(self, exc_type, exc_val, exc_tb):
     try:
         self.unlock()
         self.lockfile.close()
     except IOError:
         LOG.exception(_("Could not release the acquired lock `%s`"),
                       self.fname)
     LOG.debug(_('Released file lock "%s"'), self.fname)

        def inner(*args, **kwargs):
            with lock(name, lock_file_prefix, external, lock_path):
                LOG.debug(_('Got semaphore / lock "%(function)s"'),
                          {'function': f.__name__})
                return f(*args, **kwargs)

            LOG.debug(_('Semaphore / lock released "%(function)s"'),
                      {'function': f.__name__})

    def on_post(self, req, resp, project_id, queue_name):
        LOG.debug(_(u'Claims collection POST - queue: %(queue)s, '
                    u'project: %(project)s'),
                  {'queue': queue_name, 'project': project_id})

        # Check for an explicit limit on the # of messages to claim
        limit = req.get_param_as_int('limit')
        claim_options = {} if limit is None else {'limit': limit}

        # Place JSON size restriction before parsing
        if req.content_length > self._metadata_max_length:
            description = _(u'Claim metadata size is too large.')
            raise wsgi_errors.HTTPBadRequestBody(description)

        # Read claim metadata (e.g., TTL) and raise appropriate
        # HTTP errors as needed.
        metadata, = wsgi_utils.filter_stream(req.stream, req.content_length,
                                             CLAIM_POST_SPEC)

        # Claim some messages
        try:
            self._validate.claim_creation(metadata, **claim_options)
            cid, msgs = self.claim_controller.create(
                queue_name,
                metadata=metadata,
                project=project_id,
                **claim_options)

            # Buffer claimed messages
            # TODO(kgriffs): optimize, along with serialization (below)
            resp_msgs = list(msgs)

        except validation.ValidationFailed as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))

        except Exception as ex:
            LOG.exception(ex)
            description = _(u'Claim could not be created.')
            raise wsgi_errors.HTTPServiceUnavailable(description)

        # Serialize claimed messages, if any. This logic assumes
        # the storage driver returned well-formed messages.
        if len(resp_msgs) != 0:
            for msg in resp_msgs:
                msg['href'] = _msg_uri_from_claim(
                    req.path.rpartition('/')[0], msg['id'], cid)

                del msg['id']

            resp.location = req.path + '/' + cid
            resp.body = utils.to_json(resp_msgs)
            resp.status = falcon.HTTP_201
        else:
            resp.status = falcon.HTTP_204

    def storage(self):
        LOG.debug(_(u'Loading storage driver'))

        if self.conf.sharding:
            LOG.debug(_(u'Storage sharding enabled'))
            storage_driver = sharding.DataDriver(self.conf, self.cache,
                                                 self.control)
        else:
            storage_driver = storage_utils.load_storage_driver(
                self.conf, self.cache)

        LOG.debug(_(u'Loading storage pipeline'))
        return pipeline.DataDriver(self.conf, storage_driver)

    def on_delete(self, req, resp, project_id, queue_name):
        LOG.debug(_(u'Queue item DELETE - queue: %(queue)s, '
                    u'project: %(project)s'),
                  {'queue': queue_name, 'project': project_id})
        try:
            self.queue_controller.delete(queue_name, project=project_id)

        except Exception as ex:
            LOG.exception(ex)
            description = _(u'Queue could not be deleted.')
            raise wsgi_errors.HTTPServiceUnavailable(description)

        resp.status = falcon.HTTP_204

def _get_storage_pipeline(resource_name, conf):
    """Constructs and returns a storage resource pipeline.

    This is a helper function for any service supporting
    pipelines for the storage layer. The function returns
    a pipeline based on the `{resource_name}_pipeline`
    config option.

    Stages in the pipeline implement controller methods
    that they want to hook. A stage can halt the
    pipeline immediate by returning a value that is
    not None; otherwise, processing will continue
    to the next stage, ending with the actual storage
    controller.

    :param conf: Configuration instance.
    :type conf: `cfg.ConfigOpts`

    :returns: A pipeline to use.
    :rtype: `Pipeline`
    """
    conf.register_opts(_PIPELINE_CONFIGS, group=_PIPELINE_GROUP)

    storage_conf = conf[_PIPELINE_GROUP]

    pipeline = []
    for ns in storage_conf[resource_name + "_pipeline"]:
        try:
            mgr = driver.DriverManager("marconi.queues.storage.stages", ns, invoke_on_load=True)
            pipeline.append(mgr.driver)
        except RuntimeError as exc:
            LOG.warning(_(u"Stage %(stage)d could not be imported: %(ex)s"), {"stage": ns, "ex": str(exc)})
            continue

    return common.Pipeline(pipeline)

 def deprecated(self, msg, *args, **kwargs):
     stdmsg = _("Deprecated: %s") % msg
     if CONF.fatal_deprecations:
         self.critical(stdmsg, *args, **kwargs)
         raise DeprecatedConfig(msg=stdmsg)
     else:
         self.warn(stdmsg, *args, **kwargs)

    def on_put(self, req, resp, project_id, queue_name):
        LOG.debug(_(u'Queue item PUT - queue: %(queue)s, '
                    u'project: %(project)s'),
                  {'queue': queue_name, 'project': project_id})

        try:
            created = self.queue_controller.create(
                queue_name, project=project_id)

        except Exception as ex:
            LOG.exception(ex)
            description = _(u'Queue could not be created.')
            raise wsgi_errors.HTTPServiceUnavailable(description)

        resp.status = falcon.HTTP_201 if created else falcon.HTTP_204
        resp.location = req.path

 def inner_func(*args, **kwargs):
     last_log_time = 0
     last_exc_message = None
     exc_count = 0
     while True:
         try:
             return infunc(*args, **kwargs)
         except Exception as exc:
             this_exc_message = unicode(exc)
             if this_exc_message == last_exc_message:
                 exc_count += 1
             else:
                 exc_count = 1
             # Do not log any more frequently than once a minute unless
             # the exception message changes
             cur_time = int(time.time())
             if (cur_time - last_log_time > 60 or
                     this_exc_message != last_exc_message):
                 logging.exception(
                     _('Unexpected exception occurred %d time(s)... '
                       'retrying.') % exc_count)
                 last_log_time = cur_time
                 last_exc_message = this_exc_message
                 exc_count = 0
             # This should be a very rare event. In case it isn't, do
             # a sleep.
             time.sleep(1)

    def deprecated(self, msg, *args, **kwargs):
        """Call this method when a deprecated feature is used.

        If the system is configured for fatal deprecations then the message
        is logged at the 'critical' level and :class:`DeprecatedConfig` will
        be raised.

        Otherwise, the message will be logged (once) at the 'warn' level.

        :raises: :class:`DeprecatedConfig` if the system is configured for
                 fatal deprecations.

        """
        stdmsg = _("Deprecated: %s") % msg
        if CONF.fatal_deprecations:
            self.critical(stdmsg, *args, **kwargs)
            raise DeprecatedConfig(msg=stdmsg)

        # Using a list because a tuple with dict can't be stored in a set.
        sent_args = self._deprecated_messages_sent.setdefault(msg, list())

        if args in sent_args:
            # Already logged this message, so don't log it again.
            return

        sent_args.append(args)
        self.warn(stdmsg, *args, **kwargs)

def bool_from_string(subject, strict=False, default=False):
    """Interpret a string as a boolean.

    A case-insensitive match is performed such that strings matching 't',
    'true', 'on', 'y', 'yes', or '1' are considered True and, when
    `strict=False`, anything else returns the value specified by 'default'.

    Useful for JSON-decoded stuff and config file parsing.

    If `strict=True`, unrecognized values, including None, will raise a
    ValueError which is useful when parsing values passed in from an API call.
    Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
    """
    if not isinstance(subject, six.string_types):
        subject = six.text_type(subject)

    lowered = subject.strip().lower()

    if lowered in TRUE_STRINGS:
        return True
    elif lowered in FALSE_STRINGS:
        return False
    elif strict:
        acceptable = ', '.join(
            "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
        msg = _("Unrecognized value '%(val)s', acceptable values are:"
                " %(acceptable)s") % {'val': subject,
                                      'acceptable': acceptable}
        raise ValueError(msg)
    else:
        return default

    def on_get(self, req, resp, project_id, queue_name):
        try:
            resp_dict = self.queue_ctrl.stats(queue_name,
                                              project=project_id)

            message_stats = resp_dict['messages']

            if message_stats['total'] != 0:
                base_path = req.path[:req.path.rindex('/')] + '/messages/'

                newest = message_stats['newest']
                newest['href'] = base_path + newest['id']
                del newest['id']

                oldest = message_stats['oldest']
                oldest['href'] = base_path + oldest['id']
                del oldest['id']

            resp.content_location = req.path
            resp.body = utils.to_json(resp_dict)
            # status defaults to 200

        except storage_errors.DoesNotExist as ex:
            LOG.debug(ex)
            raise falcon.HTTPNotFound()

        except Exception as ex:
            LOG.exception(ex)
            description = _(u'Queue stats could not be read.')
            raise wsgi_errors.HTTPServiceUnavailable(description)

    def on_put(self, req, resp, project_id, queue_name):
        LOG.debug(
            u"Queue metadata PUT - queue: %(queue)s, " u"project: %(project)s",
            {"queue": queue_name, "project": project_id},
        )

        try:
            # Place JSON size restriction before parsing
            self._validate.queue_metadata_length(req.content_length)
        except validation.ValidationFailed as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))

        # Deserialize queue metadata
        metadata, = wsgi_utils.filter_stream(req.stream, req.content_length, spec=None)

        try:
            self.queue_ctrl.set_metadata(queue_name, metadata=metadata, project=project_id)

        except validation.ValidationFailed as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))

        except storage_errors.QueueDoesNotExist:
            raise falcon.HTTPNotFound()

        except Exception as ex:
            LOG.exception(ex)
            description = _(u"Metadata could not be updated.")
            raise wsgi_errors.HTTPServiceUnavailable(description)

        resp.status = falcon.HTTP_204
        resp.location = req.path