def increment_failure_count(schedule_id):
    """
    Increment the number of consecutive failures, and if it has met or exceeded
    the threshold, disable the schedule.

    :param schedule_id: ID of the schedule whose count should be incremented
    :type  schedule_id: str
    """
    try:
        spec = {'_id': ObjectId(schedule_id)}
    except InvalidId:
        raise exceptions.InvalidValue(['schedule_id'])
    delta = {
        '$inc': {'consecutive_failures': 1},
        '$set': {'last_updated': time.time()},
    }
    schedule = ScheduledCall.get_collection().find_and_modify(
        query=spec, update=delta, new=True)
    if schedule:
        scheduled_call = ScheduledCall.from_db(schedule)
        if scheduled_call.failure_threshold is None or not scheduled_call.enabled:
            return
        if scheduled_call.consecutive_failures >= scheduled_call.failure_threshold:
            _logger.info(_('disabling schedule %(id)s with %(count)d consecutive failures') % {
                'id': schedule_id, 'count': scheduled_call.consecutive_failures
            })
            delta = {'$set': {
                'enabled': False,
                'last_updated': time.time(),
            }}
            ScheduledCall.get_collection().update(spec, delta)

 def tearDown(self):
     super(SchedulerTests, self).tearDown()
     ScheduledCall.get_collection().drop()
     self.scheduler = None
     dispatch_factory.coordinator = self._coordinator_factory
     self._coordinator_factory = None
     dispatch_factory._SCHEDULER = None

def update(schedule_id, delta):
    """
    Updates the schedule with unique ID schedule_id. This only allows updating
    of fields in ScheduledCall.USER_UPDATE_FIELDS.

    :param schedule_id: a unique ID for a schedule
    :type  schedule_id: basestring
    :param delta:       a dictionary of keys with values that should be modified
                        on the schedule.
    :type  delta:       dict

    :return:    instance of ScheduledCall representing the post-update state
    :rtype      ScheduledCall

    :raise  exceptions.UnsupportedValue
    :raise  exceptions.MissingResource
    """
    unknown_keys = set(delta.keys()) - ScheduledCall.USER_UPDATE_FIELDS
    if unknown_keys:
        raise exceptions.UnsupportedValue(list(unknown_keys))

    delta['last_updated'] = time.time()

    try:
        spec = {'_id': ObjectId(schedule_id)}
    except InvalidId:
        raise exceptions.InvalidValue(['schedule_id'])
    schedule = ScheduledCall.get_collection().find_and_modify(
        query=spec, update={'$set': delta}, safe=True, new=True)
    if schedule is None:
        raise exceptions.MissingResource(schedule_id=schedule_id)
    return ScheduledCall.from_db(schedule)

    def test_since_first(self):
        call = ScheduledCall('2014-01-03T10:15Z/PT1H', 'pulp.tasks.dosomething')

        since_first = call._calculate_times()[2]
        now = time.time()

        self.assertTrue(since_first + 1388744100 - now < 1)

    def test_get(self, mock_path, mock_ok, mock_utils_get):
        call = ScheduledCall('PT1M', 'pulp.tasks.frequent')
        mock_utils_get.return_value = [call]

        ret = self.controller._get(call.id)
        schedule = mock_ok.call_args[0][0]

        self.assertEqual(ret, mock_ok.return_value)
        self.assertEqual(len(mock_ok.call_args[0]), 1)

        # spot-check the schedule
        self.assertEqual(schedule['_id'], call.id)
        self.assertEqual(schedule['schedule'], 'PT1M')
        self.assertEqual(schedule['task'], 'pulp.tasks.frequent')
        self.assertEqual(schedule['_href'], mock_path.return_value)

        # next_run is calculated on-demand, and there is a small chance that it
        # will be re-calculated in the call.for_display() call as 1 second later
        # than it was calculated above. Thus we will test that equality here
        # with a tolerance of 1 second
        for_display = call.for_display()
        call_next_run = dateutils.parse_iso8601_datetime(call.next_run)
        display_next_run = dateutils.parse_iso8601_datetime(for_display['next_run'])
        self.assertTrue(display_next_run - call_next_run <= timedelta(seconds=1))

        # now check overall equality with the actual for_display value
        del schedule['_href']
        del schedule['next_run']
        del for_display['next_run']
        self.assertEqual(schedule, for_display)

        # make sure we called the manager layer correctly
        mock_utils_get.assert_called_once_with([call.id])

    def test_now(self):
        call = ScheduledCall('PT1H', 'pulp.tasks.dosomething')

        now = datetime.utcnow().replace(tzinfo=dateutils.utc_tz())
        next_run = dateutils.parse_iso8601_datetime(call.calculate_next_run())

        self.assertTrue(next_run - now < timedelta(seconds=1))

    def test_run_every(self):
        call = ScheduledCall('2014-01-03T10:15Z/PT1H', 'pulp.tasks.dosomething')

        run_every_s = call._calculate_times()[3]

        # 1 hour, as specified in the ISO8601 string above
        self.assertEqual(run_every_s, 3600)

    def test_last_scheduled_run_no_first_run(self):
        call = ScheduledCall('PT1H', 'pulp.tasks.dosomething')

        last_scheduled_run_s = call._calculate_times()[4]
        first_run_s = call._calculate_times()[1]

        self.assertEqual(last_scheduled_run_s, first_run_s)

    def test_no_last_run(self):
        call = ScheduledCall('PT1M', 'pulp.tasks.dosomething')

        entry = call.as_schedule_entry()

        # celery actually calculates it, so we don't need to test the value
        self.assertTrue(isinstance(entry.last_run_at, datetime))

    def test_first_run_now(self):
        call = ScheduledCall('PT1M', 'pulp.tasks.dosomething')

        first_run_s = call._calculate_times()[1]

        # make sure this gives us a timestamp that reasonably represents "now"
        self.assertTrue(time.time() - first_run_s < 1)

def delete_by_resource(resource):
    """
    Deletes all schedules for the specified resource

    :param resource:    string indicating a unique resource
    :type  resource:    basestring
    """
    ScheduledCall.get_collection().remove({'resource': resource}, safe=True)

    def test_last_scheduled_run_with_first_run(self, mock_time):
        # specify a start time and current time such that we know the difference
        mock_time.return_value = 1389307330.966561
        call = ScheduledCall('2014-01-09T17:15Z/PT1H', 'pulp.tasks.dosomething')

        last_scheduled_run_s = call._calculate_times()[4]

        self.assertEqual(last_scheduled_run_s, 1389305700)

    def test_first_run_scheduled(self):
        call = ScheduledCall('2014-01-03T10:15Z/PT1H', 'pulp.tasks.dosomething')

        first_run_s = call._calculate_times()[1]

        # make sure this gives us a timestamp for the date and time
        # specified above
        self.assertEqual(first_run_s, 1388744100)

    def test_no_runs(self):
        call = ScheduledCall('PT1H', 'pulp.tasks.dosomething')
        entry = call.as_schedule_entry()

        is_due, seconds = entry.is_due()

        self.assertTrue(is_due)
        # make sure this is very close to one hour
        self.assertTrue(3600 - seconds < 1)

    def test_expected_runs_positive(self, mock_time):
        # specify a start time and current time such that we know the difference
        mock_time.return_value = 1389307330.966561
        call = ScheduledCall('2014-01-09T17:15Z/PT1H', 'pulp.tasks.dosomething')

        expected_runs = call._calculate_times()[5]

        # we know that it's been more than 5 hours since the first scheduled run
        self.assertEqual(expected_runs, 5)

    def test_new(self, mock_get_collection):
        mock_insert = mock_get_collection.return_value.insert
        call = ScheduledCall('PT1M', 'pulp.tasks.dosomething')

        call.save()

        expected = call.as_dict()
        expected['_id'] = bson.ObjectId(expected['_id'])
        mock_insert.assert_called_once_with(expected, safe=True)
        self.assertFalse(call._new)

    def test_existing(self, mock_get_collection):
        mock_update = mock_get_collection.return_value.update
        fake_id = bson.ObjectId()
        call = ScheduledCall('PT1M', 'pulp.tasks.dosomething', id=fake_id)

        call.save()

        expected = call.as_dict()
        del expected['_id']
        mock_update.assert_called_once_with({'_id': fake_id}, expected)

    def test_expected_runs_future(self, mock_time):
        # specify a start time and current time such that the start appears to
        # be in the future
        mock_time.return_value = 1389307330.966561
        call = ScheduledCall('2014-01-19T17:15Z/PT1H', 'pulp.tasks.dosomething')

        expected_runs = call._calculate_times()[5]

        # the first run is scheduled in the future (relative to the mock time),
        # so there should not be any runs.
        self.assertEqual(expected_runs, 0)

    def create_schedule(cls, action, consumer_id, units, options, schedule, failure_threshold=None, enabled=True):
        """
        Creates a new schedule for a consumer action

        :param action:          a unique identified for an action, one of
                                UNIT_INSTALL_ACTION, UNIT_UPDATE_ACTION,
                                UNIT_UNINSTALL_ACTION
        :type  action:          basestring
        :param consumer_id:     a unique ID for a consumer
        :type  consumer_id:     basestring
        :param units:           A list of content units to be installed, each as
                                a dict in the form:
                                    { type_id:<str>, unit_key:<dict> }
        :type  units:           list
        :param options:         a dictionary that will be passed to the
                                action-appropriate task as the "options"
                                argument
        :type  options:         dict
        :param schedule:        ISO8601 string representation of the schedule
        :type  schedule:        basestring
        :param failure_threshold:   optional positive integer indicating how
                                many times this schedule's execution can fail
                                before being automatically disabled.
        :type  failure_threshold:   int or NoneType
        :param enabled:         boolean indicating if this schedule should
                                be actively loaded and executed by the
                                scheduler. Defaults to True.
        :type  enabled:         bool
        :return:    instance of the new ScheduledCal
        :rtype:     pulp.server.db.models.dispatch.ScheduledCall

        :raise:     pulp.server.exceptions.MissingValue
        """
        cls._validate_consumer(consumer_id)
        utils.validate_initial_schedule_options(schedule, failure_threshold, enabled)
        if not units:
            raise MissingValue(["units"])

        task = ACTIONS_TO_TASKS[action]
        args = [consumer_id]
        kwargs = {"units": units, "options": options}
        resource = Consumer.build_resource_tag(consumer_id)

        schedule = ScheduledCall(
            schedule,
            task,
            args=args,
            kwargs=kwargs,
            resource=resource,
            failure_threshold=failure_threshold,
            enabled=enabled,
        )
        schedule.save()
        return schedule

    def test_future(self, mock_time):
        mock_time.return_value = 1389307330.966561
        call = ScheduledCall('2014-01-19T17:15Z/PT1H', 'pulp.tasks.dosomething')

        next_run = call.calculate_next_run()

        # make sure the next run is equal to the specified first run.
        # don't want to compare a generated ISO8601 string directly, because there
        # could be subtle variations that are valid but break string equality.
        self.assertEqual(dateutils.parse_iso8601_interval(call.iso_schedule)[1],
                         dateutils.parse_iso8601_datetime(next_run))