def delete_orphan_content_units_by_type(type_id, content_unit_ids=None):
        """
        Delete the orphaned content units for the given content type.
        This method only applies to new style content units that are loaded via entry points

        NOTE: this method deletes the content unit's bits from disk, if applicable.

        :param type_id: id of the content type
        :type type_id: basestring
        :param content_unit_ids: list of content unit ids to delete; None means delete them all
        :type content_unit_ids: iterable or None
        :return: count of units deleted
        :rtype: int
        """
        # get the model matching the type
        content_model = plugin_api.get_unit_model_by_id(type_id)
        if content_unit_ids:
            query_sets = []
            for page in plugin_misc.paginate(content_unit_ids):
                qs = content_model.objects(id__in=page).only('id', '_storage_path')
                query_sets.append(qs)
            content_units = itertools.chain(*query_sets)
        else:
            content_units = content_model.objects.only('id', '_storage_path')

        count = 0

        # Paginate the content units
        for units_group in plugin_misc.paginate(content_units):
            # Build the list of ids to search for an easier way to access units in the group by id
            unit_dict = dict()
            for unit in units_group:
                unit_dict[unit.id] = unit

            id_list = list(unit_dict.iterkeys())

            # Clear the units that are currently associated from unit_dict
            non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
                .distinct('unit_id')
            for non_orphan_id in non_orphan:
                unit_dict.pop(non_orphan_id)

            # Remove the unit, lazy catalog entries, and any content in storage.
            for unit_to_delete in unit_dict.itervalues():
                model.LazyCatalogEntry.objects(
                    unit_id=str(unit_to_delete.id),
                    unit_type_id=str(type_id)
                ).delete()
                unit_to_delete.delete()
                if unit_to_delete._storage_path:
                    OrphanManager.delete_orphaned_file(unit_to_delete._storage_path)
                count += 1

        return count

def get_existing_units(search_dicts, unit_class, repo):
    """
    Get units from the given repository that match the search terms. The unit instances will only
    have their unit key fields populated.

    :param search_dicts:    iterable of dictionaries that should be used to search units
    :type  search_dicts:    iterable
    :param unit_class:      subclass representing the type of unit to search for
    :type  unit_class:      pulp_rpm.plugins.db.models.Package
    :param repo:            repository to search in
    :type  repo:            pulp.server.db.model.Repository

    :return:    generator of unit_class instances with only their unit key fields populated
    :rtype:     generator
    """
    unit_fields = unit_class.unit_key_fields
    for segment in paginate(search_dicts):
        unit_filters = {'$or': list(segment)}
        units_q = mongoengine.Q(__raw__=unit_filters)
        association_q = mongoengine.Q(unit_type_id=unit_class._content_type_id.default)

        for result in repo_controller.find_repo_content_units(repo, units_q=units_q,
                                                              repo_content_unit_q=association_q,
                                                              unit_fields=unit_fields,
                                                              yield_content_unit=True):
            yield result

    def delete_orphan_content_units_by_type(type_id):
        """
        Delete the orphaned content units for the given content type.
        This method only applies to new style content units that are loaded via entry points

        NOTE: this method deletes the content unit's bits from disk, if applicable.

        :param type_id: id of the content type
        :type type_id: basestring
        """
        # get the model matching the type
        content_model = plugin_api.get_unit_model_by_id(type_id)
        content_units = content_model.objects().only('id', 'storage_path')

        # Paginate the content units
        for units_group in plugin_misc.paginate(content_units):
            # Build the list of ids to search for an easier way to access units in the group by id
            unit_dict = dict()
            for unit in units_group:
                unit_dict[unit.id] = unit

            id_list = list(unit_dict.iterkeys())

            # Clear the units that are currently associated from unit_dict
            non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
                .distinct('unit_id')
            for non_orphan_id in non_orphan:
                unit_dict.pop(non_orphan_id)

            # Remove the unit and any references on disk
            for unit_to_delete in unit_dict.itervalues():
                unit_to_delete.delete()
                if unit_to_delete.storage_path:
                    OrphanManager.delete_orphaned_file(unit_to_delete.storage_path)

    def process_main(self, item=None):
        """
        given the passed-in unit keys, determine which of them already exist in
        pulp, and save those with the conduit found on the parent.

        :param item: The item to process or none if get_iterator is not defined
        :param item: object or None
        """
        # any units that are already in pulp
        units_we_already_had = set()

        # If available_units was defined in the constructor, let's use it. Otherwise let's use the
        # default of self.parent.available_units
        available_units = self.available_units or self.parent.available_units

        for units_group in misc.paginate(available_units, self.unit_pagination_size):
            # Get this group of units
            query = units_controller.find_units(units_group)

            for found_unit in query:
                units_we_already_had.add(hash(found_unit))
                repo_controller.associate_single_unit(self.get_repo().repo_obj, found_unit)

            for unit in units_group:
                if hash(unit) not in units_we_already_had:
                    self.units_to_download.append(unit)

def get_unit_model_querysets(repo_id, model_class, repo_content_unit_q=None):
    """
    Return a generator of mongoengine.queryset.QuerySet objects that collectively represent the
    units in the specified repo that are of the type corresponding to the model_class and that
    match the optional query.

    Results are broken up into multiple QuerySet objects, because units are requested by their ID,
    and we do not want to exceed the maximum size for a single query by stuffing too many IDs in one
    QuerySet object.

    You are welcome and encouraged to convert the return value into one generator of ContentUnit
    objects by using itertools.chain()

    :param repo_id:     ID of the repo whose units should be queried
    :type  repo_id:     str
    :param model_class: a subclass of ContentUnit that defines a unit model
    :type  model_class: pulp.server.db.model.ContentUnit
    :param repo_content_unit_q: any additional filters that should be applied to the
                                RepositoryContentUnit search
    :type  repo_content_unit_q: mongoengine.Q

    :return:    generator of mongoengine.queryset.QuerySet
    :rtype:     generator
    """
    for chunk in paginate(get_associated_unit_ids(repo_id,
                                                  model_class._content_type_id.default,
                                                  repo_content_unit_q)):
        yield model_class.objects(id__in=chunk)

def find_units(units, pagination_size=50):
    """
    Query for units matching the unit key fields of an iterable of ContentUnit objects.

    This requires that all the ContentUnit objects are of the same content type.

    :param units: Iterable of content units with the unit key fields specified.
    :type units: iterable of pulp.server.db.model.ContentUnit
    :param pagination_size: How large a page size to use when querying units.
    :type pagination_size: int (default 50)

    :returns: unit models that pulp already knows about.
    :rtype: Generator of pulp.server.db.model.ContentUnit
    """
    # get the class from the first unit
    model_class = None

    for units_group in misc.paginate(units, pagination_size):
        q_object = mongoengine.Q()
        # Build a query for the units in this group
        for unit in units_group:
            if model_class is None:
                model_class = unit.__class__

            # Build the query for all the units, the | operator here
            # creates the equivalent of a mongo $or of all the unit keys
            unit_q_obj = mongoengine.Q(**unit.unit_key)
            q_object = q_object | unit_q_obj

        # Get this group of units
        query = model_class.objects(q_object)

        for found_unit in query:
            yield found_unit

    def test_empty_list(self):
        ret = misc.paginate([], 3)

        self.assertTrue(inspect.isgenerator(ret))

        pieces = list(ret)

        self.assertEqual(pieces, [])

    def test_generator_one_page(self):
        iterable = (x for x in range(10))
        ret = misc.paginate(iterable, 100)

        self.assertTrue(inspect.isgenerator(ret))

        pieces = list(ret)

        self.assertEqual(pieces, [tuple(range(10))])

    def test_list(self):
        iterable = list(range(10))
        ret = misc.paginate(iterable, 3)

        self.assertTrue(inspect.isgenerator(ret))

        pieces = list(ret)

        self.assertEqual(pieces, [(0,1,2), (3,4,5), (6,7,8), (9,)])

def disassociate_units(repository, unit_iterable):
    """
    Disassociate all units in the iterable from the repository

    :param repository: The repository to update.
    :type repository: pulp.server.db.model.Repository
    :param unit_iterable: The units to disassociate from the repository.
    :type unit_iterable: iterable of pulp.server.db.model.ContentUnit
    """
    for unit_group in misc.paginate(unit_iterable):
        unit_id_list = [unit.id for unit in unit_group]
        qs = model.RepositoryContentUnit.objects(
            repo_id=repository.repo_id, unit_id__in=unit_id_list)
        qs.delete()

def get_existing_units(search_dicts, unit_fields, unit_type, search_method):
    """

    :param search_dicts:
    :param unit_fields:
    :param unit_type:
    :param search_method:
    :return:    generator of Units
    """
    for segment in paginate(search_dicts):
        unit_filters = {'$or': list(segment)}
        criteria = UnitAssociationCriteria([unit_type], unit_filters=unit_filters,
                                           unit_fields=unit_fields, association_fields=[])
        for result in search_method(criteria):
            yield result

    def get_requirements(self, units):
        """
        For an iterable of RPM Units, return a generator of Require() instances that
        represent the requirements for those RPMs.

        :param units:   iterable of pulp_rpm.plugins.models.RPM
        :type  units:   iterable

        :return:    generator of Require() instances
        :rtype:     generator
        """
        for segment in paginate(units):
            for unit in segment:
                for require in unit.requires or []:
                    yield Requirement(**require)

    def open_cursors(unit_ids):
        """
        Get a generator of unit cursors.

        :param unit_ids: A dictionary of unit_ids keyed by type_id.
        :type unit_ids: dict
        :return: A list of open cursors.
        :rtype: generator
        """
        for type_id, id_list in unit_ids.items():
            for page in paginate(id_list):
                query = {'_id': {'$in': page}}
                collection = type_units_collection(type_id)
                cursor = collection.find(query)
                yield cursor

    def get_requirements(self, units):
        """
        For an iterable of RPM Units, return a generator of Require() instances that
        represent the requirements for those RPMs.

        :param units:   iterable of RPMs for which a query should be performed to
                        retrieve their Requires entries.
        :type  units:   iterable of pulp.plugins.model.Unit

        :return:    generator of Require() instances
        :rtype:     generator
        """
        for segment in paginate(units):
            unit_ids = [unit.id for unit in segment]
            fields = ['requires', 'id']
            for result in models.RPM.objects.filter(id__in=unit_ids).only(*fields):
                for require in result.requires or []:
                    yield Requirement(**require)

    def get_requirements(self, units):
        """
        For an iterable of RPM Units, return a generator of Require() instances that
        represent the requirements for those RPMs.

        :param units:   iterable of pulp_rpm.plugins.models.RPM
        :type  units:   iterable

        :return:    generator of Require() instances
        :rtype:     generator
        """
        for segment in paginate(units):
            unit_ids = [unit.id for unit in segment]
            # The "requires" field is required by this workflow, but is not populated on the
            # incoming "units". Thus we need to re-fetch units with that field.
            fields = ['requires', 'id']
            for result in models.RPM.objects.filter(id__in=unit_ids).only(*fields):
                for require in result.requires or []:
                    yield Requirement(**require)

    def get_content_unit_ids(content_type, unit_keys):
        """
        Return a generator of ids that uniquely identify the content units that match the
        given unique keys dictionaries.

        :param content_type: unique id of content collection
        :type  content_type: str
        :param unit_keys: list of keys dictionaries that uniquely identify
                          content units in the given content type collection
        :type  unit_keys: list of dicts

        :return:    generator of unit IDs as strings
        :rtype:     generator
        """
        collection = content_types_db.type_units_collection(content_type)
        for segment in paginate(unit_keys):
            spec = _build_multi_keys_spec(content_type, segment)
            fields = ["_id"]
            for item in collection.find(spec, fields=fields):
                yield str(item["_id"])

    def get_requirements(self, units):
        """
        For an iterable of RPM Units, return a generator of Require() instances that
        represent the requirements for those RPMs.

        :param units:   iterable of RPMs for which a query should be performed to
                        retrieve their Requires entries.
        :type  units:   iterable of pulp.plugins.model.Unit

        :return:    generator of Require() instances
        :rtype:     generator
        """
        for segment in paginate(units):
            search_dicts = [unit.unit_key for unit in segment]
            filters = {'$or': search_dicts}
            fields = list(models.RPM.UNIT_KEY_NAMES)
            fields.extend(['requires', 'id'])
            criteria = UnitAssociationCriteria(type_ids=[models.RPM.TYPE], unit_filters=filters, unit_fields=fields)
            for result in self.search_method(criteria):
                for require in result.metadata.get('requires', []):
                    yield Requirement(**require)

 def get_multiple_units_by_keys_dicts(self, content_type, unit_keys_dicts, model_fields=None):
     """
     Look up multiple content units in the collection for the given content
     type collection that match the list of keys dictionaries.
     :param content_type: unique id of content collection
     :type content_type: str
     :param unit_keys_dicts: list of dictionaries whose key, value pairs can
                             uniquely identify a content unit
     :type unit_keys_dicts: list of dict's
     :param model_fields: fields of each content unit to report,
                          None means all fields
     :type model_fields: None or list of str's
     :return: tuple of content units found in the content type collection
              that match the given unit keys dictionaries
     :rtype: (possibly empty) tuple of dict's
     :raises ValueError: if any of the keys dictionaries are invalid
     """
     collection = content_types_db.type_units_collection(content_type)
     for segment in paginate(unit_keys_dicts, page_size=50):
         spec = _build_multi_keys_spec(content_type, segment)
         cursor = collection.find(spec, fields=model_fields)
         for unit_dict in cursor:
             yield unit_dict

    def queue_regenerate_applicability_for_repos(repo_criteria):
        """
        Queue a group of tasks to generate and save applicability data affected by given updated
        repositories.

        :param repo_criteria: The repo selection criteria
        :type repo_criteria: dict
        """
        repo_criteria = Criteria.from_dict(repo_criteria)

        # Process repo criteria
        repo_criteria.fields = ['id']
        repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]

        task_group_id = uuid4()

        for repo_id in repo_ids:
            profile_hashes = RepoProfileApplicability.get_collection().find(
                {'repo_id': repo_id}, {'profile_hash': 1})
            for batch in paginate(profile_hashes, 10):
                batch_regenerate_applicability_task.apply_async((repo_id, batch),
                                                                **{'group_id': task_group_id})
        return task_group_id

def get_all_existing_units(search_dicts, unit_fields, unit_type, search_method):
    """
    Get all existing units on the server which match given search_dicts using
    given search_method.

    :param search_dicts:  unit keys generator
    :type search_dicts:   iterator of unit keys
    :param unit_fields:   unit fields to be requested to the search_method
    :type unit_fields:    list or tuple
    :param unit_type:     unit type
    :type unit_type:      basestring
    :param search_method: search method to be used to search for non-repo-specific units
    :type search_method:  a search method accepting a unit type and
                          pulp.server.db.criteria.Criteria as parameters
    :return:              generator of Units found using the search_method
    :rtype:               iterator of pulp.plugins.model.Unit
    """
    # Instead of separate query for each unit, we are using paginate to query
    # for a lot of units at once.
    for segment in paginate(search_dicts):
        unit_filters = {'$or': list(segment)}
        criteria = Criteria(filters=unit_filters, fields=unit_fields)
        for result in search_method(unit_type, criteria):
            yield result