def get_treefile(feed, tmp_dir, nectar_config):
    """
    Download the treefile and return its full path on disk, or None if not found

    :param feed:            URL to the repository
    :type  feed:            str
    :param tmp_dir:         full path to the temporary directory being used
    :type  tmp_dir:         str
    :param nectar_config:   download config to be used by nectar
    :type  nectar_config:   nectar.config.DownloaderConfig

    :return:        full path to treefile on disk, or None if not found
    :rtype:         str or NoneType
    """
    for filename in constants.TREE_INFO_LIST:
        path = os.path.join(tmp_dir, filename)
        url = os.path.join(feed, filename)
        request = DownloadRequest(url, path)
        listener = AggregatingEventListener()
        downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
        downloader.download([request])
        if len(listener.succeeded_reports) == 1:
            # bz 1095829
            strip_treeinfo_repomd(path)
            return path

    def __init__(self, repo_url, dst_dir, nectar_config, url_modify=None):
        """
        :param repo_url:        URL for the base of a yum repository
        :type  repo_url:        basestring
        :param dst_dir:         full path to a destination to which files
                                should be downloaded
        :type  dst_dir:         basestring
        :param nectar_config:   download config for nectar
        :type  nectar_config:   nectar.config.DownloaderConfig
        :param url_modify:      Optional URL modifier
        :type  url_modify:      pulp_rpm.plugins.importers.yum.utils.RepoURLModifier
        """
        super(MetadataFiles, self).__init__()

        self._url_modify = url_modify or utils.RepoURLModifier()
        self.repo_url = self._url_modify(repo_url)
        self.dst_dir = dst_dir
        self.event_listener = AggregatingEventListener()

        self.downloader = nectar_factory.create_downloader(self.repo_url, nectar_config,
                                                           self.event_listener)

        self.revision = None
        self.metadata = {}
        self.dbs = {}
        self.rpm_count = None

def get_distribution_file(feed, tmp_dir, nectar_config):
    """
    Download the pulp_distribution.xml and return its full path on disk, or None if not found

    :param feed:            URL to the repository
    :type  feed:            str
    :param tmp_dir:         full path to the temporary directory being used
    :type  tmp_dir:         str
    :param nectar_config:   download config to be used by nectar
    :type  nectar_config:   nectar.config.DownloaderConfig

    :return:        full path to distribution file on disk, or None if not found
    :rtype:         str or NoneType
    """
    filename = constants.DISTRIBUTION_XML

    path = os.path.join(tmp_dir, filename)
    url = os.path.join(feed, filename)
    request = DownloadRequest(url, path)
    listener = AggregatingEventListener()
    downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
    downloader.download([request])
    if len(listener.succeeded_reports) == 1:
        return path

    return None

    def download_files(self, tmp_dir, files):
        """
        Download distribution files.

        :param tmp_dir: The absolute to where the downloaded files.
        :type tmp_dir: str
        :param files: List of distribution dictionary files.
        :type files: list
        :return: generator of: (destination, location)
            The *destination* is the absolute path to the downloaded file.
            The *location* is the relative path within the tmp_dir.
        :rtype: generator
        :raise DownloadFailed: if any of the downloads fail.
        """
        listener = DistFileListener(self)
        self.progress_report.set_initial_values(len(files))
        downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
        requests = (self.file_to_download_request(f, tmp_dir) for f in files)
        downloader.download(requests)
        if len(listener.failed_reports):
            _logger.error(_('some distro file downloads failed'))
            self.progress_report['state'] = constants.STATE_FAILED
            self.progress_report['error_details'] = [
                (fail.url, fail.error_report) for fail in listener.failed_reports
            ]
            raise DownloadFailed()
        for report in listener.succeeded_reports:
            location = os.path.relpath(report.destination, tmp_dir)
            yield report.destination, location

    def _parse_as_mirrorlist(self, feed):
        """
        Treats the provided feed as mirrorlist. Parses its content and extracts
        urls to sync.

        :param feed: feed that should be treated as mirrorlist
        :type:       str

        :return:    list the URLs received from the mirrorlist
        :rtype:     list
        """
        url_file = StringIO()
        downloader = nectar_factory.create_downloader(feed, self.nectar_config)
        request = DownloadRequest(feed, url_file)
        downloader.download_one(request)
        url_file.seek(0)
        url_parse = url_file.read().split('\n')
        repo_url = []
        # Due to the fact, that format of mirrorlist can be different, this regex
        # matches the cases when the url is not commented out and does not have any
        # punctuation characters in front.
        pattern = re.compile("(^|^[\w\s=]+\s)((http(s)?)://.*)")
        for line in url_parse:
            for match in re.finditer(pattern, line):
                repo_url.append(match.group(2))
        random.shuffle(repo_url)
        return repo_url

    def __init__(self, repo_url, nectar_config, package_model_iterator, dst_dir,
                 event_listener=None):
        self.repo_url = repo_url
        self.package_model_iterator = package_model_iterator
        self.dst_dir = dst_dir

        self.downloader = nectar_factory.create_downloader(repo_url, nectar_config,
                                                           event_listener)

def sync(sync_conduit, feed, working_dir, nectar_config, report, progress_callback):
    """
    Look for a distribution in the target repo and sync it if found

    :param sync_conduit:        conduit provided by the platform
    :type  sync_conduit:        pulp.plugins.conduits.repo_sync.RepoSyncConduit
    :param feed:                URL of the yum repo being sync'd
    :type  feed:                basestring
    :param working_dir:         full path to the directory to which files
                                should be downloaded
    :type  working_dir:         basestring
    :param nectar_config:       download config to be used by nectar
    :type  nectar_config:       nectar.config.DownloaderConfig
    :param report:              progress report object
    :type  report:              pulp_rpm.plugins.importers.yum.report.DistributionReport
    :param progress_callback:   function that takes no arguments but induces
                                the current progress report to be sent.
    """
    # this temporary dir will hopefully be moved to the unit's storage path
    # if all downloads go well. If not, it will be deleted below, ensuring a
    # complete cleanup
    tmp_dir = tempfile.mkdtemp(dir=working_dir)
    try:
        treefile_path = get_treefile(feed, tmp_dir, nectar_config)
        if not treefile_path:
            _LOGGER.debug('no treefile found')
            report['state'] = constants.STATE_COMPLETE
            return

        try:
            model, files = parse_treefile(treefile_path)
        except ValueError:
            _LOGGER.error('could not parse treefile')
            report['state'] = constants.STATE_FAILED
            return

        report.set_initial_values(len(files))
        listener = DistroFileListener(report, progress_callback)
        downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
        _LOGGER.debug('downloading distribution files')
        downloader.download(file_to_download_request(f, feed, tmp_dir) for f in files)
        if len(listener.failed_reports) == 0:
            unit = sync_conduit.init_unit(ids.TYPE_ID_DISTRO, model.unit_key, model.metadata, model.relative_path)
            model.process_download_reports(listener.succeeded_reports)
            # remove pre-existing dir
            shutil.rmtree(unit.storage_path, ignore_errors=True)
            shutil.move(tmp_dir, unit.storage_path)
            # mkdtemp is very paranoid, so we'll change to more sensible perms
            os.chmod(unit.storage_path, 0o775)
            sync_conduit.save_unit(unit)
        else:
            _LOGGER.error('some distro file downloads failed')
            report['state'] = constants.STATE_FAILED
            report['error_details'] = [(fail.url, fail.error_report) for fail in listener.failed_reports]
            return
        report['state'] = constants.STATE_COMPLETE
    finally:
        shutil.rmtree(tmp_dir, ignore_errors=True)

 def get_downloader(self, conduit, config, url):
     """
     Get object suitable for downloading content published
     in the content catalog by a content source.
     :param conduit: Access to pulp platform API.
     :type conduit: pulp.server.plugins.conduits.cataloger.CatalogerConduit
     :param config: The content source configuration.
     :type config: dict
     :param url: The URL for the content source.
     :type url: str
     :return: A configured downloader.
     :rtype: nectar.downloaders.base.Downloader
     """
     return nectar_factory.create_downloader(url, self.nectar_config(config))

 def __init__(self, base_url, nectar_conf, units, dst_dir, listener):
     """
     :param base_url: The repository base url.
     :type base_url: str
     :param units: An iterable of units to download.
     :type units: iterable
     :param dst_dir: The absolute path to where the packages are to be downloaded.
     :type dst_dir: str
     :param listener: A nectar listener.
     :type listener: nectar.listener.DownloadListener
     """
     self.base_url = base_url
     self.units = units
     self.dst_dir = dst_dir
     self.listener = ContainerListener(listener)
     self.primary = create_downloader(base_url, nectar_conf)
     self.container = ContentContainer()
     self.canceled = Event()

    def get_distribution_file(self, tmp_dir):
        """
        Download the pulp_distribution.xml and return its full path on disk, or None if not found

        :param tmp_dir: The absolute path to the temporary directory
        :type tmp_dir: str
        :return: The absolute path to distribution file on disk, or None if not found
        :rtype: str or None
        """
        filename = constants.DISTRIBUTION_XML
        path = os.path.join(tmp_dir, filename)
        url = os.path.join(self.feed, filename)
        request = DownloadRequest(url, path)
        listener = AggregatingEventListener()
        downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
        downloader.download([request])
        if len(listener.succeeded_reports) == 1:
            return path
        return None

    def get_treefile(self, tmp_dir):
        """
        Download the treefile and return its full path on disk, or None if not found

        :param tmp_dir: The absolute path to the temporary directory
        :type tmp_dir: str
        :return: The absolute path to treefile on disk, or None if not found
        :rtype: str or None
        """
        for filename in constants.TREE_INFO_LIST:
            path = os.path.join(tmp_dir, filename)
            url = os.path.join(self.feed, filename)
            request = DownloadRequest(url, path)
            listener = AggregatingEventListener()
            downloader = nectar_factory.create_downloader(self.feed, self.nectar_config, listener)
            downloader.download([request])
            if len(listener.succeeded_reports) == 1:
                # bz 1095829
                self.strip_treeinfo_repomd(path)
                return path

    def __init__(self, repo_url, dst_dir, nectar_config):
        """
        :param repo_url:        URL for the base of a yum repository
        :type  repo_url:        basestring
        :param dst_dir:         full path to a destination to which files
                                should be downloaded
        :type  dst_dir:         basestring
        :param nectar_config:   download config for nectar
        :type  nectar_config:   nectar.config.DownloaderConfig
        """
        super(MetadataFiles, self).__init__()
        self.repo_url = repo_url
        self.dst_dir = dst_dir
        self.event_listener = AggregatingEventListener()

        self.downloader = nectar_factory.create_downloader(repo_url, nectar_config, self.event_listener)

        self.revision = None
        self.metadata = {}
        self.dbs = {}

 def __init__(self, base_url, nectar_conf, units, dst_dir, listener, url_modify=None):
     """
     :param base_url: The repository base url.
     :type base_url: str
     :param units: An iterable of units to download.
     :type units: iterable
     :param dst_dir: The absolute path to where the packages are to be downloaded.
     :type dst_dir: str
     :param listener: A nectar listener.
     :type listener: nectar.listener.DownloadListener
     :param url_modify: Optional URL modifier
     :type url_modify: pulp_rpm.plugins.importers.yum.utils.RepoURLModifier
     """
     self.base_url = base_url
     self.units = units
     self.dst_dir = dst_dir
     self.listener = ContainerListener(listener)
     self.primary = create_downloader(base_url, nectar_conf)
     self.container = ContentContainer()
     self.url_modify = url_modify or RepoURLModifier()

def sync(sync_conduit, feed, working_dir, nectar_config, report, progress_callback):
    """
    Look for a distribution in the target repo and sync it if found

    :param sync_conduit:        conduit provided by the platform
    :type  sync_conduit:        pulp.plugins.conduits.repo_sync.RepoSyncConduit
    :param feed:                URL of the yum repo being sync'd
    :type  feed:                basestring
    :param working_dir:         full path to the directory to which files
                                should be downloaded
    :type  working_dir:         basestring
    :param nectar_config:       download config to be used by nectar
    :type  nectar_config:       nectar.config.DownloaderConfig
    :param report:              progress report object
    :type  report:              pulp_rpm.plugins.importers.yum.report.DistributionReport
    :param progress_callback:   function that takes no arguments but induces
                                the current progress report to be sent.
    """
    # this temporary dir will hopefully be moved to the unit's storage path
    # if all downloads go well. If not, it will be deleted below, ensuring a
    # complete cleanup
    tmp_dir = tempfile.mkdtemp(dir=working_dir)
    try:
        treefile_path = get_treefile(feed, tmp_dir, nectar_config)
        if not treefile_path:
            _LOGGER.debug('no treefile found')
            return

        try:
            model, files = parse_treefile(treefile_path)
        except ValueError:
            _LOGGER.error('could not parse treefile')
            report['state'] = constants.STATE_FAILED
            return

        distribution_type_criteria = UnitAssociationCriteria(type_ids=[ids.TYPE_ID_DISTRO])
        existing_units = sync_conduit.get_units(criteria=distribution_type_criteria)

        # skip this whole process if the upstream treeinfo file hasn't changed
        if len(existing_units) == 1 and existing_distribution_is_current(existing_units[0], model):
            _LOGGER.debug('upstream distribution unchanged; skipping')
            return

        # Get any errors
        dist_files = process_distribution(feed, tmp_dir, nectar_config, model, report)
        files.extend(dist_files)

        report.set_initial_values(len(files))
        listener = DistroFileListener(report, progress_callback)
        downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
        _LOGGER.debug('downloading distribution files')
        downloader.download(file_to_download_request(f, feed, tmp_dir) for f in files)
        if len(listener.failed_reports) == 0:
            unit = sync_conduit.init_unit(ids.TYPE_ID_DISTRO, model.unit_key, model.metadata,
                                          model.relative_path)
            model.process_download_reports(listener.succeeded_reports)
            # remove pre-existing dir
            shutil.rmtree(unit.storage_path, ignore_errors=True)
            pulp_copytree(tmp_dir, unit.storage_path)
            # mkdtemp is very paranoid, so we'll change to more sensible perms
            os.chmod(unit.storage_path, 0o775)
            sync_conduit.save_unit(unit)
            # find any old distribution units and remove them. See BZ #1150714
            for existing_unit in existing_units:
                if existing_unit != unit:
                    _LOGGER.info("Removing out-of-date distribution unit %s for repo %s" %
                                 (existing_unit.unit_key, sync_conduit.repo_id))
                    sync_conduit.remove_unit(existing_unit)
        else:
            _LOGGER.error('some distro file downloads failed')
            report['state'] = constants.STATE_FAILED
            report['error_details'] = [(fail.url, fail.error_report) for fail in
                                       listener.failed_reports]
            return
    finally:
        shutil.rmtree(tmp_dir, ignore_errors=True)

 def test_file_url(self):
     downloader = nectar_factory.create_downloader('file:///foo', self.mock_config,
                                                   self.mock_event_listener)
     self.assertTrue(isinstance(downloader, HTTPCurlDownloader))

def sync(repo, sync_conduit, feed, working_dir, nectar_config, report, progress_callback):
    """
    Look for a distribution in the target repo and sync it if found

    :param repo: The repository that is the target of the sync
    :type repo: pulp.server.db.model.Repository
    :param sync_conduit:        conduit provided by the platform
    :type  sync_conduit:        pulp.plugins.conduits.repo_sync.RepoSyncConduit
    :param feed:                URL of the yum repo being sync'd
    :type  feed:                basestring
    :param working_dir:         full path to the directory to which files
                                should be downloaded
    :type  working_dir:         basestring
    :param nectar_config:       download config to be used by nectar
    :type  nectar_config:       nectar.config.DownloaderConfig
    :param report:              progress report object
    :type  report:              pulp_rpm.plugins.importers.yum.report.DistributionReport
    :param progress_callback:   function that takes no arguments but induces
                                the current progress report to be sent.
    """
    tmp_dir = tempfile.mkdtemp(dir=working_dir)
    try:
        treefile_path = get_treefile(feed, tmp_dir, nectar_config)
        if not treefile_path:
            _LOGGER.debug('no treefile found')
            return

        try:
            model, files = parse_treefile(treefile_path)
        except ValueError:
            _LOGGER.error('could not parse treefile')
            report['state'] = constants.STATE_FAILED
            return

        existing_units = repo_controller.find_repo_content_units(
            repo, repo_content_unit_q=mongoengine.Q(unit_type_id=ids.TYPE_ID_DISTRO),
            yield_content_unit=True)
        existing_units = list(existing_units)

        # skip this whole process if the upstream treeinfo file hasn't changed
        if len(existing_units) == 1 and existing_distribution_is_current(existing_units[0], model):
            _LOGGER.debug('upstream distribution unchanged; skipping')
            return

        # Get any errors
        dist_files = process_distribution(feed, tmp_dir, nectar_config, model, report)
        files.extend(dist_files)

        report.set_initial_values(len(files))
        listener = DistroFileListener(report, progress_callback)
        downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
        _LOGGER.debug('downloading distribution files')
        downloader.download(file_to_download_request(f, feed, tmp_dir) for f in files)
        if len(listener.failed_reports) == 0:
            model.set_content(tmp_dir)
            model.save()
            # The save sets the content path, which is needed to generate the download_reports
            # Long term this should be done by a serializer
            model.process_download_reports(listener.succeeded_reports)
            model.save()

            repo_controller.associate_single_unit(repo, model)

            # find any old distribution units and remove them. See BZ #1150714
            for existing_unit in existing_units:
                if existing_unit != model:
                    _LOGGER.info("Removing out-of-date distribution unit %s for repo %s" %
                                 (existing_unit.unit_key, sync_conduit.repo_id))
                    platform_models.RepositoryContentUnit.objects(
                        repo_id=sync_conduit.repo_id,
                        unit_type_id=models.Distribution._content_type_id.default,
                        unit_id=existing_unit.id).delete()
        else:
            _LOGGER.error('some distro file downloads failed')
            report['state'] = constants.STATE_FAILED
            report['error_details'] = [(fail.url, fail.error_report) for fail in
                                       listener.failed_reports]
            return
    finally:
        shutil.rmtree(tmp_dir, ignore_errors=True)

 def test_https_url(self):
     downloader = nectar_factory.create_downloader('https://foo', self.mock_config,
                                                   self.mock_event_listener)
     self.assertTrue(isinstance(downloader, HTTPThreadedDownloader))