def __init__(self, repo, publish_conduit, config, repo_content_unit_q=None):
        """
        :param repo: Pulp managed Yum repository
        :type  repo: pulp.plugins.model.Repository
        :param publish_conduit: Conduit providing access to relative Pulp functionality
        :type  publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
        :param config: Pulp configuration for the distributor
        :type  config: pulp.plugins.config.PluginCallConfiguration
        :param repo_content_unit_q: optional Q object that will be applied to the queries performed
                                    against RepoContentUnit model
        :type  repo_content_unit_q: mongoengine.Q
        """
        super(WebPublisher, self).__init__(
            step_type=constants.PUBLISH_STEP_WEB_PUBLISHER, repo=repo,
            publish_conduit=publish_conduit, config=config)

        docker_api_version = 'v1'
        publish_dir = configuration.get_web_publish_dir(repo, config, docker_api_version)
        app_file = configuration.get_redirect_file_name(repo)
        app_publish_location = os.path.join(
            configuration.get_app_publish_dir(config, docker_api_version), app_file)
        self.working_dir = os.path.join(self.get_working_dir(), docker_api_version)
        misc.mkdir(self.working_dir)
        self.web_working_dir = os.path.join(self.get_working_dir(), 'web')
        master_publish_dir = configuration.get_master_publish_dir(repo, config, docker_api_version)
        atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(),
                                                         [('web', publish_dir),
                                                          (app_file, app_publish_location)],
                                                         master_publish_dir,
                                                         step_type=constants.PUBLISH_STEP_OVER_HTTP)
        atomic_publish_step.description = _('Making v1 files available via web.')
        self.add_child(PublishImagesStep(repo_content_unit_q=repo_content_unit_q))
        self.add_child(atomic_publish_step)

    def generate_download_requests(self):
        """
        generator that yields DownloadRequests for needed units.

        :return:    generator of DownloadRequest instances
        :rtype:     collections.Iterable[DownloadRequest]
        """
        download_url = self.get_config().get("download-root")

        for deb_unit in self.step_get_local_units.units_to_download:
            key_hash = deb_unit.unit_key_hash
            # Don't save all the units in one directory as there could be 50k + units
            hash_dir = generate_internal_storage_path(self.deb_data[key_hash]['file_name'])
            # make sure the download directory exists
            dest_dir = os.path.join(self.working_dir, hash_dir)
            download_dir = os.path.dirname(dest_dir)
            misc.mkdir(download_dir)
            file_path = self.deb_data[key_hash]['file_path']
            packages_url = urlparse.urljoin(download_url, file_path)
            
            # check that the package is not already saved on local disk
            test_path = "/var/lib/pulp/content/deb/" + generate_internal_storage_path(hash_dir)
            if os.path.exists(test_path):
                continue

            yield DownloadRequest(packages_url, dest_dir)

    def __init__(self, repo, conduit, config):
        """
        :param repo: Pulp managed Yum repository
        :type  repo: pulp.plugins.model.Repository
        :param conduit: Conduit providing access to relative Pulp functionality
        :type  conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
        :param config: Pulp configuration for the distributor
        :type  config: pulp.plugins.config.PluginCallConfiguration
        """
        super(WebPublisher, self).__init__(
            constants.PUBLISH_STEP_WEB_PUBLISHER,
            repo,
            conduit,
            config)

        publish_dir = configuration.get_web_publish_dir(repo, config)
        self.web_working_dir = os.path.join(self.get_working_dir(), repo.id)
        master_publish_dir = configuration.get_master_publish_dir(repo, config)
        atomic_publish = AtomicDirectoryPublishStep(
            self.get_working_dir(),
            [(repo.id, publish_dir)],
            master_publish_dir,
            step_type=constants.PUBLISH_STEP_OVER_HTTP)

        atomic_publish.description = _('Making files available via web.')

        main = MainStep()
        self.add_child(main)
        self.add_child(atomic_publish)
        mkdir(self.web_working_dir)

 def __init__(self, repo, conduit, config, working_dir=None, **kwargs):
     """
     :param repo: The repository being published.
     :type  repo: pulp.plugins.model.Repository
     :param conduit: Conduit providing access to relative Pulp functionality
     :type  conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
     :param config: Pulp configuration for the distributor
     :type  config: pulp.plugins.config.PluginCallConfiguration
     :param working_dir: The temp directory this step should use for processing.
     :type  working_dir: str
     """
     super(WebPublisher, self).__init__(
         step_type=constants.PUBLISH_STEP_WEB_PUBLISHER,
         repo=repo,
         conduit=conduit,
         config=config,
         working_dir=working_dir,
         plugin_type=constants.WEB_DISTRIBUTOR_TYPE_ID,
         **kwargs)
     self.publish_dir = os.path.join(self.get_working_dir(), repo.id)
     atomic_publish = AtomicDirectoryPublishStep(
         self.get_working_dir(),
         [(repo.id, configuration.get_web_publish_dir(repo.repo_obj, config))],
         configuration.get_master_publish_dir(repo.repo_obj, config),
         step_type=constants.PUBLISH_STEP_OVER_HTTP)
     atomic_publish.description = _('Making files available via web.')
     main = MainStep()
     self.add_child(main)
     self.add_child(atomic_publish)
     mkdir(self.publish_dir)

    def migrate(plan, unit_id, path, new_path):
        """
        Migrate the unit.
          1. move content
          2. update the DB

        :param plan: A plan object.
        :type plan: Plan
        :param unit_id: A unit UUID.
        :type unit_id: str
        :param path: The current storage path.
        :type path: str
        :param new_path: The new storage path.
        :type new_path: str
        """
        if os.path.exists(path):
            mkdir(os.path.dirname(new_path))
            shutil.move(path, new_path)
        plan.collection.update_one(
            filter={
                '_id': unit_id
            },
            update={
                '$set': {'_storage_path': new_path}
            })

    def __init__(self, repo, publish_conduit, config):
        """
        :param repo: Pulp managed Yum repository
        :type  repo: pulp.plugins.model.Repository
        :param publish_conduit: Conduit providing access to relative Pulp functionality
        :type  publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
        :param config: Pulp configuration for the distributor
        :type  config: pulp.plugins.config.PluginCallConfiguration
        """
        super(WebPublisher, self).__init__(constants.PUBLISH_STEP_WEB_PUBLISHER, repo, publish_conduit, config)

        docker_api_version = "v1"
        publish_dir = configuration.get_web_publish_dir(repo, config, docker_api_version)
        app_file = configuration.get_redirect_file_name(repo)
        app_publish_location = os.path.join(configuration.get_app_publish_dir(config, docker_api_version), app_file)
        self.working_dir = os.path.join(self.get_working_dir(), docker_api_version)
        misc.mkdir(self.working_dir)
        self.web_working_dir = os.path.join(self.get_working_dir(), "web")
        master_publish_dir = configuration.get_master_publish_dir(repo, config, docker_api_version)
        atomic_publish_step = AtomicDirectoryPublishStep(
            self.get_working_dir(),
            [("web", publish_dir), (app_file, app_publish_location)],
            master_publish_dir,
            step_type=constants.PUBLISH_STEP_OVER_HTTP,
        )
        atomic_publish_step.description = _("Making v1 files available via web.")
        self.add_child(PublishImagesStep())
        self.add_child(atomic_publish_step)

    def __init__(self, repo, publish_conduit, config):
        """
        Initialize the V2WebPublisher.

        :param repo: Pulp managed Yum repository
        :type  repo: pulp.plugins.model.Repository
        :param publish_conduit: Conduit providing access to relative Pulp functionality
        :type  publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
        :param config: Pulp configuration for the distributor
        :type  config: pulp.plugins.config.PluginCallConfiguration
        """
        super(V2WebPublisher, self).__init__(constants.PUBLISH_STEP_WEB_PUBLISHER,
                                             repo, publish_conduit, config)

        # Map tags we've seen to the "newest" manifests that go with them
        self.tags = {}
        docker_api_version = 'v2'
        publish_dir = configuration.get_web_publish_dir(repo, config, docker_api_version)
        app_file = configuration.get_redirect_file_name(repo)
        app_publish_location = os.path.join(
            configuration.get_app_publish_dir(config, docker_api_version), app_file)
        self.working_dir = os.path.join(self.get_working_dir(), docker_api_version)
        misc.mkdir(self.working_dir)
        self.web_working_dir = os.path.join(self.get_working_dir(), 'web')
        master_publish_dir = configuration.get_master_publish_dir(repo, config, docker_api_version)
        atomic_publish_step = publish_step.AtomicDirectoryPublishStep(
            self.get_working_dir(), [('', publish_dir), (app_file, app_publish_location)],
            master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP)
        atomic_publish_step.description = _('Making v2 files available via web.')
        self.add_child(PublishBlobsStep())
        self.publish_manifests_step = PublishManifestsStep()
        self.add_child(self.publish_manifests_step)
        self.add_child(PublishTagsStep())
        self.add_child(atomic_publish_step)
        self.add_child(RedirectFileStep(app_publish_location))

    def _ensure_destination_dir(self, destination):
        """
        Ensure that the directory specified by destination exists

        :param destination: The full path to the directory to create
        :type destination: str
        """
        mkdir(destination)

    def process_main(self):
        """
        Ensure the local ostree repository has been created
        and the configured.

        :raises PulpCodedException:
        """
        path = self.parent.storage_path
        mkdir(path)
        mkdir(os.path.join(os.path.dirname(path), constants.LINKS_DIR))
        self._init_repository(path)

 def finalize(self):
     """
     Write the Tag list file so that clients can retrieve the list of available Tags.
     """
     tags_path = os.path.join(self.parent.get_working_dir(), 'tags')
     misc.mkdir(tags_path)
     with open(os.path.join(tags_path, 'list'), 'w') as list_file:
         tag_data = {
             'name': configuration.get_repo_registry_id(self.get_repo(), self.get_config()),
             'tags': list(self._tag_names)}
         list_file.write(json.dumps(tag_data))
     # We don't need the tag names anymore
     del self._tag_names

    def process_main(self):
        """
        Publish the JSON file for Crane.
        """
        registry = configuration.get_repo_registry_id(self.get_repo(), self.get_config())
        redirect_url = configuration.get_redirect_url(self.get_config(), self.get_repo(), 'v2')

        redirect_data = {
            'type': 'pulp-docker-redirect', 'version': 2, 'repository': self.get_repo().id,
            'repo-registry-id': registry, 'url': redirect_url,
            'protected': self.get_config().get('protected', False)}

        misc.mkdir(os.path.dirname(self.app_publish_location))
        with open(self.app_publish_location, 'w') as app_file:
            app_file.write(json.dumps(redirect_data))

 def _add_ref(path, branch, commit):
     """
     Write a branch (ref) file into the published repository.
     :param path: The absolute path to the repository.
     :type path: str
     :param branch: The branch relative path.
     :type branch: str
     :param commit: The commit hash.
     :type commit: str
     """
     path = os.path.join(path, 'refs', 'heads', os.path.dirname(branch))
     mkdir(path)
     path = os.path.join(path, os.path.basename(branch))
     with open(path, 'w+') as fp:
         fp.write(commit)

def migrate():
    """
    Move all files and directories from /var/lib/pulp/published/docker/ to
    /var/lib/pulp/published/docker/v1/.
    """
    misc.mkdir(NEW_DOCKER_V1_PUBLISH_PATH)

    for folder in os.listdir(OLD_DOCKER_V1_PUBLISH_PATH):
        if folder == 'v1':
            continue
        folder = os.path.join(OLD_DOCKER_V1_PUBLISH_PATH, folder)
        if os.path.exists(folder):
            shutil.move(folder, NEW_DOCKER_V1_PUBLISH_PATH)

    # Now we must look for and repair broken symlinks
    _repair_links()

    def _write_pem_file(self, config_key, path):
        """
        Write the PEM data from self.config[config_key] to the given path, if the key is defined and
        is "truthy".

        :param config_key: The key corresponding to a value in self.config to write to path.
        :type  config_key: basestring
        :param path:       The path to write the PEM data to.
        :type  path:       basestring
        """
        if config_key in self.config and self.config[config_key]:
            if not os.path.exists(self._pki_path):
                misc.mkdir(os.path.dirname(self._pki_path))
                os.mkdir(self._pki_path, 0700)
            with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0600), 'w') as pem_file:
                pem_file.write(self.config[config_key])

    def publish_repo(self, repo, publish_conduit, config):
        """
        Publish the repository by "installing" each puppet module into the given
        destination directory. This effectively means extracting each module's
        tarball in that directory.

        :param repo: plugin repository object
        :type repo: pulp.plugins.model.Repository
        :param publish_conduit: provides access to relevant Pulp functionality
        :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
        :param config: plugin configuration
        :type config: pulp.plugins.config.PluginConfiguration

        :return: report describing the publish run
        :rtype: pulp.plugins.model.PublishReport
        """
        # get dir from config
        destination = config.get(constants.CONFIG_INSTALL_PATH)
        subdir = config.get(constants.CONFIG_SUBDIR)
        if not destination:
            return publish_conduit.build_failure_report(_('install path not provided'),
                                                        self.detail_report.report)
        if subdir:
            destination = os.path.join(destination, subdir)

        units = list(repo_controller.find_repo_content_units(repo.repo_obj,
                                                             yield_content_unit=True))
        duplicate_units = self._find_duplicate_names(units)
        if duplicate_units:
            for unit in duplicate_units:
                self.detail_report.error(unit.unit_key,
                                         'another unit in this repo also has this name')
            return publish_conduit.build_failure_report(_('duplicate unit names'),
                                                        self.detail_report.report)

        # check for unsafe paths in tarballs, and fail early if problems are found
        self._check_for_unsafe_archive_paths(units, destination)
        if self.detail_report.has_errors:
            return publish_conduit.build_failure_report('failed', self.detail_report.report)

        # ensure the destination directory exists
        try:
            mkdir(destination)
            temporarydestination = self._create_temporary_destination_directory(destination)
        except OSError, e:
            return publish_conduit.build_failure_report(
                _('failed to create destination directory: %s') % str(e), self.detail_report.report)

 def migrate(self, unit_id, path, new_path):
     """
     Migrate the unit.
       1. copy content
       2. update the DB
     :param unit_id: A unit UUID.
     :type unit_id: str
     :param path: The current storage path.
     :type path: str
     :param new_path: The new storage path.
     :type new_path: str
     """
     # the content should be copied(and not moved) due to this issue #1944
     if os.path.exists(path):
         mkdir(os.path.dirname(new_path))
         shutil.copy(path, new_path)
     self.collection.update_one(filter={"_id": unit_id}, update={"$set": {"_storage_path": new_path}})

    def generate_download_requests(self):
        """
        generator that yields DownloadRequests for needed units.

        :return:    generator of DownloadRequest instances
        :rtype:     collections.Iterable[DownloadRequest]
        """
        feed_url = self.get_config().get('feed')
        for unit_key in self.step_get_local_units.units_to_download:
            key_hash = get_key_hash(unit_key)
            # Don't save all the units in one directory as there could be 50k + units
            hash_dir = generate_internal_storage_path(self.deb_data[key_hash]['file_name'])
            # make sure the download directory exists
            dest_dir = os.path.join(self.working_dir, hash_dir)
            download_dir = os.path.dirname(dest_dir)
            misc.mkdir(download_dir)
            file_path = self.deb_data[key_hash]['file_path']
            packages_url = urlparse.urljoin(feed_url, file_path)
            yield DownloadRequest(packages_url, dest_dir)

 def test_mode(self, fake_mkdir):
     path = "path-123"
     mode = 0750
     misc.mkdir(path, mode)
     fake_mkdir.assert_called_once_with(path, mode)

 def test_succeeded(self, fake_mkdir):
     path = "path-123"
     misc.mkdir(path)
     fake_mkdir.assert_called_once_with(path)

 def initialize(self):
     """
     Perform setup required before we start processing the individual units
     """
     misc.mkdir(self.get_working_dir())