def test_maven_artifact(self):
        artifact1 = MavenArtifact.createFromGAV("org.jboss:jboss-parent:pom:10")
        self.assertEqual(artifact1.groupId, "org.jboss")
        self.assertEqual(artifact1.artifactId, "jboss-parent")
        self.assertEqual(artifact1.version, "10")
        self.assertEqual(artifact1.getArtifactType(), "pom")
        self.assertEqual(artifact1.getClassifier(), "")
        self.assertEqual(artifact1.getArtifactFilename(), "jboss-parent-10.pom")
        self.assertEqual(artifact1.getArtifactFilepath(), "org/jboss/jboss-parent/10/jboss-parent-10.pom")

        artifact2 = MavenArtifact.createFromGAV("org.jboss:jboss-foo:jar:1.0")
        self.assertEqual(artifact2.getArtifactFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0.jar")
        self.assertEqual(artifact2.getPomFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0.pom")
        self.assertEqual(artifact2.getSourcesFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0-sources.jar")

        artifact3 = MavenArtifact.createFromGAV("org.jboss:jboss-test:jar:client:2.0.0.Beta1")
        self.assertEqual(artifact3.getClassifier(), "client")
        self.assertEqual(artifact3.getArtifactFilename(), "jboss-test-2.0.0.Beta1-client.jar")
        self.assertEqual(artifact3.getArtifactFilepath(),
                "org/jboss/jboss-test/2.0.0.Beta1/jboss-test-2.0.0.Beta1-client.jar")

        artifact4 = MavenArtifact.createFromGAV("org.acme:jboss-bar:jar:1.0-alpha-1:compile")
        self.assertEqual(artifact4.getArtifactFilepath(), "org/acme/jboss-bar/1.0-alpha-1/jboss-bar-1.0-alpha-1.jar")

        artifact5 = MavenArtifact.createFromGAV("com.google.guava:guava:pom:r05")
        self.assertEqual(artifact5.groupId, "com.google.guava")
        self.assertEqual(artifact5.artifactId, "guava")
        self.assertEqual(artifact5.version, "r05")
        self.assertEqual(artifact5.getArtifactType(), "pom")
        self.assertEqual(artifact5.getClassifier(), "")
        self.assertEqual(artifact5.getArtifactFilename(), "guava-r05.pom")

    def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
        pomMain = True
        # The pom is main only if no other main artifact is available
        if len(extsAndClass) > 1 and self._containsMainArtifact(extsAndClass) and "pom" in extsAndClass:
            pomMain = False

        artTypes = []
        for ext, classifiers in extsAndClass.iteritems():
            main = ext == "pom" and pomMain
            if not main:
                for classifier in classifiers:
                    extClassifier = "%s:%s" % (ext, classifier or "")
                    main = extClassifier not in self.notMainExtClassifiers
                    if main:
                        break
            artTypes.append(ArtifactType(ext, main, classifiers))

        mavenArtifact = MavenArtifact(groupId, artifactId, None, version)
        if suffix is not None:
            mavenArtifact.snapshotVersionSuffix = suffix
        if mavenArtifact in artifacts:
            artifacts[mavenArtifact].merge(ArtifactSpec(url, artTypes))
        else:
            logging.debug("Adding artifact %s", str(mavenArtifact))
            artifacts[mavenArtifact] = ArtifactSpec(url, artTypes)

 def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
     if len(extsAndClass) > 1 and self._containsNonPomWithoutClassifier(extsAndClass) and "pom" in extsAndClass:
         del extsAndClass["pom"]
     for ext in extsAndClass:
         mavenArtifact = MavenArtifact(groupId, artifactId, ext, version)
         if suffix is not None:
             mavenArtifact.snapshotVersionSuffix = suffix
         logging.debug("Adding artifact %s", str(mavenArtifact))
         artifacts[mavenArtifact] = ArtifactSpec(url, extsAndClass[ext])

    def _filterExcludedTypes(self, artifactList):
        '''
        Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with
        not-excluded artifact types only.

        :param artifactList: artifactList to be filtered.
        :param exclTypes: list of excluded types
        :returns: artifactList without artifacts that matched specified types and had no other main types.
        '''
        logging.debug("Filtering artifacts with excluded types.")
        regExps = maven_repo_util.getRegExpsFromStrings(self.config.gatcvWhitelist)
        exclTypes = self.config.excludedTypes
        for ga in artifactList.keys():
            for priority in artifactList[ga].keys():
                for version in artifactList[ga][priority].keys():
                    artSpec = artifactList[ga][priority][version]
                    for artType in list(artSpec.artTypes.keys()):
                        if artType in exclTypes:
                            artTypeObj = artSpec.artTypes[artType]
                            classifiers = artTypeObj.classifiers
                            (groupId, artifactId) = ga.split(':')
                            for classifier in list(classifiers):
                                art = MavenArtifact(groupId, artifactId, artType, version, classifier)
                                gatcv = art.getGATCV()
                                if not maven_repo_util.somethingMatch(regExps, gatcv):
                                    logging.debug("Dropping classifier \"%s\" of %s:%s:%s from priority %i because of "
                                                  "excluded type.", classifier, ga, artType, version, priority)
                                    classifiers.remove(classifier)
                                else:
                                    logging.debug("Skipping drop of %s:%s:%s:%s from priority %i because it matches a "
                                                  "whitelist pattern.", ga, artType, classifier, version, priority)
                            if not classifiers:
                                logging.debug("Dropping %s:%s:%s from priority %i because of no classifier left.", ga,
                                              artType, version, priority)
                                del(artSpec.artTypes[artType])
                    noMain = True
                    for artType in artSpec.artTypes.keys():
                        artTypeObj = artSpec.artTypes[artType]
                        if artTypeObj.mainType:
                            noMain = False
                            break
                    if not artSpec.artTypes or noMain:
                        if noMain:
                            logging.debug("Dropping GAV %s:%s from priority %i because of no main artifact left.",
                                          ga, version, priority)
                        else:
                            logging.debug("Dropping GAV %s:%s from priority %i because of no artifact type left.",
                                          ga, version, priority)
                        del artifactList[ga][priority][version]
                if not artifactList[ga][priority]:
                    logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
                    del artifactList[ga][priority]
            if not artifactList[ga]:
                logging.debug("Dropping GA %s because of no priority left.", ga)
                del artifactList[ga]
        return artifactList

    def test_listDependencies(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['http://repo.maven.apache.org/maven2/']
        gavs = [
            'com.sun.faces:jsf-api:2.0.11',
            'org.apache.ant:ant:1.8.0'
        ]
        dependencies = {
            'javax.servlet:javax.servlet-api:jar:3.0.1': set(['javadoc', 'sources']),
            'javax.servlet.jsp.jstl:jstl-api:jar:1.2': set(['javadoc', 'sources']),
            'xml-apis:xml-apis:jar:1.3.04': set(['source', 'sources']),
            'javax.servlet:servlet-api:jar:2.5': set(['sources']),
            'javax.el:javax.el-api:jar:2.2.1': set(['javadoc', 'sources']),
            'junit:junit:jar:3.8.2': set(['javadoc', 'sources']),
            'xerces:xercesImpl:jar:2.9.0': set([]),
            'javax.servlet.jsp:jsp-api:jar:2.1': set(['sources']),
            'javax.servlet.jsp:javax.servlet.jsp-api:jar:2.2.1': set(['javadoc', 'sources']),
            'org.apache.ant:ant-launcher:jar:1.8.0': set([])
        }
        expectedArtifacts = {}
        for dep in dependencies:
            artifact = MavenArtifact.createFromGAV(dep)
            expectedArtifacts[artifact] = ArtifactSpec(repoUrls[0], dependencies[dep])

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listDependencies(repoUrls, gavs, False, False)

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)

def generateArtifactList(options, args):
    """
    Generates artifact "list" from sources defined in the given configuration in options. The result
    is dictionary with following structure:

    <repo url> (string)
      L list of MavenArtifact
    """

    artifactList = _generateArtifactList(options, args)
    #build sane structure - url to MavenArtifact list
    urlToMAList = {}
    for ga in artifactList:
        priorityList = artifactList[ga]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                artSpec = versionList[version]
                url = artSpec.url
                for artType in artSpec.artTypes.keys():
                    for classifier in artSpec.artTypes[artType].classifiers:
                        if classifier:
                            gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, version)
                        else:
                            gatcv = "%s:%s:%s" % (ga, artType, version)
                        artifact = MavenArtifact.createFromGAV(gatcv)
                        urlToMAList.setdefault(url, []).append(artifact)
    return urlToMAList

def generateArtifactList(options):
    """
    Generates artifact "list" from sources defined in the given configuration in options. The result
    is dictionary with following structure:

    <repo url> (string)
      L artifacts (list of MavenArtifact)
    """

    options.allclassifiers = (options.classifiers == '__all__')

    artifactList = _generateArtifactList(options)
    #build sane structure - url to MavenArtifact list
    urlToMAList = {}
    for gat in artifactList:
        priorityList = artifactList[gat]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                artSpec = versionList[version]
                url = artSpec.url
                for classifier in artSpec.classifiers:
                    if classifier == "" or options.allclassifiers:
                        artifact = MavenArtifact.createFromGAV(gat + ((":" + classifier) if classifier else "") +
                                                               ":" + version)
                        urlToMAList.setdefault(url, []).append(artifact)
    return urlToMAList

    def test_listRepository_file(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['file://./tests/testrepo']
        gavPatterns = [
            'bar:foo-bar:1.1',
            'foo.baz:baz-core:1.0'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(repoUrls[0], set([])),
            MavenArtifact.createFromGAV(gavPatterns[1]): ArtifactSpec(repoUrls[0], set(['javadoc', 'sources']))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)

    def test_listRepository_http(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['http://repo.maven.apache.org/maven2/']
        gavPatterns = [
            'com.sun.faces:jsf-api:2.0.11',
            'org.apache.ant:ant:1.8.0'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(repoUrls[0], set(['javadoc', 'sources'])),
            MavenArtifact.createFromGAV(gavPatterns[1]): ArtifactSpec(repoUrls[0], set([]))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)

        def findArtifact(gav, urls, artifacts):
            artifact = MavenArtifact.createFromGAV(gav)
            for url in urls:
                if maven_repo_util.gavExists(url, artifact):
                    #Critical section?
                    artifacts[artifact] = ArtifactSpec(url, [ArtifactType(artifact.artifactType, True, set(['']))])
                    return

            logging.warning('Artifact %s not found in any url!', artifact)

    def test_listRepository_file_gatcvs(self):
        config = configuration.Configuration()
        config.addClassifiers = "__all__"
        repoUrls = ['file://./tests/testrepo']
        gatcvs = [
            'bar:foo-bar:pom:1.1',
            'foo.baz:baz-core:jar:1.0'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, None, gatcvs)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gatcvs[0]): ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set(['']))]),
            MavenArtifact.createFromGAV(gatcvs[1]): ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set([''])),
                                        ArtifactType("jar", True, set(['', 'javadoc', 'sources']))])
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)

    def _getExpectedArtifacts(self, repoUrl, dependencies):

        artSpecDict = {}
        for dep in dependencies.keys():
            artifact = MavenArtifact.createFromGAV(dep)
            artTypes = []
            artTypes.append(ArtifactType(artifact.artifactType, artifact.artifactType != "pom", dependencies[dep]))
            artSpec = ArtifactSpec(repoUrl, artTypes)
            gav = artifact.getGAV()
            if gav in artSpecDict.keys():
                artSpecDict[gav].merge(artSpec)
            else:
                artSpecDict[gav] = artSpec
        expectedArtifacts = {}
        for (gav, artSpec) in artSpecDict.iteritems():
            artifact = MavenArtifact.createFromGAV(gav)
            expectedArtifacts[artifact] = artSpec

        return expectedArtifacts

def depListToArtifactList(depList):
    """Convert the maven GAV to a URL relative path"""
    regexComment = re.compile('#.*$')
    #regexLog = re.compile('^\[\w*\]')
    artifactList = []
    for nextLine in depList:
        nextLine = regexComment.sub('', nextLine)
        nextLine = nextLine.strip()
        gav = maven_repo_util.parseGATCVS(nextLine)
        if gav:
            artifactList.append(MavenArtifact.createFromGAV(gav))
    return artifactList

    def _filterExcludedRepositories(self, artifactList):
        """
        Filter artifactList removing artifacts existing in specified repositories.

        :param artifactList: artifactList to be filtered.
        :returns: artifactList without artifacts that exists in specified repositories.
        """

        logging.debug("Filtering artifacts contained in excluded repositories.")

        pool = ThreadPool(maven_repo_util.MAX_THREADS)
        # Contains artifact to be removed
        delArtifacts = []
        for ga in artifactList.keys():
            groupId = ga.split(':')[0]
            artifactId = ga.split(':')[1]
            for priority in artifactList[ga].keys():
                for version in artifactList[ga][priority].keys():
                    artifact = MavenArtifact(groupId, artifactId, "pom", version)
                    pool.apply_async(
                        _artifactInRepos,
                        [self.config.excludedRepositories, artifact, priority, delArtifacts]
                    )

        # Close the pool and wait for the workers to finnish
        pool.close()
        pool.join()
        for artifact, priority in delArtifacts:
            ga = artifact.getGA()
            logging.debug("Dropping GAV %s:%s from priority %i because it was found in an excluded repository.",
                          ga, artifact.version, priority)
            del artifactList[ga][priority][artifact.version]
            if not artifactList[ga][priority]:
                logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
                del artifactList[ga][priority]
            if not artifactList[ga]:
                logging.debug("Dropping GA %s because of no priority left.", ga)
                del artifactList[ga]

        return artifactList

def generate_report(output, config, artifact_list, report_name):
    """
    Generates report. The report consists of a summary page, groupId pages, artifactId pages and leaf artifact pages.
    Summary page contains list of roots, list of BOMs, list of multi-version artifacts, links to all artifacts and list
    of artifacts that do not match the BOM version. Each artifact has a separate page containing paths from roots to
    the artifact with path explanation.
    """
    multiversion_gas = dict()
    malformed_versions = dict()
    if os.path.exists(output):
        logging.warn("Target report path %s exists. Deleting...", output)
        shutil.rmtree(output)
    os.makedirs(os.path.join(output, "pages"))

    roots = []
    for artifact_source in config.artifactSources:
        if artifact_source["type"] == "dependency-graph":
            roots.extend(artifact_source["top-level-gavs"])

    groupids = dict()
    version_pattern = re.compile("^.*[.-]redhat-[^.]+$")
    for ga in artifact_list:
        (groupid, artifactid) = ga.split(":")
        priority_list = artifact_list[ga]
        for priority in priority_list:
            versions = priority_list[priority]
            if versions:
                groupids.setdefault(groupid, dict()).setdefault(artifactid, dict()).update(versions)
                if len(groupids[groupid][artifactid]) > 1:
                    multiversion_gas.setdefault(groupid, dict())[artifactid] = groupids[groupid][artifactid]
                for version in versions:
                    if not version_pattern.match(version):
                        malformed_versions.setdefault(groupid, dict()).setdefault(artifactid, dict())[
                            version
                        ] = groupids[groupid][artifactid][version]

    optional_artifacts = dict()
    for groupid in groupids.keys():
        artifactids = groupids[groupid]
        for artifactid in artifactids.keys():
            versions = artifactids[artifactid]
            for version in versions.keys():
                art_spec = versions[version]

                ma = MavenArtifact.createFromGAV("%s:%s:%s" % (groupid, artifactid, version))
                generate_artifact_page(ma, roots, art_spec.paths, art_spec.url, output, groupids, optional_artifacts)
            generate_artifactid_page(groupid, artifactid, versions, output)
        generate_groupid_page(groupid, artifactids, output)
    generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts)
    generate_css(output)

def depListToArtifactList(depList):
    """Convert the maven GAV to a URL relative path"""
    regexComment = re.compile('#.*$')
    #regexLog = re.compile('^\[\w*\]')
    # Match pattern groupId:artifactId:type:[classifier:]version[:scope]
    regexGAV = re.compile('(([\w\-.]+:){3}([\w\-.]+:)?([\d][\w\-.]+))(:[\w]*\S)?')
    artifactList = []
    for nextLine in depList:
        nextLine = regexComment.sub('', nextLine)
        nextLine = nextLine.strip()
        gav = regexGAV.search(nextLine)
        if gav:
            artifactList.append(MavenArtifact.createFromGAV(gav.group(1)))
    return artifactList

    def _filterExcludedRepositories(self, artifactList):
        """
        Filter artifactList removing artifacts existing in specified repositories.

        :param artifactList: artifactList to be filtered.
        :returns: artifactList without arifacts that exists in specified repositories.
        """

        logging.debug("Filtering artifacts contained in excluded repositories.")

        pool = ThreadPool(maven_repo_util.MAX_THREADS)
        # Contains artifact to be removed
        delArtifacts = []
        for gat in artifactList.keys():
            groupId = gat.split(':')[0]
            artifactId = gat.split(':')[1]
            artifactType = gat.split(':')[2]
            for priority in artifactList[gat].keys():
                for version in artifactList[gat][priority].keys():
                    artifact = MavenArtifact(groupId, artifactId, artifactType, version)
                    pool.apply_async(
                        _artifactInRepos,
                        [self.config.excludedRepositories, artifact, priority, delArtifacts]
                    )
                if not artifactList[gat][priority]:
                    del artifactList[gat][priority]
            if not artifactList[gat]:
                del artifactList[gat]

        # Close the pool and wait for the workers to finnish
        pool.close()
        pool.join()
        for artifact, priority in delArtifacts:
            del artifactList[artifact.getGAT()][priority][artifact.version]

        return artifactList

    def _listDependencies(self, repoUrls, gavs):
        """
        Loads maven artifacts from mvn dependency:list.

        :param repoUrls: URL of the repositories that contains the listed artifacts
        :param gavs: List of top level GAVs
        :returns: Dictionary where index is MavenArtifact object and value is
                  it's repo root URL, or empty dictionary if something goes wrong.
        """
        artifacts = {}

        for gav in gavs:
            artifact = MavenArtifact.createFromGAV(gav)

            pomDir = 'poms'
            fetched = False
            for repoUrl in repoUrls:
                pomUrl = repoUrl + '/' + artifact.getPomFilepath()
                if fetchArtifact(pomUrl, pomDir):
                    fetched = True
                    break

            if not fetched:
                logging.warning("Failed to retrieve pom file for artifact %s",
                                gav)
                continue

            # Build dependency:list
            mvnOutDir = "maven"
            if not os.path.isdir(mvnOutDir):
                os.makedirs(mvnOutDir)
            mvnOutFilename = mvnOutDir + "/" + artifact.getBaseFilename() + "-maven.out"
            with open(mvnOutFilename, "w") as mvnOutputFile:
                retCode = call(['mvn', 'dependency:list', '-N', '-f',
                                pomDir + '/' + artifact.getPomFilename()], stdout=mvnOutputFile)

                if retCode != 0:
                    logging.warning("Maven failed to finish with success. Skipping artifact %s",
                                    gav)
                    continue

            # Parse GAVs from maven output
            gavList = self._parseDepList(mvnOutFilename)

            artifacts.update(self._listArtifacts(repoUrls, gavList))

        return artifacts

    def test_listMeadTagArtifacts(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        kojiUrl = "http://brewhub.devel.redhat.com/brewhub/"
        tagName = "mead-import-maven"
        downloadRootUrl = "http://download.devel.redhat.com/brewroot/packages/"
        gavPatterns = [
            'org.apache.maven:maven-core:2.0.6'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listMeadTagArtifacts(kojiUrl, downloadRootUrl, tagName, gavPatterns)
        expectedUrl = downloadRootUrl + "org.apache.maven-maven-core/2.0.6/1/maven/"
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(expectedUrl, set(['javadoc', 'sources']))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)

    def _listDependencyGraph(self, aproxUrl, wsid, sourceKey, gavs):
        """
        Loads maven artifacts from dependency graph.

        :param aproxUrl: URL of the AProx instance
        :param gavs: List of top level GAVs
        :param wsid: workspace ID
        :returns: Dictionary where index is MavenArtifact object and value is
                  ArtifactSpec with its repo root URL
        """
        from aprox_apis import AproxApi10
        aprox = AproxApi10(aproxUrl)

        deleteWS = False

        if not wsid:
            # Create workspace
            ws = aprox.createWorkspace()
            wsid = ws["id"]
            deleteWS = True

        # Resolve graph MANIFEST for GAVs
        urlmap = aprox.urlmap(wsid, sourceKey, gavs, self.configuration.allClassifiers)

        # parse returned map
        artifacts = {}
        for gav in urlmap:
            artifact = MavenArtifact.createFromGAV(gav)
            groupId = artifact.groupId
            artifactId = artifact.artifactId
            version = artifact.version

            filenames = urlmap[gav]["files"]
            url = urlmap[gav]["repoUrl"]

            (extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)

            self._addArtifact(artifacts, groupId, artifactId, version, extsAndClass, suffix, url)

        # cleanup
        if deleteWS:
            aprox.deleteWorkspace(wsid)

        return artifacts