def test_send_artifact_collection(self, mock_post):
        """Can add a artifact collections to a TreeherderRequest."""
        mock_post.return_value = self._expected_response_return_object()

        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            tac.add(tac.get_artifact(artifact))

        client = TreeherderClient(
            protocol='http',
            host='host',
            client_id='client-abc',
            secret='secret123',
            )

        client.post_collection('project', tac)

        path, resp = mock_post.call_args

        self.assertEqual(mock_post.call_count, 1)
        self.assertEqual(
            tac.get_collection_data(),
            resp['json']
            )

def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb):
    """Post a list of artifacts to a job."""

    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url)
    logger.debug("Downloading/parsing log for %s", log_description)

    job_log = JobLog.objects.get(job__guid=job_guid, url=job_log_url)

    credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID)
    client = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        client_id=credentials.client_id,
        secret=str(credentials.secret),
    )

    try:
        artifact_list = extract_artifacts_cb(project, job_log_url, job_guid)
    except Exception as e:
        job_log.update_status(JobLog.FAILED)

        # unrecoverable http error (doesn't exist or permission denied)
        # (apparently this can happen somewhat often with taskcluster if
        # the job fails, so just warn about it -- see
        # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
        if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
            logger.warning("Unable to retrieve log for %s: %s", log_description, e)
            return
        # possibly recoverable http error (e.g. problems on our end)
        elif isinstance(e, urllib2.URLError):
            logger.error("Failed to download log for %s: %s", log_description, e)
            _retry(e)
        # parse error or other unrecoverable error
        else:
            logger.error("Failed to download/parse log for %s: %s", log_description, e)
        # re-raise exception if we're not retrying, so new relic sees the
        # error
        raise

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact(artifact)
        tac.add(ta)

    try:
        client.post_collection(project, tac)
        job_log.update_status(JobLog.PARSED)
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)

    def test_artifact_collection(self):
        """Confirm the collection matches the sample data"""
        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            ta = TreeherderArtifact(artifact)
            tac.add(ta)

        self.assertTrue(len(self.artifact_data) == len(tac.data))

    def test_chunk_endpoint_base(self):
        """Confirm the endpoint_base of the chunks is the same as the original"""
        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            ta = TreeherderArtifact(artifact)
            tac.add(ta)

        for chunk in tac.get_chunks(3):
            assert tac.endpoint_base == chunk.endpoint_base

def post_log_artifacts(project,
                       job_guid,
                       job_log_url,
                       retry_task,
                       extract_artifacts_cb,
                       check_errors=False):
    """Post a list of artifacts to a job."""
    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
    logger.debug("Downloading/parsing log for %s", log_description)

    credentials = OAuthCredentials.get_credentials(project)
    client = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST
    )

    try:
        artifact_list = extract_artifacts_cb(job_log_url['url'],
                                             job_guid, check_errors)
    except Exception as e:
        client.update_parse_status(project, credentials.get('consumer_key'),
                                   credentials.get('consumer_secret'),
                                   job_log_url['id'], 'failed')
        if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
            logger.debug("Unable to retrieve log for %s: %s", log_description, e)
            return
        logger.error("Failed to download/parse log for %s: %s", log_description, e)
        _retry(e)

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact(artifact)
        tac.add(ta)

    try:
        client.post_collection(project, credentials.get('consumer_key'),
                               credentials.get('consumer_secret'),
                               tac)
        client.update_parse_status(project, credentials.get('consumer_key'),
                                   credentials.get('consumer_secret'),
                                   job_log_url['id'], 'parsed')
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)

    def test_send_artifact_collection(self, mock_post):
        """Can add a artifact collections to a TreeherderRequest."""
        mock_post.return_value = self._expected_response_return_object()

        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:

            tac.add(tac.get_artifact(artifact))

        client = TreeherderClient(protocol="http", host="host")

        auth = TreeherderAuth("key", "secret", "project")
        client.post_collection("project", tac, auth=auth)

        path, resp = mock_post.call_args

        self.assertEqual(mock_post.call_count, 1)
        self.assertEqual(tac.get_collection_data(), resp["json"])

    def test_send_artifact_collection(self):
        """Can add a artifact collections to a TreeherderRequest."""
        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            tac.add(tac.get_artifact(artifact))

        client = TreeherderClient(protocol="http", host="host", client_id="client-abc", secret="secret123")

        def request_callback(request):
            # Check that the expected content was POSTed.
            posted_json = json.loads(request.body)
            self.assertEqual(posted_json, tac.get_collection_data())
            return (200, {}, '{"message": "Artifacts stored successfully"}')

        url = client._get_project_uri("project", tac.endpoint_base)
        responses.add_callback(
            responses.POST, url, match_querystring=True, callback=request_callback, content_type="application/json"
        )

        client.post_collection("project", tac)

    def test_collection_chunking(self):
        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            ta = TreeherderArtifact(artifact)
            tac.add(ta)

        # reconstruct the chunks and make sure we have the same data
        rebuilt_data = []
        chunk_num = 0
        for chunk in tac.get_chunks(3):
            chunk_data = chunk.get_collection_data()
            rebuilt_data.extend(chunk_data)

            chunk_num += 1
            # the last one should be the "remainder" in an uneven size
            if chunk_num == 4:
                assert len(chunk_data) == 1
            else:
                assert len(chunk_data) == 3

        assert rebuilt_data == tac.get_collection_data()

    def test_artifact_sample_data(self):
        """Test all add methods for building an artifact"""

        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:

            ta = TreeherderArtifact()

            ta.add_blob(artifact['blob'])
            ta.add_job_guid(artifact['job_guid'])
            ta.add_name(artifact['name'])
            ta.add_type(artifact['type'])

            self.compare_structs(ta.data, artifact)

            tac.add(ta)

            # confirm we get the same thing if we initialize from
            # a resultset dict
            ta_struct = TreeherderArtifact(artifact)

            self.compare_structs(ta_struct.data, artifact)

    def test_send_artifact_collection(self):
        """Can add a artifact collections to a TreeherderRequest."""
        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            tac.add(tac.get_artifact(artifact))

        client = TreeherderClient(
            server_url='http://host',
            client_id='client-abc',
            secret='secret123',
            )

        def request_callback(request):
            # Check that the expected content was POSTed.
            posted_json = json.loads(request.body)
            self.assertEqual(posted_json, tac.get_collection_data())
            return (200, {}, '{"message": "Artifacts stored successfully"}')

        url = client._get_endpoint_url(tac.endpoint_base, project='project')
        responses.add_callback(responses.POST, url, match_querystring=True,
                               callback=request_callback, content_type='application/json')

        client.post_collection('project', tac)

    def test_send_artifact_collection(self, mock_send):
        """Can add a artifact collections to a TreeherderRequest."""

        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:

            tac.add(tac.get_artifact(artifact))

        req = TreeherderRequest(
            protocol='http',
            host='host',
            project='project',
            oauth_key='key',
            oauth_secret='secret',
        )

        req.post(tac)

        self.assertEqual(mock_send.call_count, 1)
        self.assertEqual(
            tac.to_json(),
            mock_send.call_args_list[0][1]["data"]
        )