def post_request(self, machine, project, job_collection, attempts, last_attempt):
        logger.debug('AutophoneTreeherder.post_request: %s, attempt=%d, last=%s' %
                     (job_collection.__dict__, attempts, last_attempt))
        client = TreeherderClient(protocol=self.protocol,
                                  host=self.server,
                                  client_id=self.client_id,
                                  secret=self.secret)

        try:
            client.post_collection(project, job_collection)
            return True
        except Exception, e:
            logger.exception('Error submitting request to Treeherder, attempt=%d, last=%s' %
                             (attempts, last_attempt))
            if self.mailer:
                if hasattr(e, 'response') and e.response:
                    response_json = json.dumps(e.response.json(),
                                               indent=2, sort_keys=True)
                else:
                    response_json = None
                self.mailer.send(
                    '%s attempt %d Error submitting request to Treeherder' %
                    (utils.host(), attempts),
                    'Phone: %s\n'
                    'Exception: %s\n'
                    'Last attempt: %s\n'
                    'Response: %s\n' % (
                        machine,
                        e,
                        last_attempt,
                        response_json))

    def post_request(self, project, job_collection, guid=None):
        self.logger.debug(type(self).__name__ + '.post_request - '
                          'job_collection =\n%s' %
                          pretty(job_collection.get_collection_data()))

        client = TreeherderClient(protocol=self.protocol,
                                  host=self.server,
                                  client_id=self.credentials['client_id'],
                                  secret=self.credentials['secret'])
        for attempt in range(1, self.retries + 1):
            try:
                client.post_collection(project, job_collection)
                self.logger.info(type(self).__name__ +
                                 '.post_request - collection posted')
                if guid:
                    job_url = self.request_job_url(project, guid)
                    self.logger.info(type(self).__name__ +
                                     '.post_request - url is %s' % job_url)
                return
            except requests.exceptions.Timeout:
                message = ('Attempt %d to post result to '
                           'Treeherder timed out.' % attempt)
                self.logger.error(message)
                time.sleep(self.retry_wait)
            except Exception as e:
                message = ('Error submitting request to Treeherder\n\n'
                           'Exception: %s\n'
                           'TreeherderJobCollection %s\n' %
                           (e, pretty(job_collection.get_collection_data())))
                self.logger.exception(message)
                return
        self.logger.error('Error submitting request to Treeherder.')

    def submit(self, revision, browser, timestamp, perf_data, version='', repo_link='', video_links='', extra_info_obj={}):

        j_dataset = self.create_job_dataset(revision=revision,
                                            browser=browser,
                                            timestamp=timestamp,
                                            perf_data=perf_data,
                                            version=version,
                                            repo_link=repo_link,
                                            video_links=video_links,
                                            extra_info_obj=extra_info_obj)
        tjc = self.create_job_collection(j_dataset)

        if self.server_url:
            client = TreeherderClient(server_url=self.server_url,
                                      client_id=self.client_id,
                                      secret=self.secret)
        else:
            client = TreeherderClient(client_id=self.client_id,
                                      secret=self.secret)

        try:
            return_result = client.post_collection(self.repo, tjc)
        except Exception as e:
            print e.message
            print traceback.print_exc()
            return None
        return return_result

    def submit(self, job):
        """Submit the job to treeherder.

        :param job: Treeherder job instance to use for submission.

        """
        job.add_submit_timestamp(int(time.time()))

        # We can only submit job info once, so it has to be done in completed
        if self._job_details:
            job.add_artifact('Job Info', 'json', {'job_details': self._job_details})

        job_collection = TreeherderJobCollection()
        job_collection.add(job)

        logger.info('Sending results to Treeherder: {}'.format(job_collection.to_json()))
        url = urlparse(self.url)
        client = TreeherderClient(protocol=url.scheme, host=url.hostname,
                                  client_id=self.client_id, secret=self.secret)
        client.post_collection(self.repository, job_collection)

        logger.info('Results are available to view at: {}'.format(
                    urljoin(self.url,
                            JOB_FRAGMENT.format(repository=self.repository,
                                                revision=self.revision))))

def on_event(data, message, dry_run, treeherder_server_url, acknowledge, **kwargs):
    if ignored(data):
        if acknowledge:
            message.ack()
        return 0  # SUCCESS

    # Cleaning mozci caches
    buildjson.BUILDS_CACHE = {}
    query_jobs.JOBS_CACHE = {}
    repo_name = data["project"]
    action = data["action"]
    times = data["times"]
    # Pulse gives us resultset_id, we need to get revision from it.
    resultset_id = data["resultset_id"]

    treeherder_client = TreeherderClient(server_url=treeherder_server_url)

    LOG.info("%s action requested by %s on repo_name %s with resultset_id: %s" % (
        data['action'],
        data["requester"],
        data["project"],
        data["resultset_id"])
    )
    revision = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]["revision"]
    status = None

    if action == "trigger_missing_jobs":
        mgr = BuildAPIManager()
        mgr.trigger_missing_jobs_for_revision(repo_name, revision, dry_run=dry_run)
        if acknowledge:
            status = 'trigger_missing_jobs request sent'
        else:
            status = 'Dry-mode, no request sent'

    elif action == "trigger_all_talos_jobs":
        trigger_all_talos_jobs(
            repo_name=repo_name,
            revision=revision,
            times=times,
            priority=-1,
            dry_run=dry_run
        )
        if acknowledge:
            status = 'trigger_all_talos_jobs: {0} times request sent with priority'\
                     'lower then normal'.format(times)
        else:
            status = 'Dry-mode, no request sent'
    else:
        raise Exception(
            'We were not aware of the "{}" action. Please address the code.'.format(action)
        )

    LOG.debug(status)

    if acknowledge:
        # We need to ack the message to remove it from our queue
        message.ack()

    return 0  # SUCCESS

def on_buildbot_event(data, message, dry_run, stage=False):
    """Act upon buildbot events."""
    # Pulse gives us a job_id and a job_guid, we need request_id.
    LOG.info(
        "%s action requested by %s on repo_name %s with job_id: %s"
        % (data["action"], data["requester"], data["project"], data["job_id"])
    )
    # Cleaning mozci caches
    buildjson.BUILDS_CACHE = {}
    query_jobs.JOBS_CACHE = {}

    if stage:
        treeherder_client = TreeherderClient(host="treeherder.allizom.org")
    else:
        treeherder_client = TreeherderClient()
    repo_name = data["project"]
    job_id = data["job_id"]
    result = treeherder_client.get_jobs(repo_name, id=job_id)
    # If result not found, ignore
    if not result:
        LOG.info("We could not find any result for repo_name: %s and " "job_id: %s" % (repo_name, job_id))
        message.ack()
        return

    result = result[0]
    buildername = result["ref_data_name"]
    resultset_id = result["result_set_id"]
    result_sets = treeherder_client.get_resultsets(repo_name, id=resultset_id)
    revision = result_sets[0]["revision"]
    action = data["action"]
    status = None

    buildername = filter_invalid_builders(buildername)

    # Treeherder can send us invalid builder names
    # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038
    if buildername is None:
        status = "Builder %s was invalid." % buildername[0]

    # Backfill action
    elif action == "backfill":
        manual_backfill(revision, buildername, max_revisions=get_maxRevisions(buildername), dry_run=dry_run)
        if not dry_run:
            status = "Backfill request sent"
        else:
            status = "Dry-run mode, nothing was backfilled"

    # Send a pulse message showing what we did
    message_sender = MessageHandler()
    pulse_message = {"job_id": job_id, "action": action, "requester": data["requester"], "status": status}
    routing_key = "{}.{}".format(repo_name, action)
    try:
        message_sender.publish_message(pulse_message, routing_key)
    except:
        LOG.warning("Failed to publish message over pulse stream.")

    if not dry_run:
        # We need to ack the message to remove it from our queue
        message.ack()

def get_revision_hash(server_url, project, revision):
    """Retrieve the Treeherder's revision hash for a given revision.

    :param server_url: URL of the Treeherder instance.
    :param project: The project (branch) to use.
    :param revision: The revision to get the hash for.
    """
    client = TreeherderClient(server_url=server_url)
    resultsets = client.get_resultsets(project, revision=revision)

    return resultsets[0]['revision_hash']

def get_job_log(repo_name, job_id):
    '''For a given job id return the URL to the log associated to it.'''
    th_client = TreeherderClient()
    query_params = {'job_id': job_id, 'name': 'text_log_summary'}
    try:
        return str(th_client.get_artifacts(repo_name, **query_params)[0]['blob']['logurl'])
    except IndexError:
        print 'No artifacts for {}'.format(job_id)
    except requests.exceptions.ConnectionError as e:
        print 'Connection failed for {}'.format(job_id)
        traceback.print_exc()

 def __init__(self, repo, platform, status_check):
     self.repo = repo
     self.platform = platform
     self.platform_option = 'opt'
     self.resultsets = []
     self.skip_status_check = status_check
     self.thclient = TreeherderClient()

 def __init__(self, server_url='https://treeherder.mozilla.org', treeherder_host=None):
     if treeherder_host:
         LOG.warning("The `TreeherderApi()` parameter `treeherder_host` is deprecated. "
                     "Use `server_url` instead, or omit entirely to use the default of "
                     "production Treeherder.")
         server_url = 'https://%s' % treeherder_host
     self.treeherder_client = TreeherderClient(server_url=server_url)

    def __init__(self, worker_subprocess, options, jobs, s3_bucket=None,
                 mailer=None):
        assert options, "options is required."

        logger = utils.getLogger()

        self.options = options
        self.jobs = jobs
        self.s3_bucket = s3_bucket
        self.mailer = mailer
        self.worker = worker_subprocess
        self.shutdown_requested = False
        logger.debug('AutophoneTreeherder')

        self.url = self.options.treeherder_url
        if not self.url:
            logger.debug('AutophoneTreeherder: no treeherder url')
            return

        self.client_id = self.options.treeherder_client_id
        self.secret = self.options.treeherder_secret
        self.retry_wait = self.options.treeherder_retry_wait

        self.client = TreeherderClient(server_url=self.url,
                                       client_id=self.client_id,
                                       secret=self.secret)

        logger.debug('AutophoneTreeherder: %s', self)

def get_all_jobs(repo_name, revision):
    '''Return dictionary of all jobs for a given revision

    Return: {'<revision_hash>': {'<job_id_1>': <job_id_1_metadata>}}
    '''
    print "Fetching Treeherder jobs for {}/{}".format(repo_name, revision)
    th_client = TreeherderClient()
    results = th_client.get_resultsets(repo_name, revision=revision)
    all_jobs = {}
    if results:
        revision_id = results[0]["id"]
        for job in th_client.get_jobs(repo_name, count=6000, result_set_id=revision_id):
            # Grab job metadata
            all_jobs[job['id']] = job

    return {revision: all_jobs}

def query_repositories(clobber=False):
    """
    Return dictionary with information about the various repositories.

    The data about a repository looks like this:

    .. code-block:: python

        "ash": {
            "repo": "https://hg.mozilla.org/projects/ash",
            "graph_branches": ["Ash"],
            "repo_type": "hg"
        }

    """
    global REPOSITORIES

    if clobber:
        REPOSITORIES = {}
        if os.path.exists(REPOSITORIES_FILE):
            os.remove(REPOSITORIES_FILE)

    if REPOSITORIES:
        return REPOSITORIES

    if os.path.exists(REPOSITORIES_FILE):
        LOG.debug("Loading %s" % REPOSITORIES_FILE)
        fd = open(REPOSITORIES_FILE)
        REPOSITORIES = json.load(fd)
    else:

        th_client = TreeherderClient(protocol="https", host=TREEHERDER_URL)
        treeherderRepos = th_client.get_repositories()
        REPOSITORIES = {}
        for th_repo in treeherderRepos:
            if th_repo["active_status"] == "active":
                repo = {}
                repo["repo"] = th_repo["url"]
                repo["repo_type"] = th_repo["dvcs_type"]
                repo["graph_branches"] = [th_repo["name"].capitalize()]
                REPOSITORIES[th_repo["name"]] = repo

        with open(REPOSITORIES_FILE, "wb") as fd:
            json.dump(REPOSITORIES, fd)

    return REPOSITORIES

def make_task_graph(public_key, signing_pvt_key, product,
                    root_template="release_graph.yml.tmpl",
                    template_dir=DEFAULT_TEMPLATE_DIR,
                    **template_kwargs):
    # TODO: some validation of template_kwargs + defaults
    env = Environment(
        loader=FileSystemLoader(path.join(template_dir, product)),
        undefined=StrictUndefined,
        extensions=['jinja2.ext.do'])
    th = TreeherderClient()

    now = arrow.now()
    now_ms = now.timestamp * 1000

    # Don't let the signing pvt key leak into the task graph.
    with open(signing_pvt_key) as f:
        pvt_key = f.read()

    template = env.get_template(root_template)
    template_vars = {
        "product": product,
        "stableSlugId": stableSlugId(),
        "chunkify": chunkify,
        "sorted": sorted,
        "now": now,
        "now_ms": now_ms,
        # This is used in defining expirations in tasks. There's no way to
        # actually tell Taskcluster never to expire them, but 1,000 years
        # is as good as never....
        "never": arrow.now().replace(years=1000),
        # Treeherder expects 12 symbols in revision
        "revision_hash": th.get_resultsets(
            template_kwargs["branch"],
            revision=template_kwargs["revision"][:12])[0]["revision_hash"],
        "get_treeherder_platform": treeherder_platform,
        "encrypt_env_var": lambda *args: encryptEnvVar(*args,
                                                       keyFile=public_key),
        "buildbot2ftp": buildbot2ftp,
        "buildbot2bouncer": buildbot2bouncer,
        "sign_task": partial(sign_task, pvt_key=pvt_key),
    }
    template_vars.update(template_kwargs)

    return yaml.safe_load(template.render(**template_vars))

    def post_request(self, machine, project, job_collection):
        logger.debug('AutophoneTreeherder.post_request: %s' % job_collection.__dict__)
        logger.debug('AutophoneTreeherder shared_lock.acquire')
        self.shared_lock.acquire()
        try:
            client = TreeherderClient(protocol=self.protocol, host=self.server)

            for attempt in range(1, self.retries+1):
                try:
                    client.post_collection(
                        project,
                        self.credentials[project]['consumer_key'],
                        self.credentials[project]['consumer_secret'],
                        job_collection)
                    return
                except requests.exceptions.Timeout:
                    msg = ('Attempt %d to post result to '
                           'Treeherder timed out.\n\n\n' % attempt)
                    logger.error(msg)
                    if self.mailer:
                        self.mailer.send('Attempt %d for Phone %s failed to post to Treeherder' %
                                         (attempt, machine), msg)
                    time.sleep(self.retry_wait)
                except Exception, e:
                    logger.exception('Error submitting request to Treeherder')
                    if self.mailer:
                        self.mailer.send('Error submitting request to Treeherder',
                                         'Phone: %s\n'
                                         'TreeherderClientError: %s\n'
                                         'TreeherderJobCollection %s\n' % (
                                             machine,
                                             e,
                                             job_collection.to_json()))
                    return
            logger.error('Error submitting request to Treeherder')
            if self.mailer:
                self.mailer.send('Error submitting request to Treeherder',
                                 'Phone: %s\n'
                                 'TreeherderClientError: %s\n'
                                 'TreeherderJobCollection %s\n' % (
                                     machine,
                                     e,
                                     job_collection.to_json()))

    def post_request(self, machine, project, job_collection):
        logger.debug('AutophoneTreeherder.post_request: %s' % job_collection.__dict__)
        logger.debug('AutophoneTreeherder shared_lock.acquire')
        self.shared_lock.acquire()
        try:
            auth = TreeherderAuth(self.credentials[project]['consumer_key'],
                                  self.credentials[project]['consumer_secret'],
                                  project)
            client = TreeherderClient(protocol=self.protocol, host=self.server, auth=auth)

            for attempt in range(1, self.retries+1):
                try:
                    client.post_collection(project, job_collection)
                    return
                except Exception, e:
                    logger.exception('Error submitting request to Treeherder')
                    if self.mailer:
                        if e.response:
                            response_json = json.dumps(e.response.json(),
                                                       indent=2, sort_keys=True)
                        else:
                            response_json = None
                        self.mailer.send(
                            'Attempt %d Error submitting request to Treeherder' %
                            attempt,
                            'Phone: %s\n'
                            'TreeherderClientError: %s\n'
                            'Response: %s\n' % (
                                machine,
                                e,
                                response_json))
                time.sleep(self.retry_wait)
            logger.error('Error submitting request to Treeherder')
            if self.mailer:
                self.mailer.send('Error submitting request to Treeherder',
                                 'Phone: %s\n'
                                 'TreeherderClientError: %s\n'
                                 'Response: %s\n'
                                 'TreeherderJobCollection %s\n' % (
                                     machine,
                                     e,
                                     response_json,
                                     job_collection.to_json()))

def on_resultset_action_event(data, message, dry_run):
    # Cleaning mozci caches
    buildjson.BUILDS_CACHE = {}
    query_jobs.JOBS_CACHE = {}
    repo_name = data["project"]
    action = data["action"]
    times = data["times"]
    # Pulse gives us resultset_id, we need to get revision from it.
    resultset_id = data["resultset_id"]
    treeherder_client = TreeherderClient()
    # We do not handle 'cancel_all' action right now, so skip it.
    if action == "cancel_all":
        message.ack()
        return
    LOG.info("%s action requested by %s on repo_name %s with resultset_id: %s" %
                (data['action'], data["requester"], data["project"], data["resultset_id"]))
    revision = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]["revision"]
    status = None

    if action == "trigger_missing_jobs":
        trigger_missing_jobs_for_revision(repo_name, revision, dry_run=dry_run)
        if not dry_run:
            status = 'trigger_missing_jobs request sent'
        else:
            status = 'Dry-mode, no request sent'
    elif action == "trigger_all_talos_jobs":
        trigger_all_talos_jobs(repo_name, revision, times, dry_run=dry_run)
        if not dry_run:
            status = 'trigger_all_talos_jobs %s times request sent' % times
        else:
            status = 'Dry-mode, no request sent'
    # Send a pulse message showing what we did
    message_sender = MessageHandler()
    pulse_message = {
        'resultset_id': resultset_id,
        'action': action,
        'requester': data['requester'],
        'status': status}
    routing_key = '{}.{}'.format(repo_name, action)
    message_sender.publish_message(pulse_message, routing_key)
    # We need to ack the message to remove it from our queue
    message.ack()

    def submit(self, job, logs=None):
        logs = logs or []

        # We can only submit job info once, so it has to be done in completed
        if self._job_details:
            job.add_artifact('Job Info', 'json', {'job_details': self._job_details})

        job_collection = TreeherderJobCollection()
        job_collection.add(job)

        print('Sending results to Treeherder: {}'.format(job_collection.to_json()))
        url = urlparse(self.url)
       
        client = TreeherderClient(protocol=url.scheme, host=url.hostname,
                                  client_id=self.client_id, secret=self.secret)
        client.post_collection(self.repository, job_collection)

        print('Results are available to view at: {}'.format(
            urljoin(self.url,
                    JOB_FRAGMENT.format(repository=self.repository, revision=self.revision))))

 def __init__(self, ldap_auth, is_triggerbot_user=lambda _: True):
     self.revmap = defaultdict(dict)
     self.revmap_threshold = TreeWatcher.revmap_threshold
     self.auth = ldap_auth
     self.lower_trigger_limit = TreeWatcher.default_retry * TreeWatcher.per_push_failures
     self.log = logging.getLogger('trigger-bot')
     self.is_triggerbot_user = is_triggerbot_user
     self.global_trigger_count = 0
     self.treeherder_client = TreeherderClient()
     self.hidden_builders = set()
     self.refresh_builder_counter = 0

    def submit_results(self, job):
        job.add_project(self.project)
        job.add_revision_hash(self.retrieve_revision_hash())
        job.add_submit_timestamp(int(time.time()))

        job_collection = TreeherderJobCollection()
        job_collection.add(job)

        # self.logger.info
        print('Sending results to Treeherder: %s' % job_collection.to_json())

        url = urlparse(self.url)
        client = TreeherderClient(protocol=url.scheme, host=url.hostname,
                                  client_id=self.client_id, secret=self.secret)
        client.post_collection(self.project, job_collection)

        # self.logger.info
        print('Results are available to view at: %s' % (
            urljoin(self.url,
                    REVISON_FRAGMENT % (self.project, self.revision))))