def make_generic_head_request(page_url):
    """Make generic HEAD request to check page existence"""
    def _get():
        req = requests.head(page_url, timeout=60)
        req.raise_for_status()

    retry(_get, attempts=5, sleeptime=1)

def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
    remote = make_hg_url(HG, repo)
    mercurial(remote, reponame)

    def tagRepo(repo, attempt, config, revision, tags):
        # set totalChangesets=1 because tag() generates exactly 1 commit
        totalChangesets = 1
        # update to the desired revision first, then to the tip of revision's
        # branch to avoid new head creation
        update(repo, revision=revision)
        update(repo)
        tag(repo, revision, tags, config['hgUsername'])
        outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
                                              ssh_username=config[
                                                  'hgUsername'],
                                              ssh_key=config['hgSshKey']))
        if len(outgoingRevs) != totalChangesets:
            raise Exception("Wrong number of outgoing revisions")

    pushRepo = make_hg_url(HG, repo, protocol='ssh')

    def tag_wrapper(r, n):
        tagRepo(r, n, config, revision, tags)

    def cleanup_wrapper():
        cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
                          config['hgSshKey'])
    retry(apply_and_push, cleanup=cleanup_wrapper,
          args=(reponame, pushRepo, tag_wrapper, pushAttempts),
          kwargs=dict(ssh_username=config['hgUsername'],
                      ssh_key=config['hgSshKey']))

def do_reboot(slaveapi, slave):
    url = furl(slaveapi)
    url.path.add("slaves").add(slave).add("actions").add("reboot")
    retry(requests.post, args=(str(url),))
    # Because SlaveAPI fully escalates reboots (all the way to IT bug filing),
    # there's no reason for us to watch for it to complete.
    log.info("%s - Reboot queued", slave)
    return

def action_update(master):
    print "sleeping 30 seconds to make sure that hg.m.o syncs NFS... ",
    time.sleep(30)
    print OK
    with show('running'):
        retry(run, args=('source bin/activate && make update',),
                kwargs={'workdir': master['basedir']}, sleeptime=10,
                retry_exceptions=(SystemExit,))
    print OK, "updated %(hostname)s:%(basedir)s" % master

def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
                 l10nIni, compareLocalesRepo, env, merge=True,
                 prevMar=None, productName=None, platform=None,
                 version=None, oldVersion=None):
    repo = "/".join([l10nBaseRepo, locale])
    localeDir = path.join(l10nRepoDir, locale)
    retry(mercurial, args=(repo, localeDir))
    update(localeDir, revision=revision)
    
    compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir,
                   l10nIni, revision=revision, merge=merge)
    env["AB_CD"] = locale
    env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
    if sys.platform.startswith('win'):
        env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
    if sys.platform.startswith('darwin'):
        env["MOZ_PKG_PLATFORM"] = "mac"
    run_cmd(["make", "installers-%s" % locale], cwd=localeSrcDir, env=env)
    UPLOAD_EXTRA_FILES = []
    if prevMar:
        nativeDistDir = path.normpath(path.abspath(path.join(localeSrcDir,
                                                             '../../dist')))
        posixDistDir = windows2msys(nativeDistDir)
        mar = '%s/host/bin/mar' % posixDistDir
        mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
        current = '%s/current' % posixDistDir
        previous = '%s/previous' % posixDistDir
        updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
        updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
        current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version)
        partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
                                                     version)
        partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
        UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
        env['MAR'] = mar
        env['MBSDIFF'] = mbsdiff
        run_cmd(['rm', '-rf', previous, current])
        run_cmd(['mkdir', previous, current])
        run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
                 '../../../../%s' % prevMar],
                cwd=path.join(nativeDistDir, 'previous'), env=env)
        run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
                 current_mar], cwd=path.join(nativeDistDir, 'current'), env=env)
        run_cmd(
            ['bash', '../../tools/update-packaging/make_incremental_update.sh',
             partial_mar, previous, current], cwd=nativeDistDir, env=env)
        if os.environ.get('MOZ_SIGN_CMD'):
            run_cmd(['bash', '-c',
                     '%s -f gpg -f mar "%s"' %
                     (os.environ['MOZ_SIGN_CMD'], partial_mar)],
                     env=env)
            UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name))

    retry(run_cmd,
          args=(["make", "upload", "AB_CD=%s" % locale,
                 'UPLOAD_EXTRA_FILES=%s' % ' '.join(UPLOAD_EXTRA_FILES)],),
          kwargs={'cwd': localeSrcDir, 'env': env})

def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
            pushAttempts, defaultBranch='default'):
    remote = make_hg_url(HG, repo)
    retry(mercurial, args=(remote, reponame))

    def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
                     defaultBranch):
        # set relbranchChangesets=1 because tag() generates exactly 1 commit
        relbranchChangesets = 1
        defaultBranchChangesets = 0

        if relbranch in get_branches(reponame):
            update(reponame, revision=relbranch)
        else:
            update(reponame, revision=revision)
            run_cmd(['hg', 'branch', relbranch], cwd=reponame)

        if len(bumpFiles) > 0:
            # Bump files on the relbranch, if necessary
            bump(reponame, bumpFiles, 'version')
            run_cmd(['hg', 'diff'], cwd=repo)
            try:
                get_output(['hg', 'commit', '-u', config['hgUsername'],
                            '-m', getBumpCommitMessage(config['productName'], config['version'])],
                           cwd=reponame)
                relbranchChangesets += 1
            except subprocess.CalledProcessError, e:
                # We only want to ignore exceptions caused by having nothing to
                # commit, which are OK. We still want to raise exceptions caused
                # by any other thing.
                if e.returncode != 1 or "nothing changed" not in e.output:
                    raise

        # We always want our tags pointing at the tip of the relbranch
        # so we need to grab the current revision after we've switched
        # branches and bumped versions.
        revision = get_revision(reponame)
        # Create the desired tags on the relbranch
        tag(repo, revision, tags, config['hgUsername'])

        # This is the bump of the version on the default branch
        # We do it after the other one in order to get the tip of the
        # repository back on default, thus avoiding confusion.
        if len(bumpFiles) > 0:
            update(reponame, revision=defaultBranch)
            bump(reponame, bumpFiles, 'nextVersion')
            run_cmd(['hg', 'diff'], cwd=repo)
            try:
                get_output(['hg', 'commit', '-u', config['hgUsername'],
                            '-m', getBumpCommitMessage(config['productName'], config['version'])],
                           cwd=reponame)
                defaultBranchChangesets += 1
            except subprocess.CalledProcessError, e:
                if e.returncode != 1 or "nothing changed" not in e.output:
                    raise

def maybe_delete_repo(server, username, sshKey, repo, repoPath):
    reponame = get_repo_name(repo)
    repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
    try:
        log.info("Trying to open %s" % repo_url)
        urlopen(repo_url)
    except URLError:
        log.info("%s doesn't exist, not deleting" % reponame)
    else:
        log.info('Deleting %s' % reponame)
        retry(run_remote_cmd, args=('edit %s delete YES' % reponame, server,
                                    username, sshKey))

def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
                  mozconfigPath, objdir, makeDirs, locales, ftpProduct,
                  appName, version, appVersion, buildNumber, stageServer,
                  stageUsername, stageSshKey, compareLocalesRepo, merge,
                  platform, stage_platform, brand, mobileDirName):
    sourceRepoName = path.split(sourceRepo)[-1]
    nightlyDir = "candidates"
    localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
    # Even on Windows we need to use "/" as a separator for this because
    # compare-locales doesn"t work any other way
    l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])

    env = {
        "MOZ_OBJDIR": objdir,
        "MOZ_PKG_VERSION": version,
        "UPLOAD_HOST": stageServer,
        "UPLOAD_USER": stageUsername,
        "UPLOAD_SSH_KEY": stageSshKey,
        "UPLOAD_TO_TEMP": "1",
        # Android signing
        "JARSIGNER": os.path.join(os.getcwd(), "scripts", "release",
                                  "signing", "mozpass.py")
    }
    build.misc.cleanupObjdir(sourceRepoName, objdir, mobileDirName)
    retry(mercurial, args=(sourceRepo, sourceRepoName))
    update(sourceRepoName, revision=revision)
    l10nRepackPrep(sourceRepoName, objdir, mozconfigPath,
                   l10nRepoDir, makeDirs, localeSrcDir, env)
    fullCandidatesDir = makeCandidatesDir(appName, version, buildNumber,
                                      protocol='http', server=stageServer,
                                      nightlyDir=nightlyDir)
    input_env = retry(downloadReleaseBuilds,
                      args=(stageServer, ftpProduct, brand, version,
                            buildNumber, stage_platform, fullCandidatesDir))
    env.update(input_env)
    print "env pre-locale: %s" % str(env)

    failed = []
    for l in locales:
        try:
            # adding locale into builddir
            env["POST_UPLOAD_CMD"] = postUploadCmdPrefix(
                to_mobile_candidates=True,
                product=appName,
                version=version,
                builddir='%s/%s' % (stage_platform, l),
                buildNumber=buildNumber,
                nightly_dir=nightlyDir,)
            print "env post-locale: %s" % str(env)
            repackLocale(str(l), l10nRepoDir, l10nBaseRepo, revision,
                         localeSrcDir, l10nIni, compareLocalesRepo, env, merge)
        except Exception, e:
            failed.append((l, format_exc()))

def commitSVN(targetSVNDirectory, product, version, fakeCommit=False):
    """
    Commit the change in the svn
    """
    retval = os.getcwd()
    os.chdir(targetSVNDirectory)
    try:
        if not os.path.isdir(".svn"):
            raise Exception("Could not find the svn directory")
        commitMSG = "'" + product + " version " + version + "'"
        if not fakeCommit:
            retry(doCommitSVN, args=(commitMSG), attempts=3, sleeptime=0)
    finally:
        os.chdir(retval)

def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni,
                   revision="default", merge=True):
    retry(mercurial, args=(repo, "compare-locales"))
    update("compare-locales", revision=revision)
    mergeDir = path.join(localeSrcDir, "merged")
    if path.exists(mergeDir):
        log.info("Deleting %s" % mergeDir)
        shutil.rmtree(mergeDir)
    run_cmd(["python", path.join("compare-locales", "scripts",
                                 "compare-locales"),
             "-m", mergeDir,
             l10nIni,
             l10nRepoDir, locale],
            env={"PYTHONPATH": path.join("compare-locales", "lib")})

 def process(self):
     """
     Process this patchset, doing the following:
         1. Check permissions on patchset
         2. Clone the repository
         3. Apply patches, with 3 attempts
     """
     # 1. Check permissions on each patch
     outgoing = self.branch if not self.try_run else 'try'
     if not has_sufficient_permissions(self.user, outgoing):
         log.error('Insufficient permissions to push to %s.'
                 % (outgoing))
         self.add_comment('Insufficient permissions to push to %s.'
                 % (outgoing))
         return (False, '\n'.join(self.comments))
     # 2. Clone the repository
     cloned_rev = None
     try:
         cloned_rev = clone_branch(self.branch, self.branch_url)
     except RetryException:
         log.error('[Branch %s] Could not clone from %s.'
                 % (self.branch, self.branch_url))
         self.add_comment('An error occurred while cloning %s.'
                 % (self.branch_url))
         return (False, '\n'.join(self.comments))
     # 3. Apply patches, with 3 attempts
     try:
         # make 3 attempts so that
         # 1st is on current clone,
         # 2nd attempt is after an update -C,
         # 3rd attempt is a fresh clone
         retry(apply_and_push, attempts=3,
                 retry_exceptions=(RetryException,),
                 cleanup=RepoCleanup(self.branch, self.branch_url),
                 args=(self.active_repo, self.push_url,
                       self.apply_patches, 1),
                 kwargs=dict(ssh_username=config['hg_username'],
                             ssh_key=config['hg_ssh_key'],
                             force=self.try_run))    # force only on try
         revision = get_revision(self.active_repo)
         shutil.rmtree(self.active_repo)
         for patch in self.patches:
             patch.delete()
     except (HgUtilError, RetryException, FailException), err:
         # Failed
         log.error('[PatchSet] Could not be applied and pushed.\n%s'
                 % (err))
         self.add_comment('Patchset could not be applied and pushed.'
                          '\n%s' % (err))
         return (False, '\n'.join(self.comments))

def get_production_slaves(slaveapi):
    url = furl(slaveapi)
    url.path.add("slaves")
    url.args["environment"] = "prod"
    url.args["enabled"] = 1
    r = retry(requests.get, args=(str(url),))
    return r.json()["slaves"]

def cleanOutgoingRevs(reponame, remote, username, sshKey):
    outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
                                          ssh_username=username,
                                          ssh_key=sshKey))
    for r in reversed(outgoingRevs):
        run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
                 r[REVISION]], cwd=reponame)

 def request(self, params=None, data=None, method='GET', url_template_vars={}):
     url = self.api_root + self.url_template % url_template_vars
     if method != 'GET' and method != 'HEAD':
         if not self.csrf_token or is_csrf_token_expired(self.csrf_token):
             res = self.session.request(
                 method='HEAD', url=self.api_root + '/csrf_token',
                 config=self.config, timeout=self.timeout,
                 auth=self.auth)
             self.csrf_token = res.headers['X-CSRF-Token']
         data['csrf_token'] = self.csrf_token
     log.debug('Request to %s' % url)
     log.debug('Data sent: %s' % data)
     try:
         return retry(self.session.request, sleeptime=5, max_sleeptime=15,
                      retry_exceptions=(requests.HTTPError, 
                                        requests.ConnectionError),
                      attempts=self.retries,
                      kwargs=dict(method=method, url=url, data=data,
                                  config=self.config, timeout=self.timeout,
                                  auth=self.auth, params=params)
                     )
     except requests.HTTPError, e:
         log.error('Caught HTTPError: %d %s' % 
                  (e.response.status_code, e.response.content), 
                  exc_info=True)
         raise

def ssh(user, identity, host, remote_cmd, port=22):
    cmd = ['ssh', '-l', user]
    if identity:
        cmd.extend(['-i', identity])
    cmd.extend(['-p', str(port), host, remote_cmd])

    return retry(do_cmd, attempts=retries + 1, sleeptime=retry_sleep, args=(cmd,))

def make_generic_get_request(page_url):
    """Make generic GET request to retrieve some page content"""
    def _get():
        req = requests.get(page_url, timeout=60)
        req.raise_for_status()
        return req.content

    return retry(_get, attempts=5, sleeptime=1)

 def tagRepo(repo, attempt, config, revision, tags):
     totalChangesets = len(tags)
     tag(repo, revision, tags, config['hgUsername'])
     outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
                                           ssh_username=config['hgUsername'],
                                           ssh_key=config['hgSshKey']))
     if len(outgoingRevs) != totalChangesets:
         raise Exception("Wrong number of outgoing revisions")

def bump_configs(server, username, sshKey, repo, repoPath, configsToBump,
                 configsToOverride):
    reponame = get_repo_name(repo)
    repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
    pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame),
                               protocol='ssh')
    retry(mercurial, args=(repo_url, reponame))

    def bump(repo, configsToBump, configsToOverride):
        """Process dynamic (version, buildNumber, etc.) variables in
        configsToBump, then append overrides files to both configsToBump and
        configsToOverride."""
        # First pass. Bump variables in configsToBump.
        configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()]
        cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip']
        cmd.extend(configs)
        run_cmd(cmd)
        # Second pass. Append override files to configsToBump and
        # configsToOverride.
        for config, overrides in \
            configsToBump.items() + configsToOverride.items():
            newContent = cat([path.join(repo, config)] +
                          [path.join(repo, x) for x in overrides])
            fh = open(path.join(repo, config), 'wb')
            fh.write(newContent)
            fh.close()
        run_cmd(['hg', 'commit', '-m', 'Automatic config bump'],
                cwd=repo)

    def bump_wrapper(r, n):
        bump(r, configsToBump, configsToOverride)

    def cleanup_wrapper():
        cleanOutgoingRevs(reponame, pushRepo, username, sshKey)

    retry(apply_and_push, cleanup=cleanup_wrapper,
          args=(reponame, pushRepo, bump_wrapper),
          kwargs=dict(ssh_username=username, ssh_key=sshKey))

    tags = []
    for configfile in configsToBump.keys():
        config = readReleaseConfig(path.join(reponame, configfile))
        tags.extend(getTags(config['baseTag'], config['buildNumber'],
                            buildTag=True))
    return tags

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--manifest", required=True)
    parser.add_argument("-a", "--api-root", required=True,
                        help="Balrog API root")
    parser.add_argument("-v", "--verbose", action="store_const",
                        dest="loglevel", const=logging.DEBUG,
                        default=logging.INFO)
    parser.add_argument("--product", help="Override product name from application.ini")
    args = parser.parse_args()
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
                        level=args.loglevel)
    logging.getLogger("requests").setLevel(logging.WARNING)
    logging.getLogger("boto").setLevel(logging.WARNING)

    balrog_username = os.environ.get("BALROG_USERNAME")
    balrog_password = os.environ.get("BALROG_PASSWORD")
    suffix = os.environ.get("BALROG_BLOB_SUFFIX")
    if not balrog_username and not balrog_password:
        raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
                           "variables should be set")
    if not suffix:
        raise RuntimeError("BALROG_BLOB_SUFFIX environment variable should be set")

    manifest = json.load(open(args.manifest))
    auth = (balrog_username, balrog_password)

    for e in manifest:
        complete_info = [{
            "hash": e["hash"],
            "size": e["size"],
        }]

        submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
                                       suffix=suffix)
        productName = args.product or e["appName"]
        retry(lambda: submitter.run(
            platform=e["platform"], productName=productName,
            version=e["toVersion"],
            build_number=e["toBuildNumber"],
            appVersion=e["version"], extVersion=e["version"],
            buildID=e["to_buildid"], locale=e["locale"],
            hashFunction='sha512', completeInfo=complete_info),
            attempts=30, sleeptime=10, max_sleeptime=60, jitter=3,
        )

def scp(user, identity, host, files, remote_dir, port=22):
    cmd = ['scp']
    if identity:
        cmd.extend(['-i', identity])
    cmd.extend(['-P', str(port)])
    cmd.extend(files)
    cmd.append("%[email protected]%s:%s" % (user, host, remote_dir))

    return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd,))