def isMozpoolReady(device):
    """ Checks if the mozpool server is available and the device is in 'ready' state

    Returns MOZPOOL_STATE_READY if Mozpool is indeed listing device as ready
    Returns MOZPOOL_STATE_UNKNOWN if Mozpool claims the device is in a non-ready state
    Returns MOZPOOL_STATE_ERROR if Mozpool reports some other error.
    Returns MOZPOOL_STATE_MISSING if there is no Mozpool server found in DNS.
    """
    import socket
    default_timeout = socket.getdefaulttimeout()
    socket.setdefaulttimeout(5)  # Don't let networking delay us too long
    try:
        socket.gethostbyname(MOZPOOL_CNAME)
    except:
        log.info("No mozpool server in this VLAN")
        return MOZPOOL_STATE_MISSING
    finally:
        # Set socket timeout back
        socket.setdefaulttimeout(default_timeout)

    mpc = MozpoolHandler("http://%s" % MOZPOOL_CNAME, log)
    try:
        result = mpc.query_device_state(device)
    except MozpoolException as e:
        log.error("Unable to get mozpool state, mozpool returned error: %s" % sys.exc_info()[1])
        return MOZPOOL_STATE_ERROR

    if result['state'] == "ready":
        log.debug("Mozpool state is 'ready'")
        return MOZPOOL_STATE_READY
    else:
        log.error("Mozpool state is '%s'" % result['state'])
        return MOZPOOL_STATE_UNKNOWN

def checkSDCard(dm):
    """ Attempt to write a temp file to the SDCard

    We use this existing verify script as the source of the temp file

    Returns False on failure, True on Success
    """
    if not dmAlive(dm):
        return False

    try:
        if not dm.dirExists("/mnt/sdcard"):
            setFlag(errorFile, "Remote Device Error: Mount of sdcard does not seem to exist")
            return False
        if dm.fileExists("/mnt/sdcard/writetest"):
            log.info("INFO: /mnt/sdcard/writetest left over from previous run, cleaning")
            dm.removeFile("/mnt/sdcard/writetest")
        log.info("INFO: attempting to create file /mnt/sdcard/writetest")
        if not dm.pushFile(os.path.join(os.path.abspath(os.path.dirname(__file__)), "verify.py"), "/mnt/sdcard/writetest"):
            setFlag(
                errorFile, "Remote Device Error: unable to write to sdcard")
            return False
        if not dm.fileExists("/mnt/sdcard/writetest"):
            setFlag(errorFile, "Remote Device Error: Written tempfile doesn't exist on inspection")
            return False
        if not dm.removeFile("/mnt/sdcard/writetest"):
            setFlag(errorFile, "Remote Device Error: Unable to cleanup from written tempfile")
            return False
    except Exception, e:
        setFlag(errorFile, "Remote Device Error: Unknown error while testing ability to write to "
                           "sdcard, see following exception: %s" % e)
        return False

def main(argv):

    if (len(argv) < 3):
        print "usage: installWatcher.py <ip address> <localfilename>"
        sys.exit(1)

    ip_addr = argv[1]
    path_to_main_apk = argv[2]

    dm = droid.DroidSUT(ip_addr)
    if not dm:
        log.error("could not get device manager!")
        return 1
    devRoot = checkDeviceRoot(dm)

    if install(dm, devRoot, path_to_main_apk, "com.mozilla.watcher"):
        log.error("install failed")
        return 1

    dm.reboot()
    waitForDevice(dm)

    # we can't use the following dm.getFile method, since only Watcher runtime user has read access
    # to this file, so we need to be root to read it, hence the slightly more convoluted version
    # using dm._runCmds below...
    # cannot use -> print dm.getFile('/data/data/com.mozilla.watcher/files/version.txt')
    status = dm._runCmds([{'cmd': 'exec su -c "cat /data/data/com.mozilla.watcher/files/version.txt"'}]).split('\n')
    log.info('Watcher version: %s' % status)
    return 0

def setWatcherINI(dm):
    """ If necessary Installs the (correct) watcher.ini for our infra

    Returns False on failure, True on Success
    """
    import hashlib
    realLoc = "/data/data/com.mozilla.watcher/files/watcher.ini"
    currentHash = hashlib.md5(watcherINI).hexdigest()

    def watcherDataCurrent():
        remoteFileHash = dm._getRemoteHash(realLoc)
        if currentHash != remoteFileHash:
            return False
        else:
            return True

    if not dmAlive(dm):
        return False

    try:
        if watcherDataCurrent():
            return True
    except:
        setFlag(errorFile, "Unable to identify if watcher.ini is current")
        return False

    tmpname = '/mnt/sdcard/watcher.ini'
    try:
        dm._runCmds([{'cmd': 'push %s %s' % (tmpname, len(
            watcherINI)), 'data': watcherINI}])
    except devicemanager.AgentError, err:
        log.info("Error while pushing watcher.ini: %s" % err)
        setFlag(errorFile, "Unable to properly upload the watcher.ini")
        return False

def doUpdate(dm):
    _oldDebug = dm.debug
    log.info("updateSUT.py: We're going to try to install SUTAgentAndroid Version %s" % target_version)
    try:
        data = download_apk()
    except Exception, e:
        log.error("Automation Error: updateSUT.py: We have failed to retrieve the SUT Agent. %s" % str(e))
        return RETCODE_APK_DL_FAILED

def download_apk():
    url = 'http://talos-bundles.pvt.build.mozilla.org/mobile/sutAgentAndroid.%s.apk' % target_version
    log.info("INFO: updateSUT.py: We're downloading the apk: %s" % url)
    req = urllib2.Request(url)
    try:
        f = urllib2.urlopen(req)
    except urllib2.URLError, e:
        reason = getattr(e, "reason", "SUT-Undef")
        code = getattr(e, "code", "SUT-Undef")
        raise Exception("Automation Error: updateSUT.py: code: %s; reason: %s" % (code, reason))

def checkVersion(dm, flag=False):
    """ Verify SUTAgent Version

    Returns False on failure, True on Success
    """
    if not dmAlive(dm):
        return False

    ver = updateSUT.version(dm)
    if not updateSUT.isVersionCorrect(ver=ver):
        if flag:
            setFlag(errorFile, "Remote Device Error: Unexpected ver on device, got '%s' expected '%s'" %
                    (ver, "SUTAgentAndroid Version %s" % updateSUT.target_version))
        return False
    log.info(
        "INFO: Got expected SUTAgent version '%s'" % updateSUT.target_version)
    return True

def cleanupDevice(device=None, dm=None):
    assert ((device is not None) or (dm is not None))  # Require one to be set

    if not device:
        device = os.environ['SUT_NAME']
    pidDir = os.path.join('/builds/', device)
    errorFile = os.path.join(pidDir, 'error.flg')
    reboot_needed = False

    processNames = ['org.mozilla.fennec_aurora',
                    'org.mozilla.fennec_unofficial',
                    'org.mozilla.fennec',
                    'org.mozilla.firefox_beta',
                    'org.mozilla.firefox',
                    'org.mozilla.roboexample.test',
                    ]

    if dm is None:
        log.info("Connecting to: " + device)
        dm = devicemanager.DeviceManagerSUT(device)
        dm.debug = 5

    packages = dm._runCmds([{'cmd': 'exec pm list packages'}])
    for package in packages.split('\n'):
        if not package.strip().startswith("package:"):
            continue  # unknown entry
        package_basename = package.strip()[8:]
        for proc in processNames:
            if package_basename == "%s" % proc or \
                    package_basename.startswith("%s_" % proc):
                log.info("Uninstalling %s..." % package_basename)
                try:
                    if 'panda' in device:
                        dm.uninstallApp(package_basename)
                        reboot_needed = True
                    else:
                        dm.uninstallAppAndReboot(package_basename)
                        waitForDevice(dm)
                except devicemanager.DMError, err:
                    setFlag(errorFile,
                            "Remote Device Error: Unable to uninstall %s and reboot: %s" % (package_basename, err))
                    return RETCODE_ERROR
                finally:
                    break  # Don't try this proc again, since we already matched

def canPing(device):
    """ Check a device is reachable by ping

    Returns False on failure, True on Success
    """
    curRetry = 0
    log.info("INFO: attempting to ping device")
    while curRetry < MAX_RETRIES:
        ret, _ = pingDevice(device)
        if not ret:
            curRetry += 1
            if curRetry == MAX_RETRIES:
                setFlag(errorFile, "Automation Error: Unable to ping device after %s attempts" % MAX_RETRIES)
                return False
            else:
                log.info("INFO: Unable to ping device after %s try. Sleeping for 90s then retrying" % curRetry)
                time.sleep(90)
        else:
            break  # we're done here
    return True

def main(argv):

    if len(argv) < 2:
        print "usage: installWatcher.py <ip address>"
        sys.exit(1)

    ip_addr = argv[1]

    dm = droid.DroidSUT(ip_addr)
    if not dm:
        log.error("could not get device manager!")
        return 1
    devRoot = checkDeviceRoot(dm)

    # we can't use the following dm.getFile method, since only Watcher runtime user has read access
    # to this file, so we need to be root to read it, hence the slightly more convoluted version
    # using dm._runCmds below...
    # cannot use -> print dm.getFile('/data/data/com.mozilla.watcher/files/version.txt')
    status = dm._runCmds([{"cmd": 'exec su -c "cat /data/data/com.mozilla.watcher/files/version.txt"'}]).split("\n")
    log.info("Watcher version: %s" % status)
    return 0

def installOneApp(dm, devRoot, app_file_local_path, errorFile, logcat=True):

    source = app_file_local_path
    filename = os.path.basename(source)
    target = os.path.join(devRoot, filename)

    log.info("Installing %s" % target)
    if dm.pushFile(source, target):
        status = dm.installApp(target)
        if status is None:
            log.info('-' * 42)
            log.info('installApp() done - gathering debug info')
            dm.getInfo('process')
            dm.getInfo('memory')
            dm.getInfo('uptime')
            if logcat:
                try:
                    print dm._runCmds([{'cmd': 'exec su -c "logcat -d -v time *:W"'}])
                except devicemanager.DMError, e:
                    setFlag(errorFile, "Remote Device Error: Exception hit while trying to run logcat: %s" % str(e))
                    return 1
        else:
            setFlag(errorFile,
                    "Remote Device Error: updateApp() call failed - exiting")
            return 1

def smoketest(device_name, number):
    global dm
    global appFileName, processName

    dm = devicemanager.DeviceManagerSUT(device_name, 20701)
    deviceRoot = dm.getDeviceRoot()

    # This does all the steps of verify.py including the cleanup.
    if verifyDevice(device_name, checksut=False, doCheckStalled=False) is False:
        log.error("failed to run verify on %s" % (device_name))
        return 1  # Not ok to proceed
    log.info("Successfully verified the device")

    if dm._sock:
        dm._sock.close()
    time.sleep(30)
    dm = devicemanager.DeviceManagerSUT(device_name, 20701)
    print "in smoketest, going to call installOneApp with dm: %s, %s" \
          % (dm, dm._sock)
    if installOneApp(dm, deviceRoot, os.path.abspath(appFileName), None, logcat=False):
        log.error("failed to install %s on device %s" % (appFileName, device_name))
        return 1
    log.info("Successfully installed the application")

    if not runTests(device_name, processName, number):
        log.error("failed to run dom tests on %s" % (device_name))
        return 1
    log.info("Successfully ran the mochitests")
    return 0

def install(dm, devRoot, app_file_local_path, package):
    source = app_file_local_path
    filename = os.path.basename(source)
    target = os.path.join(devRoot, filename)

    log.info("Pushing new apk to the device: %s" % target)
    status = dm.pushFile(source, target)

    dm.shellCheckOutput(["dd", "if=%s" % target, "of=/data/local/watcher.apk"], root=True)
    dm.shellCheckOutput(["chmod", "666", "/data/local/watcher.apk"], root=True)

    log.info("Uninstalling %s" % package)
    try:
        dm.uninstallApp(package)
        log.info('uninstallation successful')
    except devicemanager.DMError, e:
        log.info('uninstallation failed -- maybe not installed? '+str(e))

def main(argv):
    global ip_addr

    if (len(argv) < 4):
        log.info("usage: logcat.py <device ip address> <output filename> <logcat options>")
        return 1
    ip_addr = argv[1]
    output_filename = argv[2]
    logcat_options = argv[3]
    max_runtime = 3600   # 3600 seconds == 1 hour

    status = 0
    dm = devicemanager.DeviceManagerSUT(ip_addr)
    if not dm:
        log.error("logcat.py: unable to open device manager")
        return 2
    command = 'execext su t=%d logcat %s' % (max_runtime, logcat_options)
    log.debug('logcat.py running SUT command: %s' % command)
    try:
        with open(output_filename, 'w') as f:
            dm._sendCmds([{'cmd': command}], f)
    except devicemanager.DMError, e:
        log.error("Remote Device Error: Exception caught running logcat: %s" % str(e))
        status = -1

def canTelnet(device):
    """ Checks if we can establish a Telnet session (via devicemanager)

    Sets global `dm`
    Returns False on failure, True on Success
    """
    global dm
    curRetry = 0
    sleepDuration = 0
    while curRetry < MAX_RETRIES:
        try:
            dm = connect(device, sleepDuration)
        except:
            curRetry += 1
            if curRetry == MAX_RETRIES:
                setFlag(errorFile, "Automation Error: Unable to connect to device after %s attempts" % MAX_RETRIES)
                return False
            else:
                log.info("INFO: Unable to connect to device after %s try" %
                         curRetry)
                sleepDuration = 90
        else:
            break  # We're done here
    return True

def checkAndFixScreen(dm, device):
    """ Verify the screen is set as we expect

    If the screen is incorrectly set, this function attempts to fix it,
    which ends up requiring a reboot of the device.

    Returns False if screen is wrong, True if correct
    """
    if not dmAlive(dm):
        return False

    # Verify we have the expected screen resolution
    info = dm.getInfo("screen")
    if not info["screen"][0] == EXPECTED_DEVICE_SCREEN:
        setFlag(errorFile, "Remote Device Error: Unexpected Screen on device, got '%s' expected '%s'" %
               (info["screen"][0], EXPECTED_DEVICE_SCREEN))
        if not dm.adjustResolution(**EXPECTED_DEVICE_SCREEN_ARGS):
            setFlag(errorFile, "Command to update resolution returned failure")
        else:
            soft_reboot(dm=dm, device=device)
                        # Reboot sooner than cp would trigger a hard Reset
        return False
    log.info("INFO: Got expected screen size '%s'" % EXPECTED_DEVICE_SCREEN)
    return True

def runTests(device, proc_name, number):
    numfailed = -1
    log.info("Starting to run tests")
    httpport = 8000 + int(number)

    # TODO: fix the host/port information so we don't have conflicts in
    # parallel runs
    cmd = ["python",
           os.path.join(os.path.dirname(__file__), "tests/mochitest/runtestsremote.py"),
           "--app=%s" % proc_name,
           "--deviceIP=%s" % device,
           "--xre-path=%s" % os.path.join(os.path.dirname(__file__), "xre"),
           "--utility-path=%s" % os.path.join(os.path.dirname(__file__), "bin"),
           # "--test-path=dom/tests/mochitest/dom-level0",
           "--test-path=dom/tests/mochitest",
           "--http-port=%s" % httpport]
    log.info("Going to run test: %s" % subprocess.list2cmdline(cmd))
    proc = ""
    try:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        proc = p.communicate()[0]
    except:
        log.error("Exception found while running unittests: %s" % sys.exc_info()[1])
    log.info("Finished running mochitests")
    refinished = re.compile('([0-9]+) INFO SimpleTest FINISHED')
    refailed = re.compile('([0-9]+) INFO Failed: ([0-9]+)')
    for line in proc.split('\n'):
        log.debug(line)
        finished = refinished.match(line)
        failed = refailed.match(line)
        if failed:
            numfailed = int(failed.group(2))
        if finished:
            if numfailed > 0:
                log.error("Found %s failures while running mochitest"
                          % numfailed)
                return False
            return True
    return False

def verifyDevice(device, checksut=True, doCheckStalled=True, watcherINI=False):
    # Returns False on failure, True on Success
    global dm, errorFile
    devicePath = os.path.join('/builds', device)
    errorFile = os.path.join(devicePath, 'error.flg')

    if doCheckStalled:
        if not cleanupFoopy(device):
            log.info("verifyDevice: failing to cleanup foopy")
            return False

    if not canPing(device):
        # TODO Reboot via PDU if ping fails
        log.info("verifyDevice: failing to ping")
        return False

    if not canTelnet(device):
        log.info("verifyDevice: failing to telnet")
        return False

    if not checkSDCard(dm):
        log.info("verifyDevice: failing to check SD card")
        return False

    if checksut and not checkVersion(dm):
        if not updateSUTVersion(dm):
            log.info("verifyDevice: failing to updateSUT")
            return False

    # Resolution Check disabled for now; Bug 737427
    if False and not checkAndFixScreen(dm, device):
        log.info("verifyDevice: failing to fix screen")
        return False

    if not cleanupDevice(device, dm):
        log.info("verifyDevice: failing to cleanup device")
        return False

    if watcherINI:
        if not setWatcherINI(dm):
            log.info("verifyDevice: failing to set watcher.ini")
            return False

    return True

def version(dm):
    ver = dm._runCmds([{'cmd': 'ver'}]).split("\n")[0]
    log.info("INFO: updateSUT.py: We're running %s" % ver)
    return ver

def one_time_setup(ip_addr, major_source):
    ''' One time setup of state

    ip_addr - of the device we want to install app at
    major_source - we've hacked this script to install
            may-also-be-needed tools, but the source we're asked to
            install has the meta data we need

    Side Effects:
        global, needed for error reporting:
            errorFile
    '''

    # set up the flag files, used throughout
    cwd = os.getcwd()
    global errorFile
    errorFile = os.path.join(cwd, '..', 'error.flg')
    deviceName = os.path.basename(cwd)

    proxyIP = getOurIP()
    proxyPort = calculatePort()

    workdir = os.path.dirname(major_source)
    inifile = os.path.join(workdir, 'fennec', 'application.ini')
    remoteappini = os.path.join(workdir, 'talos', 'remoteapp.ini')
    log.info('copying %s to %s' % (inifile, remoteappini))
    runCommand(['cp', inifile, remoteappini])

    log.info("connecting to: %s" % ip_addr)
    dm = devicemanager.DeviceManagerSUT(ip_addr)
# Moar data!
    dm.debug = 3

    devRoot = checkDeviceRoot(dm)

    if devRoot is None or devRoot == '/tests':
        setFlag(errorFile, "Remote Device Error: devRoot from devicemanager [%s] is not correct - exiting" % devRoot)
        return None, None

    try:
        log.info("%s, %s" % (proxyIP, proxyPort))
        getDeviceTimestamp(dm)
        setDeviceTimestamp(dm)
        getDeviceTimestamp(dm)
        dm.getInfo('process')
        dm.getInfo('memory')
        dm.getInfo('uptime')

        width, height = getResolution(dm)
        # adjust resolution down to allow fennec to install without memory
        # issues
        if (width == 1600 or height == 1200):
            dm.adjustResolution(1024, 768, 'crt')
            log.info('forcing device reboot')
            if not powermanagement.soft_reboot_and_verify(device=deviceName, dm=dm, ipAddr=proxyIP, port=proxyPort):
                return None, None

            width, height = getResolution(dm)
            if width != 1024 and height != 768:
                setFlag(errorFile, "Remote Device Error: Resolution change failed.  Should be %d/%d but is %d/%d" % (1024, 768, width, height))
                return None, None

    except devicemanager.AgentError, err:
        log.error("remoteDeviceError: while doing one time setup for installation: %s" % err)
        return None, None