def test(cmd):
    util.info("")
    util.info("- Starting " + cmd)
    util.info("")
    util.run(cmd)
    
    start = time.time()
    
    clientlist = []

    for i in range(0, NUM_CLIENT):
        client = testit("Client-" + str(i))
        clientlist.append(client)
        client.start()
    
    for client in clientlist:
        client.join()


    end = time.time()
    util.info("Elapsed time (in seconds): " + str(end-start))

    if util.is_server_alive(cmd) == -1:
        util.error("Ouch! Server is dead!"
                   " Your bounded buffered may not be well protected");

def run_on_kubernetes(args):
    create_gcloud_secret()
    context   = util.get_cluster_prefix()
    namespace = util.get_current_namespace()
    if len(args.number) == 0:
        # Figure out the nodes based on the names of persistent disks, or just node 0 if none.
        args.number = range(max(1,len(get_persistent_disks(context, namespace))))
    if 'storage-projects' not in util.get_services():
        util.run(['kubectl', 'create', '-f', 'conf/service.yaml'])
    args.local = False # so tag is for gcloud

    tag = util.get_tag(args, NAME, build)
    if not args.tag:
        tag = tag[:tag.rfind('-')]   # get rid of the final -[service] part of the tag.

    t = open(join('conf', '{name}.template.yaml'.format(name=NAME))).read()

    ensure_ssh()
    for number in args.number:
        deployment_name = "{name}{number}".format(name=NAME, number=number)
        ensure_persistent_disk_exists(context, namespace, number, args.size, args.type)
        with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as tmp:
            tmp.write(t.format(image         = tag,
                               number        = number,
                               gcloud_bucket = gcloud_bucket(namespace=namespace),
                               pd_name       = pd_name(context=context, namespace=namespace, number=number),
                               health_delay  = args.health_delay,
                               pull_policy   = util.pull_policy(args)))
            tmp.flush()
            util.update_deployment(tmp.name)

    def __init__(self, vlab, stdin=sys.stdin):
        """Instantiates a CLI object
        :param vlab: Vlab class to be run"""
        self.stdin = stdin
        self.vlab = vlab
        Cmd.__init__(self)
        print('Starting CLI:\n')

        # Setup history if readline is available
        try:
            import readline
        except ImportError:
            pass
        else:
            history_path = os.path.expanduser('~/.vlab_history')
            if os.path.isfile(history_path):
                readline.read_history_file(history_path)
            atexit.register(
                lambda: readline.write_history_file(history_path))

        while True:
            try:
                if self.isatty():
                    run('stty sane')
                self.cmdloop()
                break
            except KeyboardInterrupt:
                print('\nInterrupt\n')

def install_kubernetes(args):
    version = args.version
    if not version:
        version = util.run("curl -s https://github.com/kubernetes/kubernetes/releases | grep kubernetes/tree",
                 get_output=True).splitlines()[0].split("tree/v")[1].split('"')[0]
        print("using latest version '%s'"%version)
    install_path = os.path.join(os.environ['HOME'], 'install')
    link_path = os.path.join(os.environ['HOME'], 'kubernetes')
    if not os.path.exists(install_path):
        os.makedirs(install_path)
    if os.path.exists(link_path) and not os.path.islink(link_path):
        raise RuntimeError("Please manually remove '%s'"%link_path)
    target_path = os.path.join(install_path, 'kubernetes-v%s'%version)
    if not os.path.exists(target_path):
        target = os.path.join(install_path, 'kubernetes.tar.gz')
        if os.path.exists(target):
            os.unlink(target)
        util.run(['wget', 'https://github.com/kubernetes/kubernetes/releases/download/v%s/kubernetes.tar.gz'%version],
                path = install_path)
        util.run(['tar', 'zvxf', target], path=install_path)
        os.unlink(target)
        shutil.move(os.path.join(install_path, 'kubernetes'), target_path)
    if os.path.exists(link_path):
        os.unlink(link_path)
    os.symlink(target_path, link_path)

def test(cmd):
    global expected
    global got
    global count

    util.info("")
    util.info("- Starting " + cmd)
    util.info("")
    util.run(cmd)
    
    start = time.time()

    clientlist = []
    expected = []
    for i in range(1, NUM_CLIENT):
        expected.append(commands.getoutput("cat ./testdata/file%s.txt" % str(i)))

    commands.getoutput("rm -rf %s" % tmpfile)

    for i in range(0, NUM_CLIENT):
        client = testit("Client-" + str(i), i)
        clientlist.append(client)
        client.start()
        time.sleep(0.3)
    
    for client in clientlist:
        client.join()

    end = time.time()
    util.info("Elapsed time (in seconds): " + str(end-start))

    time.sleep(CGI_SPIN_TIME + 2)
    res = commands.getoutput("cat %s" % tmpfile)

    if util.is_server_alive(cmd) == -1:
        util.error("Ouch! Server is dead!"
                   " Your bounded buffered may not be well protected");

    pos0 = res.find(expected[0])
    pos1 = res.find(expected[1])
    pos2 = res.find(expected[2])
    passed = pos0 > 0 and pos1 > 0 and pos2 > 0 and pos0 < pos1 and pos1 < pos2
    
    util.info(res)

    if passed:
        print ""
        print "#####################################"
        print "GOOD! you implement SFF correctly"
        print "#####################################"
        print ""
        count = count + 1
    else:
        print ""
        print "#####################################"
        print "Oh oh! ERROR ERROR!"
        print "SFF is not implemented correctly"
        print "#####################################"
        print ""
        sys.exit(-1)

def build(tag, rebuild):
    for service in SERVICES:
        v = ['sudo', 'docker', 'build', '-t', full_tag(tag, service)]
        if rebuild:
            v.append("--no-cache")
        v.append('.')
        util.run(v, path=join(SCRIPT_PATH, 'images', service))

def build_libuv_windows(arch):
  args = ["cmd", "/c", "vcbuild.bat", "release", "vs2017"]
  if arch == "-32":
    args.append("x86")
  elif arch == "-64":
    args.append("x64")
  run(args, cwd=LIB_UV_DIR)

def build(tag, rebuild):
    # Next build smc-hub, which depends on smc-hub-base.
    v = ['sudo', 'docker', 'build', '-t', tag]
    if rebuild:  # will cause a git pull to happen
        v.append("--no-cache")
    v.append('.')
    util.run(v, path=join(SCRIPT_PATH, 'image'))

def mixup(model, root_dir, prev_dir, model_list, mix_size, estimateVarFloor=0):
    """
    Run HHEd to initialize a mixup to mix_size gaussians
    """

    output_dir = '%s/HMM-%d-%d' %(root_dir, mix_size, 0)
    util.create_new_dir(output_dir)

    ## Make the hed script
    mix_hed = '%s/mix_%d.hed' %(output_dir, mix_size)
    fh = open(mix_hed, 'w')

    if estimateVarFloor:
            fh.write('LS %s/stats\n' %prev_dir)
            fh.write('FA 0.1\n')
            
    fh.write('MU %d {(sil,sp).state[2-%d].mix}\n' %(2*mix_size,model.states-1))
    fh.write('MU %d {*.state[2-%d].mix}\n' %(mix_size, model.states-1))
    fh.close()

    hhed_log = '%s/hhed_mix.log' %output_dir

    cmd  = 'HHEd -A -D -T 1 -H %s/MMF -M %s' %(prev_dir, output_dir)
    cmd += ' %s %s > %s' %(mix_hed, model_list, hhed_log)
    if model.local == 1: os.system(cmd)
    else: util.run(cmd, output_dir)

    return output_dir

def test(cmd):
    print ""
    print "Starting " + cmd
    util.run(cmd)

    clientlist = []
    
    start = time.time()

    for i in range(0, NUM_CLIENT):
        client = testit("Client-" + str(i))
        client.setDaemon(True)
        clientlist.append(client)
        client.start()
    
    for client in clientlist:
        client.join()

    end = time.time()

    if util.is_server_alive(cmd) == -1:
        util.error("Ouch! Server is dead!"
                   " Your bounded buffered may not be well protected");

    print "Elapsed time (in seconds): " + str(end-start)
    if end - start > EXPECTED_TIME:
        util.error("your server is not multithreaded")

def gn_gen(mode):
    os.environ["DENO_BUILD_MODE"] = mode

    # Rather than using gn gen --args we write directly to the args.gn file.
    # This is to avoid quoting/escaping complications when passing overrides as
    # command-line arguments.
    args_filename = os.path.join(build_path(), "args.gn")

    # Check if args.gn exists, and if it was auto-generated or handcrafted.
    existing_gn_args, hand_edited = read_gn_args(args_filename)

    # If args.gn wasn't handcrafted, regenerate it.
    if hand_edited:
        print "%s: Using gn options from hand edited '%s'." % (mode,
                                                               args_filename)
        gn_args = existing_gn_args
    else:
        print "%s: Writing gn options to '%s'." % (mode, args_filename)
        gn_args = generate_gn_args(mode)
        if gn_args != existing_gn_args:
            write_gn_args(args_filename, gn_args)

    for line in gn_args:
        print "  " + line

    run([third_party.gn_path, "gen", build_path()],
        env=third_party.google_env())

def predict(experiment_dir, config_map, predict_meta_file):
  result_dir = getResultDir(experiment_dir)
  util.maybeMakeDir(result_dir)
  result_file = getResultPath(result_dir)

  data_dir = getDataDir(experiment_dir)
  data_file = getDataPath(data_dir)
  if config_map['use_classification']:
    label_file = getLabelPath(data_dir)
  else:
    label_file = getRlabelPath(data_dir)
  meta_file = getMetaPath(data_dir)
  model_dir = getModelDir(experiment_dir)
  imputer_dir = getImputerDir(experiment_dir)

  model_prefix = '%s-' % getModelName(config_map)
  model_suffix = '-%d' % config_map['train_window']
  imputer_prefix = 'imputer-'
  imputer_suffix = '-%d' % config_map['train_window']

  cmd = ('%s/predict_all.py --data_file=%s --label_file=%s '
         '--meta_file=%s --model_dir=%s --model_prefix="%s" '
         '--model_suffix="%s" --imputer_dir=%s --imputer_prefix="%s" '
         '--imputer_suffix="%s" --prediction_window=%d '
         '--delay_window=%d --predict_meta_file=%s --result_file=%s' % (
            CODE_DIR, data_file, label_file, meta_file,
            model_dir, model_prefix, model_suffix,
            imputer_dir, imputer_prefix, imputer_suffix,
            config_map['predict_window'],
            config_map['delay_window'], predict_meta_file,
            result_file))
  util.run(cmd)

def init_tri_from_mono(model, root_dir, mono_dir, tri_mlf, mono_list, tri_list):
    """
    Convert a monophone model and triphone mlf to triphones
    """

    ## Create the xword directory and the current output directory
    output_dir = '%s/HMM-0-0' %root_dir
    util.create_new_dir(root_dir)
    util.create_new_dir(output_dir)

    mktri_hed = '%s/mktri.hed' %output_dir
    hhed_log = '%s/hhed_clone_mono.log' %output_dir

    ## Create an HHEd script to clone monophones to triphones
    fh = open(mktri_hed, 'w')
    for line in open(mono_list):
        mono = line.strip()
        fh.write('TI T_%s {(%s).transP}\n' %(mono, mono))
    fh.write('CL %s\n' %tri_list)
    fh.close()

    ## Run HHEd to clone monophones and tie transition matricies
    cmd  = 'HHEd -A -T 1 -H %s/MMF' %mono_dir
    cmd += ' -M %s' %output_dir
    cmd += ' %s %s > %s' %(mktri_hed, mono_list, hhed_log)

    if model.local: os.system(cmd)
    else: util.run(cmd, output_dir)

    return output_dir

def mixdown_mono(model, root_dir, prev_dir, phone_list):
    """
    Run HHEd to mixdown monophones
    """

    output_dir = '%s/HMM-1-0' %root_dir
    util.create_new_dir(output_dir)

    ## Create the full list of possible triphones
    phones = open(phone_list).read().splitlines()
    non_sil_phones = [p for p in phones if p not in ['sp', 'sil']]

    ## Make the hed script
    mixdown_hed = '%s/mix_down.hed' %output_dir
    fh = open(mixdown_hed, 'w')
    fh.write('MD 12 {(sil,sp).state[2-%d].mix}\n' %(model.states-1))
    for phone in non_sil_phones:
        fh.write('MD 1 {%s.state[2-%d].mix}\n' %(phone, model.states-1))
    fh.close()

    hhed_log = '%s/hhed_mixdown.log' %output_dir

    cmd  = 'HHEd -A -D -T 1 -H %s/MMF -M %s' %(prev_dir, output_dir)
    cmd += ' %s %s > %s' %(mixdown_hed, phone_list, hhed_log)
    if model.local == 1: os.system(cmd)
    else: util.run(cmd, output_dir)

    return output_dir

def test(cmd):
    util.info("")
    util.info("- Starting " + cmd)
    util.info("")
    util.run(cmd)
    
    commands.getoutput("rm -rf " + file1)
    util.info("")
    util.info("- Sending ./testclient localhost 2010 /testdata/file1.txt")

    res = commands.getoutput("./testclient localhost 2010 /testdata/file1.txt")
    arrival = util.get_stat2(res, "Stat-req-arrival")
    dispatch = util.get_stat2(res, "Stat-req-dispatch")
    read = util.get_stat2(res, "Stat-req-read")
    complete = util.get_stat2(res, "Stat-req-complete")
    print ""
    print "dispatch = " + str(dispatch)
    print "read = " + str(read)
    print "complete = " + str(complete)


    if dispatch >= 0 and read >=0 and complete >= 0 and dispatch + read <= complete:
        util.good("You passed this test")
    else:
        util.error("Expected dispatch >= 0 and read >=0 and complete >= 0 and" 
                   " dispatch + read <= complete:")

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('--computer',
      default='/Users/lnyang/lab/qd2/qd2/compute_basic_feature.py')
  parser.add_argument('--processed_dir', required=True,
                      help='dir of processed sf1 data')
  parser.add_argument('--ticker_file', required=True)
  parser.add_argument('--feature_base_dir', required=True)
  parser.add_argument('--info_dir', required=True)
  parser.add_argument('--use_mrx', action='store_true')
  args = parser.parse_args()

  for indicator, dimensions in ITEMS:
    for dimension in dimensions:
      if args.use_mrx:
        dimension = util.getMrx(dimension)
      folder = '%s-%s' % (indicator, dimension)
      feature_dir = '%s/%s' % (args.feature_base_dir, folder)
      info_file = '%s/%s' % (args.info_dir, folder)
      if not shouldRun(feature_dir, info_file):
        continue
      cmd = ('%s --processed_dir=%s --ticker_file=%s --dimension=%s '
             '--header=%s --feature_dir=%s --info_file=%s' % (
             args.computer, args.processed_dir, args.ticker_file, dimension,
             indicator, feature_dir, info_file))
      util.run(cmd)

def maybeRun(args, input_dir, feature, windows):
  if not shouldRun(args.feature_base_dir, args.info_dir, feature, windows):
    return
  cmd = ('%s --input_dir=%s --output_dir=%s --feature=%s --windows=%s '
         '--ticker_file=%s --info_base_dir=%s' % (
         args.computer, input_dir, args.feature_base_dir, feature, windows,
         args.ticker_file, args.info_dir))
  util.run(cmd)

def createSetupLock():
    global locked
    try:
        run("mkdir %s" % env.SETUP_LOCK)
        locked = True
    except:
        logger.error("!!! First-time setup is already in progress. Exiting...")
        sys.exit(1)

def run_all():
    x = util.get_deployments()
    for name in ['rethinkdb-proxy', 'smc-webapp-static', 'smc-hub', 'haproxy']:
        if name not in x:
            if name == 'rethinkdb0':
                name = 'rethinkdb'
            print('\n******\nRUNNING {name}\n******\n'.format(name=name))
            util.run([join(SCRIPT_PATH,'..',name,'control.py'), 'run'])

def resize_cluster(args):
    prefix = util.get_cluster_prefix()
    if args.name:
        group = '{prefix}-{name}-minion-group'.format(prefix=prefix, name=args.name)
    else:
        group = '{prefix}-minion-group'.format(prefix=prefix)
    util.run(['gcloud', 'compute', 'instance-groups', 'managed', 'resize', group,
         '--size', str(args.size)])