def setup_experiment(args, config):
  # Grab parameters
  if args.exp_name:
    config.exp_name = args.exp_name
  # elif not hasattr(config, 'exp_name'):
  #   config.exp_name = exp_lifecycle.guess_config_name(config)

  if not hasattr(config, 'results_dir'):
    config.results_dir = "experiments/%s" % config.exp_name

  if args.timestamp_results is not None:
    # Note that argparse returns a list
    config.timestamp_results = args.timestamp_results

  if hasattr(config, 'timestamp_results') and config.timestamp_results:
    now = timestamp_string()
    config.results_dir += "_" + str(now)

  # Set up results directory
  mkdir_p("./experiments")
  create_clean_dir(config.results_dir)

  # Make sure that there are no uncommited changes
  if args.publish:
    exp_lifecycle.publish_prepare(config.exp_name, config.results_dir)

  # Record machine information for this experiment
  additional_metadata = None
  if hasattr(config, "get_additional_metadata"):
    additional_metadata = config.get_additional_metadata()

  exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir,
                              additional_metadata=additional_metadata)

    def install_base(self):
        """
        Install pathogen.vim and create vimpyre directory.

            >>> bat = Bat()
            >>> bat.install_base()
            => => Send a bat to catch pathogen.vim ...
            Catch done! Please add the following message to your .vimrc:
            call pathogen#runtime_append_all_bundles("vimpyre")
        """
        try:
            console('=> => Send a bat to catch pathogen.vim ...')
            raw_urlopen = urllib.urlopen(self.PATHOGEN_URL)
            if raw_urlopen.getcode() == 200:
                util.mkdir_p(self.AUTOLOAD_PATH)
                util.mkdir_p(self.VIMPYRE_PATH)
                raw_pathogen = raw_urlopen.read()
                pathogen = path.join(self.AUTOLOAD_PATH, 'pathogen.vim')
                with open(pathogen, 'w') as f:
                    f.write(raw_pathogen)
                console('Catch done! Please add the following to your .vimrc:')
                console('call pathogen#runtime_append_all_bundles("vimpyre")')
            else:
                console('Catch fail! Please try again!')
        except:
            console('[Unexpected Error] Catch fail! Please try again!')

    def save_gfx(self, path, data=None):
        if data is None:
            print('got data')
            data = self.get_data()

        for k, v in data.items():
            if type(v) is dict:
                self.save_gfx(path, v)
                pass
            elif k == 'icon':
                icon_path = data['icon'].split('/')

                if icon_path[0] not in self.modlist.path_map:
                    print('Unknown content path %s for %s/%s' % (icon_path[0], data['type'], data['name']))
                    continue

                icon_path[0] = self.modlist.path_map[icon_path[0]]
                icon_path = '/'.join(icon_path)

                out_dir = '%s/%s' % (path, data['type'])
                out_path = '%s/%s.png' % (out_dir, data['name'])
                mkdir_p(out_dir)

                if os.path.exists(out_path):
                    print('Overwriting %s/%s' % (data['type'], data['name']))

                shutil.copy2(icon_path, out_path)

	def mutate(self):
		"""
		Performs the mutation.  Applies mutation operator to each source file,
		then stores a diff between the original and mutated file.

		# mutants = # source files x # mutation operators
		"""
		count = 0
		start = time()

		for src_file in self.project.source_files():
			original_path = join(self.project.settings["source_path"], src_file)
			mutant_path = join(out_dir, src_file)
			mkdir_p(dirname(mutant_path))

			for (op, invoke) in self.project.settings["mutants"].items():
				if invoke:
					p = Popen(["txl", original_path, join("vendor", "conman", "%s.Txl" % op)], stdout=open(mutant_path, "w"), stderr=open("/dev/null"))
					self.store.put(diff(relpath(original_path), relpath(mutant_path)), op, src_file)
					count += 1

					if count % 1000 == 0:
						print("Generated %d mutants.  Elapsed time %.02f seconds." % (count, (time() - start)))

		stop = time()
		print("Generated %d mutants in %d seconds." % (count, (stop - start)))

 def __init__(self, settings, compiler):
     self._settings = settings
     self.compiler_name = settings.get('codeblocks').get('compiler_name', 'gcc')
     self.projectsdir = settings.get('projectsdir')
     util.mkdir_p(self.projectsdir)
     self.configurations = compiler.get_configurations()
     cvars = compiler.get_global_variables()
     self.cflags = cvars['cflags']
     self.lflags = cvars['lflags']

 def write_vct(self, outdir):
     outfile = os.path.join(outdir, self.file_id + ".vct")
     util.mkdir_p(os.path.dirname(outfile))
     if os.path.exists(outfile):
         os.remove(outfile)
     if os.path.exists(outfile + ".gz"):
         os.remove(outfile + ".gz")
     with gzip.open(outfile + ".gz", "wb") as f:
         f.write("\n".join(self.vct) + "\n")

def save_transform_to_database(data_transforms):
    """ Save data transforms to database"""
    # pylint: disable= too-many-locals, bare-except
    conn = sqlite3.connect(util.DB_PATH)
    conn.text_factory = str

    for img in data_transforms:
        cursor = conn.execute('''SELECT pid from Images where id = ? ''', (img.image_id,))
        pid = cursor.fetchone()[0]

        folder = util.DATA_FOLDER + str(pid) + "/registration_transforms/"
        util.mkdir_p(folder)

        transform_paths = ""
        print(img.get_transforms())
        for _transform in img.get_transforms():
            print(_transform)
            dst_file = folder + util.get_basename(_transform) + '.h5.gz'
            if os.path.exists(dst_file):
                os.remove(dst_file)
            with open(_transform, 'rb') as f_in, gzip.open(dst_file, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
            transform_paths += str(pid) + "/registration_transforms/" +\
                basename(_transform) + '.h5.gz' + ", "
        transform_paths = transform_paths[:-2]

        cursor2 = conn.execute('''UPDATE Images SET transform = ? WHERE id = ?''',
                               (transform_paths, img.image_id))
        cursor2 = conn.execute('''UPDATE Images SET fixed_image = ? WHERE id = ?''',
                               (img.fixed_image, img.image_id))

        folder = util.DATA_FOLDER + str(pid) + "/reg_volumes_labels/"
        util.mkdir_p(folder)
        vol_path = util.compress_vol(img.processed_filepath)
        shutil.copy(vol_path, folder)

        volume_db = str(pid) + "/reg_volumes_labels/" + basename(vol_path)
        cursor2 = conn.execute('''UPDATE Images SET filepath_reg = ? WHERE id = ?''',
                               (volume_db, img.image_id))

        cursor = conn.execute('''SELECT filepath, id from Labels where image_id = ? ''',
                              (img.image_id,))
        for (row, label_id) in cursor:
            temp = util.compress_vol(move_vol(util.DATA_FOLDER + row,
                                              img.get_transforms(), True))
            shutil.copy(temp, folder)
            label_db = str(pid) + "/reg_volumes_labels/" + basename(temp)
            cursor2 = conn.execute('''UPDATE Labels SET filepath_reg = ? WHERE id = ?''',
                                   (label_db, label_id))

        conn.commit()
        cursor.close()
        cursor2.close()

#    cursor = conn.execute('''VACUUM; ''')
    conn.close()

def write_tokens(start_dt, end_dt):
    # Write token file at end of main. gae_download.py runs under an hourly,
    # cronjob, so load_emr_daily.sh will start only when all 24 token files
    # are present.
    datestr = str(start_dt.date())
    dirname = "/home/analytics/kabackup/daily_new/"
    dirname += "%s/tokens/" % datestr
    mkdir_p(dirname)
    filename = "token-%s-%s.txt" % (start_dt.time(), end_dt.time())
    f = open(dirname + filename, "w")
    f.close()

def get_archive_file_name(config, kind, start_dt, end_dt):
    """get the archive file name. has the format of
       {ARCHIVE_DIR}/YY-mm-dd/{kind}/kind-start_dt-end_dt.pickle
    """
    datestr = str(start_dt.date())
    dirname = "%s/%s/%s" % (config['archive_dir'], datestr, kind)
    mkdir_p(dirname)
    filename = "%s/%s-%s-%s-%s-%s.pickle" % (dirname, kind,
        str(start_dt.date()), str(start_dt.time()),
        str(end_dt.date()), str(end_dt.time()))
    return filename

def init_hello_world():
    targets_filepath = 'source/hello_world/targets.json'
    helloworld_filepath = 'source/hello_world/hello_world.cpp'
    critical_error_if_file_exists(helloworld_filepath)
    critical_error_if_file_exists(targets_filepath)
    util.mkdir_p(os.path.dirname(targets_filepath))

    with open(targets_filepath, 'w+') as  fd:
        fd.write(util.get_resource('defaults/targets.json'))
    with open(helloworld_filepath, 'w+') as  fd:
        fd.write(util.get_resource('defaults/hello_world.cpp'))

def generate(ninja_targets, settings):
    project_name = settings.get('project_name')
    working_dir = '$rootdir'

    make_all = BuildSystem('make - ' + project_name, 'make build', working_dir)
    make_all.add_variant('All', 'make all')
    make_all.add_variant('Clean', 'make clean')
    make_all.add_variant('Sublime Text Project', 'make sublime')
    make_all.add_variant('Doxygen Documentation', 'make doxygen')
    ninja_all = BuildSystem('ninja - ' + project_name, 'ninja', working_dir)
    ninja_all.add_variant('All', 'ninja all')
    ninja_all.add_variant('Clean', 'ninja -t clean')
    ninja_all.add_variant('Doxygen Documentation', 'ninja doxygen')

    build_systems = [make_all.data, ninja_all.data]

    for target in ninja_targets.build_targets:
        build_target = BuildSystem(
            target.config + ' - ' + target.name,
            'ninja %s' % target.phony_name,
            working_dir)
        bindir = get_bin(target.config, settings)
        binary = Path.join(working_dir, bindir, target.name)
        build_target.add_variant(
            'Run',
            'ninja %s && %s' % (target.phony_name, binary))
        build_target.add_variant(
            'Clean',
            'ninja -t clean %s' % target.phony_name)
        build_systems.append(build_target.data)

    sublime_settings = settings.get('sublime')

    template_filename = settings.expand_variables(sublime_settings.get('project_template', None))
    if template_filename is not None:
        with open(template_filename, 'r') as fd:
            template = fd.read()
        if not template:
            critical_error('%s seems to be empty', template_filename)
    else:
        template = util.get_resource('defaults/tmpl.sublime-project')

    project = re.sub(
        r'(\$build_systems)',
        json.dumps(build_systems, indent=4, separators=(',', ': ')),
        template)
    project = settings.expand_variables(project)

    filename = settings.expand_variables('$projectsdir/$project_name.sublime-project')
    util.mkdir_p(os.path.dirname(filename))
    with open(filename, 'w+') as out:
        out.write(project)

def test_prerequesites():
    '''check we have the right directories and tools to run tests'''
    print("Testing prerequesites")
    util.mkdir_p(util.reltopdir('../buildlogs'))
    if not os.path.exists(util.reltopdir('../HILTest/hil_quad.py')):
        print('''
You need to install HILTest in %s

You can get it from git://git.samba.org/tridge/UAV/HILTest.git

        ''' % util.reltopdir('../HILTest'))
        return False
    return True

def create_inventory_id_and_tags(ec2_conf, ec2_info):
    # mock the inventory and ids that should be generated by vagrant
    inventory_dir = '.vagrant/provisioners/ansible/inventory'
    mkdir_p(inventory_dir)
    inventory = open(os.path.join(inventory_dir, 'vagrant_ansible_inventory'), 'w')

    machine_dir = '.vagrant/machines'
    mkdir_p(machine_dir)

    tag = ec2_conf['Tag']
    tags = {}
    for i, instance in enumerate(ec2_info['instances']):
        host = "TachyonMaster" if i == 0 else "TachyonWorker{id}".format(id=i)
        inventory.write("{host} ansible_ssh_host={ip} ansible_ssh_port=22\n".format(host=host, ip=instance['public_ip']))

        instance_id = str(instance['id'])
        id_dir = os.path.join(machine_dir, host, 'aws')
        mkdir_p(id_dir)
        with open(os.path.join(id_dir, 'id'), 'w') as f:
            f.write(instance_id)

        tags[instance_id] = '{tag}-{host}'.format(tag=tag, host=host)

    inventory.close()

    # map instance id to its tag
    spot_tag_dir = 'spot/roles/tag_ec2/vars'
    mkdir_p(spot_tag_dir)
    tag_vars = open(os.path.join(spot_tag_dir, 'main.yml'), 'w')
    var = {
        "tags": tags,
        "region": ec2_conf["Region"],
    }
    yaml.dump(var, tag_vars, default_flow_style=False)
    tag_vars.close()

def filter_cruise(host_assignments, output_dir, process_count=16):
    cruises = [x[0] for x in host_assignments[env.host_string]]
    cruise_results = {}
    workdir = "/mnt/raid"
    with cd(workdir):
        for c in cruises:
            puts("Filtering cruise {}".format(c))
            with hide("commands"):
                run("mkdir {}".format(c))
            with cd(c):
                text = {"cruise": c, "process_count": process_count}
                with settings(warn_only=True), hide("output"):
                    #result = run("seaflowpy_filter --s3 -c {cruise} -d {cruise}.db -l 10 -p 2 -o {cruise}_opp".format(**text))
                    result = run(
                        "seaflowpy_filter --s3 -c {cruise} -d {cruise}.db -t -p {process_count} -o {cruise}_opp".format(**text),
                        timeout=10800
                    )
                    cruise_results[c] = result

            puts(result)

            cruise_output_dir = os.path.join(output_dir, c)

            if result.succeeded:
                puts("Filtering successfully completed for cruise {}".format(c))
                puts("Returning results for cruise {}".format(c))
                util.mkdir_p(cruise_output_dir)
                rsyncout = execute(
                    # rsync files in cruise results dir to local cruise dir
                    rsync_get,
                    os.path.join(workdir, c) + "/",
                    cruise_output_dir,
                    hosts=[env.host_string]
                )
                # Print rsync output on source host, even though this is run
                # on local
                puts(rsyncout[env.host_string])
            else:
                sys.stderr.write("Filtering failed for cruise {}\n".format(c))

            # Always write log output
            util.mkdir_p(cruise_output_dir)
            logpath = os.path.join(cruise_output_dir, "seaflowpy_filter.{}.log".format(c))
            with open(logpath, "w") as logfh:
                logfh.write("command={}\n".format(result.command))
                logfh.write("real_command={}\n".format(result.real_command))
                logfh.write(result + "\n")

    return cruise_results

 def download_and_save(self):
     mkdir_p(self.chan, abort=True, msg="Couldn't create directory for chan")
     mkdir_p("%s/%s" % (self.chan, self.boardname), abort=True, msg="Couldn't create directory for board")
     catalog = self.get_catalog(False)
     i = 0
     for thread in catalog:
         i += 1
         if not self.suppressed_output:
             print "Downloading thread %d (%d/%d)" % (thread, i, len(catalog))
         filename = "%s/%s/%d.json" % (self.chan, self.boardname, thread)
         connection = httplib.HTTPSConnection(self.host)
         with open(filename, "w") as file:
             connection.request("GET", self.get_thread_json_url(thread))
             file.write(connection.getresponse().read())
         connection.close()

def mock_vagrant_info(instance_id_to_tag_ip):
    inventory_dir = '.vagrant/provisioners/ansible/inventory'
    mkdir_p(inventory_dir)
    inventory = open(os.path.join(inventory_dir, 'vagrant_ansible_inventory'), 'w')
    for instance_id, tag_ip in instance_id_to_tag_ip.iteritems():
        tag, ip = tag_ip
        host = get_host(tag)

        inventory.write("{} ansible_ssh_host={} ansible_ssh_port=22\n".format(host, ip))

        id_dir = os.path.join('.vagrant', 'machines', host, 'aws')
        mkdir_p(id_dir)
        with open(os.path.join(id_dir, 'id'), 'w') as f:
            f.write(instance_id)
    inventory.close()

def get_archive_file_name(config, kind, start_dt, end_dt, ftype='pickle'):
    """Get the archive file name. has the format of
    {ARCHIVE_DIR}/YY-mm-dd/{kind}/kind-start_dt-end_dt.pickle

    """

    # Note that Hadoop does not like leading underscores in files, so we strip
    # out leading underscores (as may be used in the case of private classes)
    kind = re.sub(r'^_*', '', kind)

    datestr = str(start_dt.date())
    dirname = "%s/%s/%s" % (config['archive_dir'], datestr, kind)
    mkdir_p(dirname)
    filename = "%s/%s-%s-%s-%s-%s.%s" % (dirname, kind,
        str(start_dt.date()), str(start_dt.time()),
        str(end_dt.date()), str(end_dt.time()), ftype)
    return filename

def link_scripts(dest_dir, src_dir=None):
    original_scripts_dir = src_dir or util.get_installed_scripts_dir()

    scripts = [
        (s['name'], get_new_script_path(s['module'], s['function']))
        for s in get_console_scripts_info()
    ]

    for (original, new_path) in scripts:
        original_path = os.path.join(original_scripts_dir, original)
        new_path = os.path.join(dest_dir, new_path)

        util.mkdir_p(os.path.dirname(new_path))

        print 'Symlinking %s to %s ...' % (original_path, new_path)
        if os.path.lexists(new_path):
            os.remove(new_path)
        os.symlink(original_path, new_path)

def convert_and_save_dataset(pid, cursor, image_type, volume_labels, volume, glioma_grade):
    """convert_and_save_dataset"""
    util.mkdir_p(util.DATA_FOLDER + str(pid))
    img_out_folder = util.DATA_FOLDER + str(pid) + "/volumes_labels/"
    util.mkdir_p(img_out_folder)

    cursor.execute('''SELECT pid from Patient where pid = ?''', (pid,))
    exist = cursor.fetchone()
    if exist is None:
        cursor.execute('''INSERT INTO Patient(pid, glioma_grade) VALUES(?, ?)''', (pid, glioma_grade))

    cursor.execute('''INSERT INTO Images(pid, modality, diag_pre_post) VALUES(?,?,?)''',
                   (pid, 'MR', image_type))
    img_id = cursor.lastrowid

    _, file_extension = os.path.splitext(volume)
    volume_temp = "volume" + file_extension
    shutil.copy(volume, volume_temp)

    volume_out = img_out_folder + str(pid) + "_" + str(img_id) + "_MR_T1_" + image_type + ".nii.gz"
    print("--->", volume_out)
    os.system(DWICONVERT_PATH + " --inputVolume " + volume_temp + " -o " + volume_out + " --conversionMode NrrdToFSL")
    volume_out_db = volume_out.replace(util.DATA_FOLDER, "")
    cursor.execute('''UPDATE Images SET filepath = ? WHERE id = ?''', (volume_out_db, img_id))
    os.remove(volume_temp)

    for volume_label in volume_labels:
        _, file_extension = os.path.splitext(volume_label)
        volume_label_temp = "volume_label" + file_extension
        shutil.copy(volume_label, volume_label_temp)

        cursor.execute('''INSERT INTO Labels(image_id, description) VALUES(?,?)''',
                       (img_id, 'all'))
        label_id = cursor.lastrowid

        volume_label_out = img_out_folder + str(pid) + "_" + str(img_id) + "_MR_T1_" + image_type\
            + "_label_all.nii.gz"
        os.system(DWICONVERT_PATH + " --inputVolume " + volume_label_temp + " -o " +
                  volume_label_out + " --conversionMode NrrdToFSL")
        volume_label_out_db = volume_label_out.replace(util.DATA_FOLDER, "")
        cursor.execute('''UPDATE Labels SET filepath = ? WHERE id = ?''',
                       (volume_label_out_db, label_id))
        os.remove(volume_label_temp)

def load_pack(pack_config: PackConfig):
    if pack_config.mods_path is None:
        pack_config.mods_path = os.path.join(pack_config.factorio_path, 'mods')

    locale = FactorioLocale()
    pack_dir = get_pack_dir(pack_config.name)
    oldcwd = os.getcwd()

    try:
        fs = FactorioState(pack_config.factorio_path, locale)

        for fn in os.listdir(pack_config.mods_path):
            fn = os.path.join(pack_config.mods_path, fn)
            if os.path.isdir(fn):
                fs.modlist.add_mod(fn)

        fs.modlist.resolve()
        fs.load_mods()

        locale.merge()
        data = fs.get_data()
    finally:
        os.chdir(oldcwd)

    mkdir_p(pack_dir)

    sort_dict(data, 'technology')

    with open('%s/out' % pack_dir, 'w') as f:
        f.write(json.dumps(data, indent=4))

    fs.save_gfx('%s/icon' % pack_dir)
    locale.save('%s/localedump.cfg' % pack_dir)

    pack_info = PackInfo(pack_config.name, pack_config.title, '', '', fs.modlist._loaded_names)

    with open('%s/info.json' % pack_dir, 'w') as f:
        f.write(pack_info.to_json())