def auto_conf(self):
     util.mkdir(self.out_root)
     self.out_dir = os.path.join(self.out_root, self.name)
     util.mkdir(self.out_dir)
     self.model_path = os.path.join(self.out_dir, 'model')
     self.log_path = os.path.join(self.out_dir, 'log')
     self.test_out_path = os.path.join(self.out_dir, 'test_out')

    def getTeamSeasonHtml(self):
        html = INVALID_STRING
        if cmp(self._htmlWebSite, INVALID_STRING) == 0:
            print "htmlWebSite has not been initialized yet"
        else:
            #If we have already synced the data we need to fetch the html from local instead of reloading the website again
            pathNeed2Test = CURRENT_PATH + self._Season0 + self._Season1 + '/' + self._Team_id
            print "Constructing path as ", pathNeed2Test
            self._dataStoredPath = pathNeed2Test
            util.mkdir(pathNeed2Test)

            htmlFile = pathNeed2Test + '/' + self._GameType + '.html'
            self._seasonDataHtmlFile = htmlFile
            print "Check if html file exist or not ", htmlFile

            if os.path.isfile(htmlFile):
                print "html file exists, open the file, read it and return the string"
                html = util.openFile(htmlFile)

                #print html
            else:
                print "html file does not exist, now loading the webpage from network"
                html = util.getHtmlFromUrl(self._htmlWebSite)

                if cmp(html, INVALID_STRING)!=0:
                    util.saveFile(htmlFile,html)

                return html


        return html

 def __write_permapage(self, posts):
     """Write blog posts to their permalink locations"""
     perma_template = self.template_lookup.get_template("permapage.mako")
     perma_template.output_encoding = "utf-8"
     for post in posts:
         if post.permalink:
             path_parts = [self.output_dir]
             path_parts.extend(urlparse.urlparse(
                     post.permalink)[2].lstrip("/").split("/"))
             path = os.path.join(*path_parts)
             logger.info("Writing permapage for post: "+path)
         else:
             #Permalinks MUST be specified. No permalink, no page.
             logger.info("Post has no permalink: "+post.title)
             continue
         try:
             util.mkdir(path)
         except OSError:
             pass
         html = self.__template_render(
             perma_template,
             { "post": post,
               "posts": posts })
         f = open(os.path.join(path,"index.html"), "w")
         f.write(html)
         f.close()

    def check(self, name):
        data = self.getSource()
        fnRef = self.fnRef(name)
        fnOrig = self.fnOrig(name)

        # save references only?
        if self.save:
            util.mkdir(self.config.REF_DIR)
            util.write(fnRef, data)

        else: # compare
            assert os.path.exists(fnRef), "Cannot compare without reference file: %s" % fnRef

            util.mkdir(self.outputDir)
            # first save original file
            util.write(fnOrig, data)

            ref = self.cleanup(util.read(fnRef))
            data = self.cleanup(data)

            # htmldiff
            result = htmldiff.htmldiff(ref, data, True)
            util.write(self.fnHtmlDiff(name), result)
            self.scenario.results[name] = self._eval_diff(result)

            # difflib
            linesRef = ref.splitlines()
            linesData = data.splitlines()
            result = difflib.HtmlDiff(wrapcolumn=80).make_file(linesRef, linesData, fnRef, fnOrig, context=True)
            util.write(self.fnDiffLib(name), result)

def script_vizForHMDB():

	out_dir='/disk2/mayExperiments/debug_finetuning/hmdb';
	clusters_file='/home/maheenrashid/Downloads/debugging_jacob/optical_flow_prediction_test/examples/opticalflow/clusters.mat';
	vid_list=os.path.join(out_dir,'video_list.txt');
	out_dir_viz=os.path.join(out_dir,'im');
	util.mkdir(out_dir_viz);
	out_file_html=out_dir_viz+'.html';
	
	path_to_hmdb='/disk2/marchExperiments/hmdb_try_2/hmdb'

	dirs=util.readLinesFromFile(vid_list);
	dirs=[os.path.join(path_to_hmdb,dir_curr) for dir_curr in dirs[2:]];
	random.shuffle(dirs);
	num_to_evaluate=100;
	out_file_tif=os.path.join(out_dir,'tif_list.txt');

	# recordContainingFiles(dirs,num_to_evaluate,out_file_flo,post_dir='images',ext='.flo');
	tif_files=util.readLinesFromFile(out_file_tif);
	tif_files=tif_files[:100];
	img_files=[file_curr.replace('.tif','.jpg') for file_curr in tif_files];
	flo_files=[file_curr.replace('.tif','.flo') for file_curr in tif_files];
	clusters=po.readClustersFile(clusters_file);

	script_writeFloVizHTML(out_file_html,out_dir_viz,flo_files,img_files,tif_files,clusters,True)

def script_reSaveMatOverlap():

    dir_old='/disk3/maheen_data/pedro_val/mat_overlap';
    dir_new='/disk3/maheen_data/pedro_val/mat_overlap_check';

    util.mkdir(dir_new);
    
    path_to_im='/disk2/ms_coco/val2014';
    path_to_gt='/disk2/mayExperiments/validation_anno';

    mat_overlap_files=util.getFilesInFolder(dir_old,ext='.npz');
    im_names=util.getFileNames(mat_overlap_files,ext=False);

    args=[];
    for idx,(mat_overlap_file,im_name) in enumerate(zip(mat_overlap_files,im_names)):
        gt_file=os.path.join(path_to_gt,im_name+'.npy');
        im_file=os.path.join(path_to_im,im_name+'.jpg');
        out_file=os.path.join(dir_new,im_name+'.npz');
        # if os.path.exists(out_file):
        #     continue;

        args.append((mat_overlap_file,gt_file,im_file,out_file,idx));

    p = multiprocessing.Pool(32);
    p.map(fixMatOverlap, args);    

    def generate(self):
        pkg_entries = [(pkg, self.db_write(pkg)) for pkg in self.pkgs]

        if self.dbdir:
            for pkg, entry in pkg_entries:
                path = os.path.join(self.dbdir, pkg.fullname())
                util.mkdir(path)
                for name, data in entry.iteritems():
                    filename = os.path.join(path, name)
                    util.mkfile(filename, data)

        if self.dbfile:
            tar = tarfile.open(self.dbfile, "w:gz")
            for pkg, entry in pkg_entries:
                # TODO: the addition of the directory is currently a
                # requirement for successful reading of a DB by libalpm
                info = tarfile.TarInfo(pkg.fullname())
                info.type = tarfile.DIRTYPE
                tar.addfile(info)
                for name, data in entry.iteritems():
                    filename = os.path.join(pkg.fullname(), name)
                    info = tarfile.TarInfo(filename)
                    info.size = len(data)
                    tar.addfile(info, StringIO(data))
            tar.close()
            # TODO: this is a bit unnecessary considering only one test uses it
            serverpath = os.path.join(self.root, util.SYNCREPO, self.treename)
            util.mkdir(serverpath)
            shutil.copy(self.dbfile, serverpath)

    def pack(self):
        emb_root = self.target_root
        if self.seed:
            emb_root = emb_root.pjoin(self.target_root)

        basedir = util.Path( os.path.dirname(self.tarpath) )
        util.mkdir(basedir)

        archive = tarfile.open(self.tarpath, 'w:bz2')
        archive.add(emb_root,
            arcname = '/',
            recursive = True
        )
        archive.close()

        curdir = os.path.realpath(os.curdir)
        os.chdir(emb_root)
        util.cmd('find ./ | cpio -H newc -o | gzip -c -9 > %s' % (self.cpiopath))
        os.chdir(curdir)

        if self.kernel:
            r = util.Path('/')
            if self.seed:
                r = self.target_root
            r = r.pjoin('/tmp/inhibitor/kernelbuild')

            kernel_link = r.pjoin('/boot/kernel')
            kernel_path = os.path.realpath( kernel_link )

            if os.path.lexists(self.kernlinkpath):
                os.unlink(self.kernlinkpath)

            shutil.copy2(kernel_path, basedir.pjoin( os.path.basename(kernel_path) ))
            os.symlink(os.path.basename(kernel_path), self.kernlinkpath)

def get_conferences():
    files = util.listdir(CONFERENCE_FOLDER)
    util.mkdir(CONFERENCE_CRALWED_FOLDER)
    cnt = 0
    conf = util.load_json('conf_name.json')
    for file_name in files:
        save_path = os.path.join(CONFERENCE_CRALWED_FOLDER, file_name)
        if util.exists(save_path):
            continue
        data = util.load_json(os.path.join(CONFERENCE_FOLDER, file_name))
        if data['short'] not in conf.keys():
            continue
        html = util.get_page(data['url'])
        subs = get_subs(data['short'], html)
        data['name'] = conf[data['short']]
        data['sub'] = {}
        for sub in subs:
            if sub not in conf.keys():
                continue
            html = util.get_page('http://dblp.uni-trier.de/db/conf/' + sub)
            data['sub'][sub] = {}
            data['sub'][sub]['pub'] = get_publications(html)
            data['sub'][sub]['name'] = conf[sub]
        cnt += 1
        print cnt, len(files), data['short']
        util.save_json(save_path, data)

def moveFilesIntoFolders(in_dir,mat_file,out_dir,out_file_commands,pad_zeros_in=8,pad_zeros_out=4):
	arr=scipy.io.loadmat(mat_file)['ranges'];
	# videos=np.unique(arr);
	commands=[];
	for shot_no in range(arr.shape[1]):
		print shot_no,arr.shape[1];
		start_idx=arr[0,shot_no];
		end_idx=arr[1,shot_no];
		video_idx=arr[2,shot_no];
		out_dir_video=os.path.join(out_dir,str(video_idx));
		util.mkdir(out_dir_video);
		# print 
		# raw_input();
		shot_idx=np.where(shot_no==np.where(video_idx==arr[2,:])[0])[0][0]+1;
		out_dir_shot=os.path.join(out_dir_video,str(shot_idx));
		util.mkdir(out_dir_shot);

		# print start_idx,end_idx
		for idx,frame_no in enumerate(range(start_idx,end_idx+1)):
			in_file=os.path.join(in_dir,padZeros(frame_no,pad_zeros_in)+'.jpg');
			out_file=os.path.join(out_dir_shot,'frame'+padZeros(idx+1,pad_zeros_out)+'.jpg');
			command='mv '+in_file+' '+out_file;
			commands.append(command);
	print len(commands);
	util.writeFile(out_file_commands,commands);

def codeProject(args,flag,data):
  PARAM_KEY = 1
  PARAM_PATH = 2
  PARAM_FORMATTER = 3
  ARGUMENTS = len(args)-1

  # JSON mapping files and storage of this
  if( keyExists("projects",args[1])):
    if( "stdout" in args[2]):
      project = json.loads(load("projects/"+args[PARAM_KEY])); # Uses key value storage
      directory = args[PARAM_PATH] + "/" + args[PARAM_KEY]
      
      mkdir(directory)
      for x in project.keys(): # Reflect that with here
        _file = json.loads(load("files/"+x));
        out = '';
        for y in _file:
          block = str(load("blocks/"+ y))
          if(ARGUMENTS == PARAM_FORMATTER): # Alter all the blocks in said fashion
            block = format.block(block, args[PARAM_FORMATTER])     
          out += block
        # Output the file with the correct file name
        save(directory + "/" + project[x],out)

  else:
    error("Error: Project does not exist")

def script_makeNegImages():

	in_dir='/disk2/aprilExperiments/testing_neg_fixed_test/'
	out_dir=os.path.join(in_dir,'visualizing_negs');
	util.mkdir(out_dir);


	# return
	im_paths=[os.path.join(in_dir,file_curr) for file_curr in os.listdir(in_dir) if file_curr.endswith('.png')];
	for im_path in im_paths:
		print im_path
		bbox=np.load(im_path.replace('.png','_bbox.npy'));
		crop_box=np.load(im_path.replace('.png','_crop.npy'));

		# print im_path.replace('.png','_bbox.npy')
		# print im_path.replace('.png','_crop.npy')
		# break;

		im = Image.open(im_path);
		im=drawCropAndBBox(im,bbox,crop_box)
		
		# write to stdout
		im.save(os.path.join(out_dir,im_path[im_path.rindex('/')+1:]), "PNG");
		# break;
	visualize.writeHTMLForFolder(out_dir,ext='png',height=300,width=300);

def rescaleImAndSaveMeta(img_paths,meta_dir,power_scale_range=(-2,1),step_size=0.5):
	img_names=util.getFileNames(img_paths);
	power_range=np.arange(power_scale_range[0],power_scale_range[1]+1,step_size);
	scales=[2**val for val in power_range];

	scale_infos=[];
	for idx,scale in enumerate(scales):
		out_dir_curr=os.path.join(meta_dir,str(idx));
		util.mkdir(out_dir_curr);
		scale_infos.append((out_dir_curr,scale));

	args=[];
	idx=0;
	for idx_img,img_path in enumerate(img_paths):
		for out_dir_curr,scale in scale_infos:
			out_file=os.path.join(out_dir_curr,img_names[idx_img]);
			
			if os.path.exists(out_file):
				continue;

			args.append((img_path,out_file,scale,idx));
			idx=idx+1;

	p=multiprocessing.Pool(multiprocessing.cpu_count());
	p.map(rescaleImAndSave,args);

def script_saveToExploreBoxes():
    # dir_mat_overlap='/disk3/maheen_data/headC_160_noFlow_bbox/mat_overlaps_1000';
    dir_mat_overlap='/disk3/maheen_data/pedro_val/mat_overlap';
    scratch_dir='/disk3/maheen_data/scratch_dir'
    util.mkdir(scratch_dir);
    out_file=os.path.join(scratch_dir,'them_neg_box.p');
    
    
    overlap_files=util.getFilesInFolder(dir_mat_overlap,ext='.npz');
    print len(overlap_files)
    to_explore=[];
    
    for idx_overlap_file,overlap_file in enumerate(overlap_files):
        print idx_overlap_file
        meta_data=np.load(overlap_file);
        pred_boxes=meta_data['pred_boxes'];
        # print pred_boxes.shape
        min_boxes=np.min(pred_boxes,axis=1);
        num_neg=np.sum(min_boxes<0);
        if num_neg>0:
            to_explore.append((overlap_file,pred_boxes));

    print len(to_explore);

    pickle.dump(to_explore,open(out_file,'wb'));

def initialize():
  print '\nWelcome to the interactive Svndae configuration!'
  options = get_configuration_defaults()
  print "\nInitializing file system in '%s'" % options['path']
  util.mkdir(options['path'])
  util.mkdir(join(options['path'],options['dir']))
  print "Generating configuration file '%s'" % options['conf']
  SvndaeConfig.generate_config(options['path'],options['conf'],options['dir'],options['file'])
  config = SvndaeConfig(options['path'],name=options['conf'])
  print "Adding default administrative groups"
  config.create_group(init.ADMIN_GROUP)
  config.add_permission(init.ADMIN_GROUP,SvndaeConfig._WRITE,SvndaeConfig._ALL_REPOS)
  config.add_permission(init.ADMIN_GROUP,SvndaeConfig._READ,SvndaeConfig._ALL_REPOS)
  while True:
    correct_in = raw_input('Would you like to add any repository ACLs? ([y]/n) ')
    if correct_in == '' or correct_in == 'y':
      get_repo_manage(config)
      break
    elif correct_in == 'n':
      break
    else:
      print_confirmation_error()
  while True:
    correct_in = raw_input('Would you like to add any additional groups? ([y]/n) ')
    if correct_in == '' or correct_in == 'y':
      get_user_groups(config)
      break
    elif correct_in == 'n':
      break
    else:
      print_confirmation_error()
  print_final()

def script_saveHumanOnlyNeg():
    out_file='/disk2/aprilExperiments/positives_160_human.txt'
    out_dir='/disk2/aprilExperiments/negatives_npy_onlyHuman';
    util.mkdir(out_dir);
    im_pre='COCO_train2014_'

    lines=util.readLinesFromFile(out_file);
    img_files=[line[:line.index(' ')] for line in lines];

    img_names=util.getFileNames(img_files,ext=False);
    img_name=img_names[0];
    
    print img_name
    
    img_name_split=img_name.split('_');
    idx_all=[int(img_name.split('_')[-1]) for img_name in img_names];

    print len(img_names),len(idx_all),idx_all[0];
    cat_id=1;

    path_to_anno='/disk2/ms_coco/annotations';
    anno_file='instances_train2014.json';
    anno=json.load(open(os.path.join(path_to_anno,anno_file),'rb'))['annotations'];
    
    script_saveBboxFiles(anno,out_dir,im_pre,idx_all,cat_id)

    def makepkg(self, path):
        """Creates an Arch Linux package archive.
        
        A package archive is generated in the location 'path', based on the data
        from the object.
        """
        self.path = os.path.join(path, self.filename())

        curdir = os.getcwd()
        tmpdir = tempfile.mkdtemp()
        os.chdir(tmpdir)

        # Generate package file system
        for f in self.files:
            util.mkfile(f, f)
            self.size += os.lstat(self.parse_filename(f))[stat.ST_SIZE]

        # .PKGINFO
        data = ["pkgname = %s" % self.name]
        data.append("pkgver = %s" % self.version)
        data.append("pkgdesc = %s" % self.desc)
        data.append("url = %s" % self.url)
        data.append("builddate = %s" % self.builddate)
        data.append("packager = %s" % self.packager)
        data.append("size = %s" % self.size)
        if self.arch:
            data.append("arch = %s" % self.arch)
        for i in self.license:
            data.append("license = %s" % i)
        for i in self.replaces:
            data.append("replaces = %s" % i)
        for i in self.groups:
            data.append("group = %s" % i)
        for i in self.depends:
            data.append("depend = %s" % i)
        for i in self.optdepends:
            data.append("optdepend = %s" % i)
        for i in self.conflicts:
            data.append("conflict = %s" % i)
        for i in self.provides:
            data.append("provides = %s" % i)
        for i in self.backup:
            data.append("backup = %s" % i)
        util.mkfile(".PKGINFO", "\n".join(data))

        # .INSTALL
        if any(self.install.values()):
            util.mkfile(".INSTALL", self.installfile())

        # safely create the dir
        util.mkdir(os.path.dirname(self.path))

        # Generate package archive
        tar = tarfile.open(self.path, "w:gz")
        for i in os.listdir("."):
            tar.add(i)
        tar.close()

        os.chdir(curdir)
        shutil.rmtree(tmpdir)

    def run(self):
        environ = self.distribution.environment

        if self.devel_support:
            for tpl in self.devel_support:
                if self.distribution.verbose:
                    print 'adding sysdevel support to ' + tpl[0]
                target = os.path.abspath(os.path.join(self.build_lib,
                                                      *tpl[0].split('.')))
                util.mkdir(target)
                source_dir = os.path.abspath(os.path.join(
                        os.path.dirname(__file__), 'support'))
                for mod in tpl[1]:
                    src_file = os.path.join(source_dir, mod + '.py.in')
                    if not os.path.exists(src_file):
                        src_file = src_file[:-3]
                    dst_file = os.path.join(target, mod + '.py')
                    util.configure_file(environ, src_file, dst_file)


        if self.antlr_modules:
            here = os.getcwd()
            for grammar in self.antlr_modules:
                if self.distribution.verbose:
                    print 'building antlr grammar "' + \
                        grammar.name + '" sources'
                ##TODO build in build_src, add to build_lib modules
                target = os.path.abspath(os.path.join(self.build_lib,
                                                      grammar.directory))
                util.mkdir(target)
                source_dir = os.path.abspath(grammar.directory)
                os.chdir(target)

                reprocess = True
                ref = os.path.join(target, grammar.name + '2Py.py')
                if os.path.exists(ref):
                    reprocess = False
                    for src in grammar.sources:
                        src_path = os.path.join(source_dir, src)
                        if os.path.getmtime(ref) < os.path.getmtime(src_path):
                            reprocess = True
                if reprocess:
                    for src in grammar.sources:
                        ## ANTLR cannot parse from a separate directory
                        shutil.copy(os.path.join(source_dir, src), '.')
                        cmd_line = list(environ['ANTLR'])
                        cmd_line.append(src)
                        status = subprocess.call(cmd_line)
                        if status != 0:
                            raise Exception("Command '" + str(cmd_line) +
                                            "' returned non-zero exit status "
                                            + str(status))
                    ## Cleanup so that it's only Python modules
                    for f in glob.glob('*.g'):
                        os.unlink(f)
                    for f in glob.glob('*.tokens'):
                        os.unlink(f)
                os.chdir(here)
        _build_src.run(self)

    def __write_files(self):
        """Write all files for the blog to _site

        Convert all templates to straight HTML
        Copy other non-template files directly"""
        #find mako templates in template_dir
        for root, dirs, files in os.walk("."):
            if root.startswith("./"):
                root = root[2:]
            for d in list(dirs):
                #Exclude some dirs
                d_path = util.path_join(root,d)
                if util.should_ignore_path(d_path):
                    logger.debug("Ignoring directory: " + d_path)
                    dirs.remove(d)
            try:
                util.mkdir(util.path_join(self.output_dir, root))
            except OSError: #pragma: no cover
                pass
            for t_fn in files:
                t_fn_path = util.path_join(root, t_fn)
                if util.should_ignore_path(t_fn_path):
                    #Ignore this file.
                    logger.debug("Ignoring file: " + t_fn_path)
                    continue
                elif t_fn.endswith(".mako"):
                    logger.info("Processing mako file: " + t_fn_path)
                    #Process this template file
                    t_name = t_fn[:-5]
                    t_file = open(t_fn_path)
                    template = Template(t_file.read().decode("utf-8"),
                                        output_encoding="utf-8",
                                        lookup=self.template_lookup)
                    t_file.close()
                    path = util.path_join(self.output_dir, root, t_name)
                    html_file = open(path, "w")
                    # Prepare the "path" variable for the template context.
                    page_path = util.path_join(root, t_name)
                    if page_path.startswith('./'):
                      page_path = page_path[2:]
                    page_path = '/' + page_path
                    context = dict(path=page_path, logger=template_logger)
                    #render the page
                    html = self.template_render(template, context)
                    #Write to disk
                    html_file.write(html)
                else:
                    #Copy this non-template file
                    f_path = util.path_join(root, t_fn)
                    logger.debug("Copying file: " + f_path)
                    out_path = util.path_join(self.output_dir, f_path)
                    if self.bf.config.site.use_hard_links:
                        # Try hardlinking first, and if that fails copy
                        try:
                            os.link(f_path, out_path)
                        except StandardError:
                            shutil.copyfile(f_path, out_path)
                    else:
                        shutil.copyfile(f_path, out_path)

 def make_profile_link(self):
     # XXX:  We also need to make the root profile link, Gentoo Bug 324179.
     for d in (self.target_root, self.target_root.pjoin(self.portage_cr)):
         targ = d.pjoin("/etc/make.profile")
         util.mkdir(os.path.dirname(targ))
         if os.path.lexists(targ):
             os.unlink(targ)
         os.symlink(self.env["PORTDIR"] + "/profiles/%s" % self.profile, targ)