void doClusterMotifDetails(struct sqlConnection *conn, struct trackDb *tdb, 
                                struct factorSource *cluster)
/* Display details about TF binding motif(s) in cluster */
{
char *motifTable = trackDbSetting(tdb, "motifTable");         // localizations
char *motifPwmTable = trackDbSetting(tdb, "motifPwmTable");   // PWM used to draw sequence logo
char *motifMapTable = trackDbSetting(tdb, "motifMapTable");   // map target to motif
struct slName *motifNames = NULL, *mn; // list of canonical motifs for the factor
struct dnaMotif *motif = NULL;
struct bed6FloatScore *hit = NULL, *maxHit = NULL;
char **row;
char query[256];

if (motifTable != NULL && sqlTableExists(conn, motifTable))
    {
    struct sqlResult *sr;
    int rowOffset;
    char where[256];

    if (motifMapTable == NULL || !sqlTableExists(conn, motifMapTable))
        {
        // Assume cluster name is motif name if there is no map table
        motifNames = slNameNew(cluster->name);
        }
    else
        {
        sqlSafef(query, sizeof(query),
                "select motif from %s where target = '%s'", motifMapTable, cluster->name);
        char *ret = sqlQuickString(conn, query);
        if (ret == NULL)
            {
            // missing target from table -- no canonical motif
            webNewEmptySection();
            return;
            }
        motifNames = slNameListFromString(ret, ',');
        }
    for (mn = motifNames; mn != NULL; mn = mn->next)
        {
        sqlSafefFrag(where, sizeof(where), "name='%s' order by score desc limit 1", mn->name);
        sr = hRangeQuery(conn, motifTable, cluster->chrom, cluster->chromStart,
                     cluster->chromEnd, where, &rowOffset);
        if ((row = sqlNextRow(sr)) != NULL)
            {
            hit = bed6FloatScoreLoad(row + rowOffset);
            if (maxHit == NULL || maxHit->score < hit->score)
                maxHit = hit;
            }
        sqlFreeResult(&sr);
        }
    }
if (maxHit == NULL)
    {
    // Maintain table layout
    webNewEmptySection();
    return;
    }
hit = maxHit;

webNewSection("Canonical Motif in Cluster");
char posLink[1024];
safef(posLink, sizeof(posLink),"<a href=\"%s&db=%s&position=%s%%3A%d-%d\">%s:%d-%d</a>",
        hgTracksPathAndSettings(), database, 
            cluster->chrom, hit->chromStart+1, hit->chromEnd,
            cluster->chrom, hit->chromStart+1, hit->chromEnd);
printf("<b>Motif Name:</b>  %s<br>\n", hit->name);
printf("<b>Motif Score");
printf(":</b>  %.2f<br>\n", hit->score);
printf("<b>Motif Position:</b> %s<br>\n", posLink);
printf("<b>Motif Strand:</b> %c<br>\n", (int)hit->strand[0]);

struct dnaSeq *seq = hDnaFromSeq(database, seqName, hit->chromStart, hit->chromEnd, dnaLower);
if (seq == NULL)
    return;
if (hit->strand[0] == '-')
    reverseComplement(seq->dna, seq->size);
if (motifPwmTable != NULL && sqlTableExists(conn, motifPwmTable))
    {
    motif = loadDnaMotif(hit->name, motifPwmTable);
    if (motif == NULL)
        return;
    motifLogoAndMatrix(&seq, 1, motif);
    }
}

matrix * NMFgetSampleAccuracy(struct hash *config)
/*Read all the folds and calculate training and testing accuracies from best models*/
{
char * trainingDir = hashMustFindVal(config, "trainingDir");
char * validationDir = hashMustFindVal(config, "validationDir");
char * modelDir = hashMustFindVal(config, "modelDir");
int fold, folds = foldsCountFromDataDir(config);
int split, splits  = splitsCountFromDataDir(config);

matrix * accuracies = NULL;
char filename[256];
FILE * fp;
for(split = 1; split <= splits; split++)
    {
    for(fold = 1; fold <= folds; fold++)
        {
        //cat togetehr the training and validation KH values and record which were used to train
        safef(filename, sizeof(filename), "%s/split%02d/fold%02d/metadata.tab", trainingDir, split, fold);
        fp = fopen(filename, "r");
        if(fp == NULL)
            errAbort("Couldn't open file %s\n", filename);
        matrix * trMetadata = f_fill_matrix(fp, 1);
        fclose(fp);

        safef(filename, sizeof(filename), "%s/split%02d/fold%02d/metadata.tab", validationDir, split, fold);
        fp = fopen(filename, "r");
        if(fp == NULL)
            errAbort("Couldn't open file %s\n", filename);
        matrix * valMetadata = f_fill_matrix(fp, 1);
        fclose(fp);

        struct slInt * trainingList = list_indices(trMetadata->cols);
        matrix * metadata = append_matrices(trMetadata, valMetadata, 1);

        safef(filename, sizeof(filename), "%s/split%02d/fold%02d/NMFpredictor.training.results", modelDir, split, fold);
        fp = fopen(filename , "r");
        if(!fp)
            errAbort("Couldn't open training results file %s", filename);
        matrix * trainingPred = f_fill_matrix(fp, 1);
        fclose(fp);

        safef(filename, sizeof(filename), "%s/split%02d/fold%02d/NMFpredictor.validation.results", modelDir, split, fold);
        fp = fopen(filename , "r");
        if(!fp)
            errAbort("Couldn't open validation results file %s", filename);
        matrix * valPred = f_fill_matrix(fp, 1);
        fclose(fp);

        //calc the accuracy by sample
        matrix * predictions = append_matrices(trainingPred, valPred, 1);
        matrix * accuraciesInFold = NMFpopulateAccuracyMatrix(predictions, metadata, trainingList);
        //add the accuracies to the running totals
        if(split == 1 && fold == 1)
            accuracies = copy_matrix(accuraciesInFold);
        else
            add_matrices_by_colLabel(accuracies, accuraciesInFold);
		//clean up
        free_matrix(trainingPred);
        free_matrix(valPred);
        free_matrix(predictions);
        free_matrix(trMetadata);
        free_matrix(valMetadata);
        free_matrix(metadata);
        free_matrix(accuraciesInFold);
        }
    }
//normalize accuracies over number of splits and folds
int i;
for(i = 0; i < accuracies->cols; i++)
    {
    if(accuracies->graph[0][i] != NULL_FLAG)
        accuracies->graph[0][i] = (accuracies->graph[0][i] / ((folds-1) * splits));
    if(accuracies->graph[1][i] != NULL_FLAG)
        accuracies->graph[1][i] = (accuracies->graph[1][i] / (1 * splits));
    }
return accuracies;
}

void sqlExecProgProfile(char *profile, char *prog, char **progArgs, int userArgc, char *userArgv[])
/* 
 * Exec one of the sql programs using user and password defined in localDb.XXX variables from ~/.hg.conf 
 * progArgs is NULL-terminate array of program-specific arguments to add,
 * which maybe NULL. userArgv are arguments passed in from the command line.
 * The program is execvp-ed, this function does not return. 
 */
{
int i, j = 0, nargc=cntArgv(progArgs)+userArgc+6, defaultFileNo, returnStatus;
pid_t child_id;
char **nargv, defaultFileName[256], defaultFileArg[256], *homeDir;

// install cleanup signal handlers
sqlProgInitSigHandlers();

/* Assemble defaults file */
if ((homeDir = getenv("HOME")) == NULL)
    errAbort("sqlExecProgProfile: HOME is not defined in environment; cannot create temporary password file");

nukeOldCnfs(homeDir);
// look for special parameter -profile=name
for (i = 0; i < userArgc; i++)
    if (startsWith("-profile=", userArgv[i]))
	profile=cloneString(userArgv[i]+strlen("-profile="));

safef(defaultFileName, sizeof(defaultFileName), "%s/.hgsql.cnf-XXXXXX", homeDir);
defaultFileNo=sqlMakeDefaultsFile(defaultFileName, profile, "client");

safef(defaultFileArg, sizeof(defaultFileArg), "--defaults-file=%s", defaultFileName);

AllocArray(nargv, nargc);

nargv[j++] = prog;
nargv[j++] = defaultFileArg;   /* --defaults-file must come before other options */
if (progArgs != NULL)
    {
    for (i = 0; progArgs[i] != NULL; i++)
        nargv[j++] = progArgs[i];
    }
for (i = 0; i < userArgc; i++)
    if (!startsWith("-profile=", userArgv[i]))
	nargv[j++] = userArgv[i];
nargv[j++] = NULL;

// flush before forking so we can't accidentally get two copies of the output
fflush(stdout);
fflush(stderr);

child_id = fork();
killChildPid = child_id;
if (child_id == 0)
    {
    execvp(nargv[0], nargv);
    _exit(42);  /* Why 42?  Why not?  Need something user defined that mysql isn't going to return */
    }
else
    {
    /* Wait until the child process completes, then delete the temp file */
    wait(&returnStatus);
    unlink (defaultFileName);
    if (WIFEXITED(returnStatus))
        {
	int childExitStatus = WEXITSTATUS(returnStatus);
        if (childExitStatus == 42)
            errAbort("sqlExecProgProfile: exec failed");
	else
	    // Propagate child's exit status:
	    _exit(childExitStatus);
        }
    else
        errAbort("sqlExecProgProfile: child process exited with abnormal status %d", returnStatus);
    }
}

void testOutSequence(struct htmlPage *tablePage, struct htmlForm *mainForm,
     char *org, char *db, char *group, char *track, char *table, 
     int expectedRows)
/* Get as sequence and make sure count agrees with expected. */
/* mainForm not used */
{
struct htmlPage *outPage;
int attempts = 0;
struct htmlFormVar *typeVar;

if (tablePage->forms == NULL) 
    errAbort("testOutSequence: Missing form (tablePage)");

htmlPageSetVar(tablePage, NULL, hgtaOutputType, "sequence");
outPage = quickSubmit(tablePage, org, db, group, track, table,
    "seqUi1", hgtaDoTopSubmit, "submit");
while (outPage == NULL && attempts < MAX_ATTEMPTS) 
    {
    printf("testOutSequence: trying again to get seqUi1\n");
    outPage = quickSubmit(tablePage, org, db, group, track, table,
        "seqUi1", hgtaDoTopSubmit, "submit");
    attempts++;
    }
if (outPage == NULL) 
    {
    qaStatusSoftError(tablesTestList->status,
        "Error in testOutSequence - couldn't get outPage");
    return;
    }
if (outPage->forms == NULL)
    {
    qaStatusSoftError(tablesTestList->status,
        "Error in testOutSequence - missing form");
    htmlPageFree(&outPage);
    return;
    }

/* Since some genomic sequence things are huge, this will
 * only test in case where it's a gene prediction. */
typeVar = htmlFormVarGet(outPage->forms, hgtaGeneSeqType);
if (typeVar != NULL)
    {
    struct htmlPage *seqPage;
    static char *types[] = {"protein", "mRNA"};
    int i;
    for (i=0; i<ArraySize(types); ++i)
        {
        char *type = types[i];
        if (slNameInList(typeVar->values, type))
             {
	     struct htmlPage *page;
	     char testName[128];
	     htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, type);
	     safef(testName, sizeof(testName), "%sSeq", type);
	     page = quickSubmit(outPage, org, db, group, track, table,
	        testName, hgtaDoGenePredSequence, "submit");
	     checkFaOutput(page, expectedRows, TRUE);
	     htmlPageFree(&page);
	     }
         }
    htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, "genomic");
    serialSubmit(&outPage, org, db, group, track, table, "seqUi2", hgtaDoGenePredSequence, "submit");
    // check that outPage != NULL

    /* On genomic page uncheck intron if it's there, then get results * and count them. */
    if (htmlFormVarGet(outPage->forms, "hgSeq.intron") != NULL)
         htmlPageSetVar(outPage, NULL, "hgSeq.intron", NULL);
    seqPage = quickSubmit(outPage, org, db, group, track, table, "genomicSeq", hgtaDoGenomicDna, "submit");
    // check that seqPage != NULL
    checkFaOutput(seqPage, expectedRows, FALSE);
    htmlPageFree(&seqPage);
    }

htmlPageFree(&outPage);
}

void doubleCellPrint(struct column *col, struct subjInfo *si,
        struct sqlConnection *conn)
/* print double value */
{
char *s = col->cellVal(col, si, conn);
char buf[256];
if (sameString(s,"."))  // known bad data value
    safef(buf,sizeof(buf),"%s", s);
else
    {
    if (sameWord(col->name, "LastPVisit") 	||
	sameWord(col->name, "LastTrVisit"))
	{
    	if (sameWord(s, "-1"))
	   {
    	   safef(buf,sizeof(buf),"N/A");
	   }
    	else if (sameWord(s, "-2"))
	   {
    	   safef(buf,sizeof(buf),"N/D");
	   }
    	else if (sameWord(s, "-3.000")||sameWord(s, "-3.0")||sameWord(s, "-3"))
	   {
    	   safef(buf,sizeof(buf),"&nbsp");
	   }
	else
	    {
    	    safef(buf,sizeof(buf),"%.1f",sqlDouble(s));
	    }
	}
    else if (sameWord(col->name, "LastTrCD4Blk")
         ||  sameWord(col->name, "LastPCD4Blk")
         ||  sameWord(col->name, "LastPAntiGP120")
         ||  sameWord(col->name, "LastTrAntiGP120"))
    	{
    	if (sameWord(s, "-3.000"))
	   {
    	   safef(buf,sizeof(buf),"&nbsp");
	   }
    	else if (sameWord(s, "-2"))
	   {
    	   safef(buf,sizeof(buf),"N/D");
	   }
    	else if (sameWord(s, "-1"))
	   {
    	   safef(buf,sizeof(buf),"N/A");
	   }
	else
	    {
    	    safef(buf,sizeof(buf),"%.3f",sqlDouble(s));
	    }
	}
    else
	{
    	safef(buf,sizeof(buf),"%.1f",sqlDouble(s));
	}
    }
freeMem(s);
hPrintf("<TD align=right>");
hPrintf("%s", buf);
hPrintf("</TD>");
}

static void doBlat(struct sqlConnection *conn, int taxon, char *db)
/* place probe seq from non-BAC with blat that have no alignments yet */
{
int rc = 0;
char *blatSpec=NULL;
char cmdLine[256];
char path1[256];
char path2[256];
struct dyString *dy = dyStringNew(0);

/* (non-BACs needing alignment) */
dyStringClear(dy);
dyStringPrintf(dy, 
    "select concat(\"vgPrb_\",e.id), e.seq"
    " from vgPrb e, vgPrbAli a"
    " where e.id = a.vgPrb"
    " and a.db = '%s'"
    " and a.status = 'new'"
    " and e.taxon = %d"
    " and e.type <> 'bac'"
    " and e.seq <> ''"
    " order by e.id"
    , db, taxon);
//restore: 
rc = sqlSaveQuery(conn, dy->string, "blat.fa", TRUE);
verbose(1,"rc = %d = count of sequences for blat, to get psls for taxon %d\n",rc,taxon);

if (rc == 0) 
    {
    unlink("blat.fa");
    system("rm -f blatNearBest.psl; touch blatNearBest.psl");  /* make empty file */
    return;
    }

/* make .ooc and blat on kolossus */

safef(path1,sizeof(path1),"/gbdb/%s/%s.2bit",db,db);
safef(path2,sizeof(path2),"%s/%s.2bit",getCurrentDir(),db);
//restore: 
verbose(1,"copy: [%s] to [%s]\n",path1,path2);  copyFile(path1,path2);

safef(cmdLine,sizeof(cmdLine),
"ssh kolossus 'cd %s; blat -makeOoc=11.ooc -tileSize=11"
" -repMatch=1024 %s.2bit /dev/null /dev/null'",
    getCurrentDir(),db);
//restore: 
system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");

safef(cmdLine,sizeof(cmdLine),
	"ssh kolossus 'cd %s; blat %s.2bit blat.fa -ooc=11.ooc -noHead blat.psl'",
	getCurrentDir(),db);
//restore: 
system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");

/* using blat even with -fastMap was way too slow - took over a day,
 * so instead I will make a procedure to write a fake psl for the BACs
 * which you will see called below */

safef(path2,sizeof(path2),"%s.2bit",db);
verbose(1,"rm %s\n",path2); unlink(path2); 

safef(path2,sizeof(path2),"11.ooc");
verbose(1,"rm %s\n",path2); unlink(path2); 


/* skip psl header and sort on query name */
safef(cmdLine,sizeof(cmdLine), "sort -k 10,10 blat.psl > blatS.psl");
verbose(1,"cmdLine=[%s]\n",cmdLine);
system(cmdLine); 

/* keep near best within 5% of the best */
safef(cmdLine,sizeof(cmdLine), 
    "pslCDnaFilter -globalNearBest=0.005 -minId=0.96 -minNonRepSize=20 -minCover=0.50"
    " blatS.psl blatNearBest.psl");
verbose(1,"cmdLine=[%s]\n",cmdLine);
system(cmdLine); 

unlink("blat.fa");
unlink("blat.psl");
unlink("blatS.psl");

freez(&blatSpec);
dyStringFree(&dy);
}

static void doPslMapAli(struct sqlConnection *conn, 
    int taxon, char *db, 
    int fromTaxon, char *fromDb)
{
char cmd[256];

struct dyString *dy = dyStringNew(0);
char path[256];
char dnaPath[256];
char toDb[12];

safef(toDb,sizeof(toDb),"%s", db);
toDb[0]=toupper(toDb[0]);

safef(dnaPath,sizeof(dnaPath),"/cluster/data/%s/nib", db);
if (!fileExists(dnaPath))
    {
    safef(dnaPath,sizeof(dnaPath),"/cluster/data/%s/%s.2bit", db, db);
    if (!fileExists(dnaPath))
	errAbort("unable to locate nib dir or .2bit for %s: %s", db, dnaPath);
    }
    
safef(path,sizeof(path),"/gbdb/%s/liftOver/%sTo%s.over.chain.gz", fromDb, fromDb, toDb);
if (!fileExists(path))
    errAbort("unable to locate chain file %s",path);

/* get non-bac $db.vgProbes not yet aligned */
getPslMapAli(conn, db, fromTaxon, fromDb, FALSE);
/* get bac $db.vgProbes not yet aligned */
getPslMapAli(conn, db, fromTaxon, fromDb, TRUE);
/* get .fa for pslRecalcMatch use */
getPslMapFa(conn, db, fromTaxon);

/* non-bac */
safef(cmd,sizeof(cmd),
"zcat %s | pslMap -chainMapFile -swapMap  nonBac.psl stdin stdout "
"|  sort -k 14,14 -k 16,16n > unscoredNB.psl"
,path);
verbose(1,"%s\n",cmd); system(cmd);

safef(cmd,sizeof(cmd),
"pslRecalcMatch unscoredNB.psl %s" 
" pslMap.fa nonBac.psl"
,dnaPath);
verbose(1,"%s\n",cmd); system(cmd);

/* bac */
safef(cmd,sizeof(cmd),
"zcat %s | pslMap -chainMapFile -swapMap  bac.psl stdin stdout "
"|  sort -k 14,14 -k 16,16n > unscoredB.psl"
,path);
verbose(1,"%s\n",cmd); system(cmd);

safef(cmd,sizeof(cmd),
"pslRecalcMatch unscoredB.psl %s" 
" pslMap.fa bacTemp.psl"
,dnaPath);
verbose(1,"%s\n",cmd); system(cmd);

safef(cmd,sizeof(cmd),
"pslCDnaFilter -globalNearBest=0.00001 -minCover=0.05"
" bacTemp.psl bac.psl");
verbose(1,"%s\n",cmd); system(cmd);

safef(cmd,sizeof(cmd),"cat bac.psl nonBac.psl > vgPrbPslMap.psl");
verbose(1,"%s\n",cmd); system(cmd);

dyStringFree(&dy);

}

static struct tissueSampleVals *getTissueSampleVals(char *geneId, boolean doLogTransform,
                                                char *version, double *maxValRet)
/* Get sample data for the gene.  Optionally log10 it. Return maximum value seen */
{
struct hash *tsHash = hashNew(0);
struct tissueSampleVals *tsv;
struct hashEl *hel;
struct slDouble *val;
double maxVal = 0;
struct gtexSampleData *sd = NULL;
char query[256];
char **row;
char buf[256];
char *sampleDataTable = "gtexSampleData";
safef(buf, sizeof(buf), "%s%s", sampleDataTable, gtexVersionSuffixFromVersion(version));
struct sqlConnection *conn = hAllocConn("hgFixed");
assert(sqlTableExists(conn, buf));
sqlSafef(query, sizeof(query), "select * from %s where geneId='%s'", buf, geneId);
struct sqlResult *sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
    {
    sd = gtexSampleDataLoad(row);
    if ((hel = hashLookup(tsHash, sd->tissue)) == NULL)
        {
        AllocVar(tsv);
        hashAdd(tsHash, sd->tissue, tsv);
        }
    else
        tsv = (struct tissueSampleVals *)hel->val;
    maxVal = max(maxVal, sd->score);
    val = slDoubleNew(sd->score);
    slAddHead(&tsv->valList, val);
    }
/*  Fill in tissue descriptions, fill values array and calculate stats for plotting
        Then make a list, suitable for sorting by tissue or score
    NOTE: Most of this not needed for R implementation */
struct gtexTissue *tis = NULL, *tissues = gtexGetTissues(version);
struct tissueSampleVals *tsList = NULL;
int i;
if (doLogTransform)
    maxVal = log10(maxVal+1.0);
for (tis = tissues; tis != NULL; tis = tis->next)
    {
    tsv = hashFindVal(tsHash, tis->name);
    if (tsv == NULL)
        {
        /* no non-zero values for this tissue/gene */
        AllocVar(tsv);
        val = slDoubleNew(0.0);
        slAddHead(&tsv->valList, val);
        }
    tsv->name = tis->name;
    tsv->description = tis->description;
    tsv->color = tis->color;
    int count = tsv->count = slCount(tsv->valList);
    double *vals = AllocArray(tsv->vals, count);
    for (i=0; i<count; i++)
        {
        val = slPopHead(&tsv->valList);
        if (doLogTransform)
            vals[i] = log10(val->val+1.0);
        else
            vals[i] = val->val;
        }
    doubleBoxWhiskerCalc(tsv->count, tsv->vals, 
                                &tsv->min, &tsv->q1, &tsv->median, &tsv->q3, &tsv->max);
    slAddHead(&tsList, tsv);
    }
if (maxValRet != NULL)
    *maxValRet = maxVal;
return tsList;
}

struct hash *makeMotifBed(char *gffDir, char *outBed)
/* Make bed file from GFFs.  Return hash of transcription factors. */
{
static char *consLevelPath[3] = {"3", "2", "0"};
static char *consLevelBed[3] = {"2", "1", "0"};
static char *pLevelPath[3] = {"p001b", "p005b", "nobind"};
static char *pLevelBed[3] = {"good", "weak", "none"};
int cIx, pIx;
FILE *f = mustOpen(outBed, "w");
struct hash *tfHash = newHash(0);
struct hash *yrcHash = newHash(18);
struct yrc *yrcList = NULL, *yrc;

for (cIx=0; cIx<3; ++cIx)
   {
   for (pIx=0; pIx<3; ++pIx)
       {
       struct lineFile *lf;
       char *row[10];
       char fileName[PATH_LEN];
       char hashKey[256];

       safef(fileName, sizeof(fileName), "%s/IGR_v24.%s.%s.GFF",
       	   gffDir, consLevelPath[cIx], pLevelPath[pIx]);
       lf = lineFileOpen(fileName, TRUE);
       while (lineFileRow(lf, row))
            {
	    char *name = row[9];
	    char *e;
	    int chromIx, chromStart, chromEnd;
	    if (!sameWord(row[8], "Site"))
	        errAbort("Expecting 'Site' line %d of %s", lf->lineIx, lf->fileName);
	    e = strchr(name, ';');
	    if (e == NULL)
	        errAbort("Expecting semicolon line %d of %s", lf->lineIx, lf->fileName);
	    *e = 0;
	    chromIx = romanToArabicChrom(row[0], lf);
	    chromStart = lineFileNeedNum(lf, row, 3);
	    chromEnd = lineFileNeedNum(lf, row, 4);
	    safef(hashKey, sizeof(hashKey), "%s.%d.%d", name, chromIx, chromStart);
	    if ((yrc = hashFindVal(yrcHash, hashKey)) == NULL)
	        {
		AllocVar(yrc);
		yrc->chromIx= chromIx;
		yrc->chromStart = chromStart;
		yrc->chromEnd = chromEnd;
		yrc->name = hashStoreName(tfHash, name);
		yrc->pLevel = pIx;
		yrc->consLevel = cIx;
		hashAdd(yrcHash, hashKey, yrc);
		slAddHead(&yrcList, yrc);
		}
	    else
	        {
		if (pIx < yrc->pLevel)
		    yrc->pLevel = pIx;
		if (cIx < yrc->consLevel)
		    yrc->consLevel = cIx;
		}
	    }
       lineFileClose(&lf);
       }
   }
for (yrc = yrcList; yrc != NULL; yrc = yrc->next)
    {
    fprintf(f, "chr%d\t", yrc->chromIx+1);
    fprintf(f, "%d\t", yrc->chromStart);
    fprintf(f, "%d\t", yrc->chromEnd);
    fprintf(f, "%s\t", yrc->name);
    fprintf(f, "%d\t", (int)(1000/(yrc->pLevel + yrc->consLevel + 1)));
    fprintf(f, "%s\t", pLevelBed[yrc->pLevel]);
    fprintf(f, "%s\n", consLevelBed[yrc->consLevel]);
    }
carefulClose(&f);
hashFree(&yrcHash);
return tfHash;
}

void getInvariants(char *db, struct snpExceptions *exceptionList, 
		   struct slName *chromList, char *fileBase)
/* write list of invariants to output file */
{
struct sqlConnection *conn     = hAllocConn(db);
struct sqlResult     *sr       = NULL;
struct snpExceptions *el       = NULL;
struct slName        *chrom    = NULL;
char                **row      = NULL;
char                  query[1024];
unsigned long int     invariantCount;
char                  thisFile[64];
FILE                 *outFile;
int                   colCount, i;
char                  idString[3];

for (el=exceptionList; el!=NULL; el=el->next)
    {
    if (el->exceptionId<10)
	safef(idString,sizeof(idString), "0%d", el->exceptionId);
    else
	safef(idString,sizeof(idString), "%d",  el->exceptionId);
    invariantCount = 0;
    if (startsWith("select",el->query))
	{
	safef(thisFile, sizeof(thisFile), "%s.%s.bed", fileBase, idString);
	outFile = mustOpen(thisFile, "w");
	fprintf(outFile, "# exceptionId:\t%d\n# query:\t%s;\n", el->exceptionId, el->query);
	for (chrom=chromList; chrom!=NULL; chrom=chrom->next)
	    {
	    fflush(outFile); /* to keep an eye on output progress */
	    sqlSafef(query, sizeof(query),
		  "%-s and chrom='%s'", el->query, chrom->name);
	    sr = sqlGetResult(conn, query);
	    colCount = sqlCountColumns(sr);
	    while ((row = sqlNextRow(sr))!=NULL)
		{
		invariantCount++; 
		fprintf(outFile, "%s", row[0]);
		for (i=1; i<colCount; i++)
		    fprintf(outFile, "\t%s", row[i]);
		fprintf(outFile, "\n");
		}
	    }
	}
    else if (startsWith("group",el->query))
	{
	struct slName *nameList = NULL;
	struct slName *name     = NULL;
	safef(thisFile, sizeof(thisFile), "%s.%s.bed", fileBase, idString);
	outFile = mustOpen(thisFile, "w");
	fprintf(outFile, "# exceptionId:\t%d\n# query:\t%s;\n", el->exceptionId, el->query);
	nameList = getGroupList(db, el->query);
	for (name=nameList; name!=NULL; name=name->next)
	    {
	    sqlSafef(query, sizeof(query),
		  "select chrom,chromStart,chromEnd,name,%d as score,class,locType,observed "
		  "from snp where name='%s'", el->exceptionId, name->name);
	    sr = sqlGetResult(conn, query);
	    colCount = sqlCountColumns(sr);
	    while ((row = sqlNextRow(sr))!=NULL)
		{
		invariantCount++; 
		fprintf(outFile, "%s", row[0]);
		for (i=1; i<colCount; i++)
		    fprintf(outFile, "\t%s", row[i]);
		fprintf(outFile, "\n");
		}
	    }
	}
    else
	{
	printf("Invariant %d has no query string\n", el->exceptionId);
	continue;
	}
    carefulClose(&outFile);
    printf("Invariant %d has %lu exceptions, written to this file: %s\n", 
	   el->exceptionId, invariantCount, thisFile);
    fflush(stdout);
    sqlSafef(query, sizeof(query),
	  "update snpExceptions set num=%lu where exceptionId=%d",
	  invariantCount, el->exceptionId);    
    sr=sqlGetResult(conn, query); /* there's probably a better way to do this */
    }
}

static void doDownload(struct sqlConnection *conn)
/* Try to force user's browser to download by giving special response headers */
{
int imageId = cartUsualInt(cart, hgpId, 0);
char url[1024];
char *p = NULL;
char dir[256];
char name[128];
char extension[64];
int w = 0, h = 0;
int sd = -1;

if (!visiGeneImageSize(conn, imageId, &w, &h))
    imageId = 0;

if (imageId == 0)
    {
    problemPage("invalid imageId","");
    }
else
    {
    p=visiGeneFullSizePath(conn, imageId);
    splitPath(p, dir, name, extension);
    safef(url,sizeof(url),"%s%s%s", dir, name, extension);
    sd = netUrlOpen(url);
    if (sd < 0)
	{
	problemPage("Couldn't open", url);
	}
    else
	{
	char *newUrl = NULL;
	int newSd = 0;
	/* url needed for err msgs and redirect url*/
	if (netSkipHttpHeaderLinesHandlingRedirect(sd, url, &newSd, &newUrl))
	    {
	    char buf[32*1024];
	    int readSize;
	    if (newUrl)
		{
		freeMem(newUrl);
		sd = newSd;
		}
	    printf("Content-Type: application/octet-stream\n");
	    printf("Content-Disposition: attachment; filename=%s%s\n", name, extension);
	    printf("\n");
	    while ((readSize = read(sd, buf, sizeof(buf))) > 0)
	        fwrite(buf, 1,  readSize, stdout);
	    close(sd);
	    sd = -1;
	    fflush(stdout);
	    fclose(stdout);
	    }
	else
	    {
	    problemPage("Skip http header problem", url);
    	    }
	freeMem(newUrl);
	}
    }
}

void doImage(struct sqlConnection *conn)
/* Put up image page. */
{
int imageId = cartUsualInt(cart, hgpId, 0);
char *sidUrl = cartSidUrlString(cart);
char buf[1024];
char url[1024];
char *p = NULL;
char dir[256];
char name[128];
char extension[64];
int w = 0, h = 0;
htmlSetBgColor(0xE0E0E0);
htmStart(stdout, "do image");

puts(
"<script type=\"text/JavaScript\">"
"document.getElementsByTagName('html')[0].style.height=\"100%\";"
"document.getElementsByTagName('body')[0].style.height=\"100%\";"
"</script>"
);

if (!visiGeneImageSize(conn, imageId, &w, &h))
    imageId = 0;

if (imageId != 0)
    {
    printf("<B>");
    smallCaption(conn, imageId);
    printf(".</B> Click image to zoom in, drag or arrow keys to move. "
	   "Caption is below.<BR>\n");

    p=visiGeneFullSizePath(conn, imageId);

    splitPath(p, dir, name, extension);
#ifdef DEBUG
    safef(buf,sizeof(buf),"../bigImageTest.html?url=%s%s/%s&w=%d&h=%d",
	    dir,name,name,w,h);
#else
    safef(buf,sizeof(buf),"../bigImage.html?url=%s%s/%s&w=%d&h=%d",
	    dir,name,name,w,h);
#endif
    printf("<IFRAME name=\"bigImg\" width=\"100%%\" height=\"90%%\" SRC=\"%s\"></IFRAME><BR>\n", buf);

    fullCaption(conn, imageId);

    safef(buf,sizeof(buf),"%s%s%s", dir, name, extension);
    safef(url,sizeof(url),"%s?%s=go&%s&%s=%d",
    	hgVisiGeneCgiName(), hgpDoDownload, sidUrl, hgpId, imageId);

    printf("<B>Full-size image:</B> %d x %d &nbsp; <A HREF='%s'> download </A> ", w, h, url);

    /* Currently this is dangerous for users with less than 1 GB RAM to use
       on large images, because their machines can thrash themselves into a coma.
       X-windows (i.e. used by FireFox) will allocate 5 bytes per pixel.
       If the image size in pixels times 5 exceeds real ram size, then
       Linux thrashes incessantly.  But you can hit ctrl-alt-F1 to
       get a text only screen, then kill the bad processes (FF) and then
       you can restore desktop with ctrl-alt-F7.  Hiram says that's a
       feature credited to SCO-Unix.  On my 1GB machines at work/home,
       I never encountered any problem what-so-ever, even with the
       largest visiGene AllenBrain - about 19000x9000 pix.

    printf(" &nbsp;&nbsp; <A HREF='%s'> view </A>\n", buf);
    */
    printf("\n");

    }
htmlEnd();
}

//.........这里部分代码省略.........
		if (differentWord(row[nameIx], el->name))
		    errAbort("ERROR: query: '%s' did not return table name '%s' != '%s'\n", query, el->name, row[nameIx]);

		SCAN_STATUS;

		if (timep < dropTime)
		    {
		    slNameAddHead(&expiredTableNames, row[nameIx]);
		    verbose(2,"%s %ld dropt %s lost table\n",
			row[timeIxUsed], (unsigned long)timep, row[nameIx]);
		    }
		else
		    verbose(3,"%s %ld OKt %s\n",
			row[timeIxUsed], (unsigned long)timep, row[nameIx]);
		}
	    sqlFreeResult(&sr);
	    }
	}
    }

/*	perhaps the table was already dropped, but not from the metaInfo */
struct hashEl *elList = hashElListHash(expiredHash);
struct hashEl *el;
for (el = elList; el != NULL; el = el->next)
    {
    verbose(2,"%s exists in %s only\n", el->name, CT_META_INFO);
    if (drop)
	ctTouchLastUse(conn, el->name, FALSE); /* removes metaInfo row */
    }

if (drop)
    {
    char comment[256];
    if (expiredTableNames)
	{
	struct slName *el;
	int droppedCount = 0;
	/* customTrash DB user permissions do not have permissions to
 	 * drop tables.  Must use standard special user that has all
 	 * permissions.  If we are not using the standard user at this
 	 * point, then switch to it.
	 */
	if (sameWord(db,CUSTOM_TRASH))
	    {
	    sqlDisconnect(&conn);
	    conn = sqlConnect(db);
	    }
	for (el = expiredTableNames; el != NULL; el = el->next)
	    {
	    verbose(2,"# drop %s\n", el->name);
	    sqlDropTable(conn, el->name);
	    ctTouchLastUse(conn, el->name, FALSE); /* removes metaInfo row */
	    ++droppedCount;
	    }
	/* add a comment to the history table and finish up connection */
	if (tableStatus)
	    safef(comment, sizeof(comment), "Dropped %d tables with "
		"total size %llu, %llu lost tables",
		    droppedCount, totalSize, lostTableCount);
	else
	    safef(comment, sizeof(comment),
		"Dropped %d tables, no size info, %llu lost tables",
		    droppedCount, lostTableCount);
	verbose(2,"# %s\n", comment);
	hgHistoryComment(conn, "%s", comment);
	}
    else
	{
	safef(comment, sizeof(comment),
	    "Dropped no tables, none expired, %llu lost tables",
		lostTableCount);
	verbose(2,"# %s\n", comment);
	}
    }
else
    {
    char comment[256];
    if (expiredTableNames)
	{
	int droppedCount = slCount(expiredTableNames);
	if (tableStatus)
	    safef(comment, sizeof(comment), "Would have dropped %d tables with "
		"total size %llu, %llu lost tables",
		    droppedCount, totalSize, lostTableCount);
	else
	    safef(comment, sizeof(comment),
		"Would have dropped %d tables, no size info, %llu lost tables",
		    droppedCount, lostTableCount);
	verbose(2,"# %s\n", comment);
	}
    else
	{
	safef(comment, sizeof(comment),
	    "Would have dropped no tables, none expired, %llu lost tables",
		lostTableCount);
	verbose(2,"# %s\n", comment);
	}
    }
sqlDisconnect(&conn);
}

static void doSeqAndExtFile(struct sqlConnection *conn, char *db, char *table)
{
int rc = 0;
char cmd[256];
char path[256];
char bedPath[256];
char gbdbPath[256];
char *fname=NULL;
struct dyString *dy = dyStringNew(0);
dyStringClear(dy);
dyStringPrintf(dy, 
"select distinct concat('vgPrb_',e.id), e.seq"
" from vgPrb e join %s.%s v"
" left join %s.seq s on s.acc = v.qName"
" where concat('vgPrb_',e.id) = v.qName"
" and s.acc is NULL"
" order by e.id"
    , db, table, db);
rc = sqlSaveQuery(conn, dy->string, "vgPrbExt.fa", TRUE);
verbose(1,"rc = %d = count of sequences for vgPrbExt.fa, to use with %s track %s\n",rc,db,table);
if (rc > 0)  /* can set any desired minimum */
    {
    safef(bedPath,sizeof(bedPath),"/cluster/data/%s/bed/visiGene/",db);
    if (!fileExists(bedPath))
	{
	safef(cmd,sizeof(cmd),"mkdir %s",bedPath);
	verbose(1,"%s\n",cmd); system(cmd);
	}
    
    safef(gbdbPath,sizeof(gbdbPath),"/gbdb/%s/visiGene/",db);
    if (!fileExists(gbdbPath))
	{
	safef(cmd,sizeof(cmd),"mkdir %s",gbdbPath);
    	verbose(1,"%s\n",cmd); system(cmd);
	}
   
    while(1)
	{
	int i=0;
	safef(path,sizeof(path),"%svgPrbExt_AAAAAA.fa",bedPath);
        char *c = rStringIn("AAAAAA",path);
        srand( (unsigned)time( NULL ) );
        for(i=0;i<6;++i)
            {
            *c++ += (int) 26 * (rand() / (RAND_MAX + 1.0));
            }
	if (!fileExists(path))
	    break;
	}

    
    safef(cmd,sizeof(cmd),"cp vgPrbExt.fa %s",path);
    verbose(1,"%s\n",cmd); system(cmd);
    
    fname = rStringIn("/", path);
    ++fname;
    
    safef(cmd,sizeof(cmd),"ln -s %s %s%s",path,gbdbPath,fname);
    verbose(1,"%s\n",cmd); system(cmd);
    
    safef(cmd,sizeof(cmd),"hgLoadSeq %s %s%s", db, gbdbPath,fname);
    verbose(1,"%s\n",cmd); system(cmd);
    }

dyStringFree(&dy);
}

void *netConnectHttpsThread(void *threadParam)
/* use a thread to run socket back to user */
{
/* child */

struct netConnectHttpsParams *params = threadParam;

pthread_detach(params->thread);  // this thread will never join back with it's progenitor

int fd=0;

char hostnameProto[256];

BIO *sbio;
SSL_CTX *ctx;
SSL *ssl;

openSslInit();

ctx = SSL_CTX_new(SSLv23_client_method());

fd_set readfds;
fd_set writefds;
int err;
struct timeval tv;


/* TODO checking certificates 

char *certFile = NULL;
char *certPath = NULL;
if (certFile || certPath)
    {
    SSL_CTX_load_verify_locations(ctx,certFile,certPath);
#if (OPENSSL_VERSION_NUMBER < 0x0090600fL)
    SSL_CTX_set_verify_depth(ctx,1);
#endif
    }

// verify paths and mode.

*/


sbio = BIO_new_ssl_connect(ctx);

BIO_get_ssl(sbio, &ssl);
if(!ssl) 
    {
    xerr("Can't locate SSL pointer");
    goto cleanup;
    }

/* Don't want any retries since we are non-blocking bio now */
//SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);


safef(hostnameProto,sizeof(hostnameProto),"%s:%d",params->hostName,params->port);
BIO_set_conn_hostname(sbio, hostnameProto);

BIO_set_nbio(sbio, 1);     /* non-blocking mode */

while (1) 
    {
    if (BIO_do_connect(sbio) == 1) 
	{
	break;  /* Connected */
	}
    if (! BIO_should_retry(sbio)) 
	{
	xerr("BIO_do_connect() failed");
	char s[256];	
	safef(s, sizeof s, "SSL error: %s", ERR_reason_error_string(ERR_get_error()));
	xerr(s);
	goto cleanup;
	}

    fd = BIO_get_fd(sbio, NULL);
    if (fd == -1) 
	{
	xerr("unable to get BIO descriptor");
	goto cleanup;
	}
    FD_ZERO(&readfds);
    FD_ZERO(&writefds);
    if (BIO_should_read(sbio)) 
	{
	FD_SET(fd, &readfds);
	}
    else if (BIO_should_write(sbio)) 
	{
	FD_SET(fd, &writefds);
	}
    else 
	{  /* BIO_should_io_special() */
	FD_SET(fd, &readfds);
	FD_SET(fd, &writefds);
	}
    tv.tv_sec = (long) (DEFAULTCONNECTTIMEOUTMSEC/1000);  // timeout default 10 seconds
    tv.tv_usec = (long) (((DEFAULTCONNECTTIMEOUTMSEC/1000)-tv.tv_sec)*1000000);
//.........这里部分代码省略.........

static void doPrimers(struct sqlConnection *conn, int taxon, char *db)
/* get probe seq from primers */
{
int rc = 0;
struct dyString *dy = dyStringNew(0);
char cmdLine[256];
char path1[256];
char path2[256];

dyStringClear(dy);
dyStringAppend(dy, "select e.id, p.fPrimer, p.rPrimer from probe p, vgPrbMap m, vgPrb e, gene g");
dyStringPrintf(dy, " where p.id = m.probe and m.vgPrb = e.id and g.id = p.gene and g.taxon = %d",taxon);
dyStringAppend(dy, " and e.state = 'new' and e.type='primersMrna'");
rc = sqlSaveQuery(conn, dy->string, "primers.query", FALSE);
verbose(1,"rc = %d = count of primers for mrna search for taxon %d\n",rc,taxon);

if (rc > 0) /* something to do */
    {

    dyStringClear(dy);
    dyStringPrintf(dy, "select qName from %s.all_mrna",db);
    rc = 0;
    rc = sqlSaveQuery(conn, dy->string, "accFile.txt", FALSE);
    safef(cmdLine,sizeof(cmdLine),"getRna %s accFile.txt mrna.fa",db);
    system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");
    
    verbose(1,"rc = %d = count of mrna for %s\n",rc,db);

    system("date"); system("isPcr mrna.fa primers.query isPcr.fa -out=fa"); system("date");
    system("ls -l");

    processIsPcr(conn,taxon,db);

    unlink("mrna.fa"); unlink("accFile.txt"); unlink("isPcr.fa");

    }
unlink("primers.query");    

/* find any remaining type primersMrna that couldn't be resolved and demote 
 * them to type primersGenome
 */
dyStringClear(dy);
dyStringAppend(dy, "update vgPrb set type='primersGenome'"); 
dyStringPrintf(dy, " where taxon = %d",taxon);
dyStringAppend(dy, " and state = 'new' and type='primersMrna'");
sqlUpdate(conn, dy->string);



/* get primers for those probes that did not find mrna isPcr matches 
 * and then do them against the genome instead */
dyStringClear(dy);
dyStringAppend(dy, "select e.id, p.fPrimer, p.rPrimer from probe p, vgPrbMap m, vgPrb e, gene g");
dyStringPrintf(dy, " where p.id = m.probe and m.vgPrb = e.id and g.id = p.gene and g.taxon = %d",taxon);
dyStringAppend(dy, " and e.state = 'new' and e.type='primersGenome'");
rc = 0;
rc = sqlSaveQuery(conn, dy->string, "primers.query", FALSE);
verbose(1,"rc = %d = count of primers for genome search for taxon %d\n",rc,taxon);

if (rc > 0) /* something to do */
    {
    safef(path1,sizeof(path1),"/gbdb/%s/%s.2bit",db,db);
    safef(path2,sizeof(path2),"%s/%s.2bit",getCurrentDir(),db);
    verbose(1,"copy: [%s] to [%s]\n",path1,path2);  copyFile(path1,path2);

    safef(cmdLine,sizeof(cmdLine),
	    "ssh kolossus 'cd %s; isPcr %s.2bit primers.query isPcr.fa -out=fa'",
	    getCurrentDir(),db);
    system("date"); verbose(1,"cmdLine: [%s]\n",cmdLine); system(cmdLine); system("date");
    safef(path2,sizeof(path2),"%s/%s.2bit",getCurrentDir(),db);
    verbose(1,"rm %s\n",path2)