struct composite *makeCompositeList(struct encode2Manifest *manList, struct hash *metaHash)
/* Return a list of composites with everything on manList */
{
struct composite *comp, *compList = NULL;
struct hash *compHash = hashNew(0);
char compName[256];
struct encode2Manifest *man;
for (man = manList; man != NULL; man = man->next)
    {
    char *realComp = tagVal(man, metaHash, "composite");
    if (realComp != NULL)
        safef(compName, sizeof(compName), "%s", realComp);
    else
        {
	char *lab = emptyForNull(tagVal(man, metaHash, "lab"));
	char *dataType = emptyForNull(tagVal(man, metaHash, "dataType"));
	safef(compName, sizeof(compName), "comp%s%s", lab, dataType);
	}
    comp = hashFindVal(compHash, compName);
    if (comp == NULL)
        {
	AllocVar(comp);
	comp->name = cloneString(compName);
	hashAdd(compHash, compName, comp);
	slAddTail(&compList, comp);
	}
    struct slRef *manRef = slRefNew(man);
    slAddTail(&comp->manRefList, manRef);
    }
hashFree(&compHash);
return compList;
}

struct polygon *makeTriangle(int x, int y, int z)
/* Make triangle about center x,y,z */
{
struct polygon *tri;
struct point *pt;
int i;
AllocVar(tri);
tri->id = nextId();
tri->pointCount = 3;
AllocVar(pt);
strcpy(pt->acc, "acc1");
pt->pt.x = pt->x = x;
pt->pt.y = pt->y = y-100;
pt->z = z;
slAddTail(&tri->points, pt);
AllocVar(pt);
strcpy(pt->acc, "acc2");
pt->pt.x = pt->x = x-100;
pt->pt.y = pt->y = y+100;
pt->z = z;
slAddTail(&tri->points, pt);
AllocVar(pt);
strcpy(pt->acc, "acc3");
pt->pt.x = pt->x = x+100;
pt->pt.y = pt->y = y+100;
pt->z = z;
slAddTail(&tri->points, pt);
AllocArray(tri->persp, 3);
for (i=0, pt = tri->points; i < 3; ++i, pt = pt->next)
    {
    tri->persp[i].x = pt->x/2;
    tri->persp[i].y = pt->y/2;
    }
return tri;
}

static struct pipeline* pipelineNew(char ***cmds, unsigned options)
/* create a new pipeline object. Doesn't start processes */
{
static char *memPseudoCmd[] = {"[mem]", NULL};
struct pipeline *pl;
int iCmd;

AllocVar(pl);
pl->pipeFd = -1;
pl->options = options;
pl->procName = joinCmds(cmds);

if (cmds[0] == NULL)
    errAbort("no commands in pipeline");

if (options & pipelineMemInput)
    {
    /* add proc for forked process to write memory to pipeline */
    slAddTail(&pl->procs, plProcNew(memPseudoCmd, pl));
    }

for(iCmd = 0; cmds[iCmd] != NULL; iCmd++)
    slAddTail(&pl->procs, plProcNew(cmds[iCmd], pl));

return pl;
}

void cJoinX(char *j1, char *j2, char *j3)
/* cJoinX - Experiment in C joining.. */
{
struct joiner *joiner = joinerRead("../../makeDb/schema/all.joiner");
struct joinerDtf *a = joinerDtfFromDottedTriple(j1);
struct joinerDtf *b = joinerDtfFromDottedTriple(j2);
struct joinerDtf *c = joinerDtfFromDottedTriple(j3);
struct joinerPair *jpList = NULL, *jp;
struct joinerDtf *fieldList = NULL;
struct hash *visitedHash = hashNew(0);

slAddTail(&fieldList, a);
slAddTail(&fieldList, b);
slAddTail(&fieldList, c);

if (joinerDtfAllSameTable(fieldList))
    printf("All in same table, easy enough!\n");
else
    {
    jpList = joinerFindRouteThroughAll(joiner, fieldList);
    for (jp = jpList; jp != NULL; jp = jp->next)
	{
	printf("%s.%s.%s -> %s.%s.%s\n", 
	    jp->a->database, jp->a->table, jp->a->field,
	    jp->b->database, jp->b->table, jp->b->field);
	}
    }
}

static struct gpFx *gpFxInExon(struct variant *variant, struct txCoords *txc, int exonIx,
			       struct genePred *pred, boolean predIsNmd,
			       struct dnaSeq *transcriptSeq, struct lm *lm)
/* Given a variant that overlaps an exon of pred, figure out what each allele does. */
{
struct gpFx *effectsList = NULL;
struct allele *allele = variant->alleles;
for ( ; allele ; allele = allele->next)
    {
    if (!allele->isReference)
	{
	if (pred->cdsStart != pred->cdsEnd)
	    {
	    // first find effects of allele in UTR, if any
	    effectsList = slCat(effectsList,
				gpFxCheckUtr(allele, pred, txc, exonIx, predIsNmd, lm));
	    if (txc->startInCds >= 0)
		effectsList = slCat(effectsList,
				    gpFxChangedCds(allele, pred, txc, exonIx, predIsNmd,
						   transcriptSeq, lm));
	    }
	else
	    effectsList = slCat(effectsList,
				gpFxChangedNoncodingExon(allele, pred, txc, exonIx, lm));

	if (!predIsNmd)
	    {
	    // Was entire exon deleted?
	    int exonNumPos = exonIx;
	    if (pred->strand[0] == '-')
		exonNumPos = pred->exonCount - 1 - exonIx;
	    uint exonStart = pred->exonStarts[exonNumPos], exonEnd = pred->exonEnds[exonNumPos];
	    if (variant->chromStart <= exonStart && variant->chromEnd >= exonEnd)
		{
		struct gpFx *effect = gpFxNew(allele->sequence, pred->name, exon_loss,
					      nonCodingExon, lm);
		setNCExonVals(effect, exonIx, txc->startInCdna);
		slAddTail(&effectsList, effect);
		}
	    else
		{
		// If variant is in exon *but* within 3 bases of splice site,
		// it also qualifies as splice_region_variant:
		if ((variant->chromEnd > exonEnd-3 && variant->chromStart < exonEnd &&
		     exonIx < pred->exonCount - 1) ||
		    (variant->chromEnd > exonStart && variant->chromStart < exonStart+3 &&
		     exonIx > 0))
		    {
		    struct gpFx *effect = gpFxNew(allele->sequence, pred->name,
						  splice_region_variant, nonCodingExon, lm);
		    setNCExonVals(effect, exonIx, txc->startInCdna);
		    slAddTail(&effectsList, effect);
		    }
		}
	    }
	}
    }
return effectsList;
}

void testPolyhOut()
{
struct polyhedron *ph;
AllocVar(ph);
ph->id = nextId();
ph->names[0] = cloneString("Jim");
ph->names[1] = cloneString("Kent");
ph->polygonCount = 2;
slAddTail(&ph->polygons, makeTriangle(0, 0, 0));
slAddTail(&ph->polygons, makeTriangle(0, 0, 100));
findBox(ph);
polyhedronCommaOut(ph, stdout);
printf("\n");
polyhedronFree(&ph);
}

static struct hash *htmlHeaderRead(char **pHtml, struct htmlCookie **pCookies)
/* Read in from second line through first blank line and
 * save in hash.  These lines are in the form name: value. */
{
struct hash *hash = hashNew(6);
for (;;)
    {
    char *line = htmlNextCrLfLine(pHtml);
    char *word;
    if (line == NULL)
	{
        warn("End of file in header");
	break;
	}
    word = nextWord(&line);
    if (word == NULL)
        break;
    line = skipLeadingSpaces(line);
    hashAdd(hash, word, cloneString(line));
    if (sameString(word, "Set-Cookie:"))
	{
	struct htmlCookie *cookie = parseCookie(line);
	if (cookie != NULL)
	    slAddTail(pCookies, cookie);
	}
    }
return hash;
}

struct repBundle *bundleReplicates(struct replicate **pRepList)
/* Bundle together replicates,  eating *pRepList in the process. */
{
struct replicate *rep, *nextRep;
struct repBundle *bundleList = NULL, *bundle;
struct dyString *hashName = dyStringNew(0);
struct hash *bundleHash = hashNew(0);

for (rep = *pRepList; rep != NULL; rep = nextRep)
    {
    nextRep = rep->next;	// Going to clobber ->next field 
    dyStringClear(hashName);
    struct slPair *tag;
    for (tag = rep->tagList; tag != NULL; tag = tag->next)
        {
	if (!sameString(tag->name, "replicate") && !sameString(tag->name, "fileName"))
	    dyStringPrintf(hashName, "%s=%s;", tag->name, (char *)(tag->val));
	}
    bundle = hashFindVal(bundleHash, hashName->string);
    if (bundle == NULL)
        {
	AllocVar(bundle);
	slAddHead(&bundleList, bundle);
	hashAddSaveName(bundleHash, hashName->string, bundle, &bundle->hashName);
	}
    slAddTail(&bundle->repList, rep);
    }
slReverse(&bundleList);

*pRepList = NULL;
return bundleList;
}

struct consWiggle *wigMafWiggles(char *db, struct trackDb *tdb)
/* get conservation wiggle table names and labels from trackDb setting,
   ignoring those where table doesn't exist */
{
char *fields[20];
int fieldCt;
int i;
char *wigTable, *wigLabel;
struct consWiggle *wig, *wigList = NULL;
char *setting = trackDbSetting(tdb, CONS_WIGGLE);
if (!setting)
    return NULL;
fieldCt = chopLine(cloneString(setting), fields);
for (i = 0; i < fieldCt; i += 3)
    {
    wigTable = fields[i];
    if (hTableExists(db, wigTable));
        {
        AllocVar(wig);
        wig->table = cloneString(wigTable);
        wigLabel = (i+1 == fieldCt ? DEFAULT_CONS_LABEL : fields[i+1]);
        wig->leftLabel = cloneString(wigLabel);
        wigLabel = (i+2 >= fieldCt ? wig->leftLabel : fields[i+2]);
        wig->uiLabel = cloneString(wigLabel);
        slAddTail(&wigList, wig);
        }
    }
return wigList;
}

/* make sure we have the whole range even if
 * there isn't a maf loaded in this region
 */
static struct mafAli *padOutAli(struct mafAli *list, char *database, 
    char *chrom, int start, int end)
{
if (list == NULL)
    {
    struct mafAli *ali = getRefAli(database, chrom, start, end);
    return ali;
    }

int aliStart = list->components->start;
if (start != aliStart)
    {
    struct mafAli *ali = getRefAli(database, chrom, start, aliStart);
    slAddHead(&list, ali);
    }

struct mafAli *last = list;
for(; last->next; last = last->next)
    ;

int aliEnd = last->components->start + last->components->size;
if (end != aliEnd)
    {
    struct mafAli *ali = getRefAli(database, chrom, aliEnd, end);
    slAddTail(&list, ali);
    }

return list;
}

void addOutKeys(struct hash *tableHash, struct joinerPair *routeList,
                struct tableJoiner **pTfList)
/* Add output keys to tableJoiners.  These are in table hash.
 * We may also have to add some fieldLess tables to the mix,
 * which will get appended to *pTfList, and added to the hash
 * if need be. */
{
    struct joinerPair *route;
    struct tableJoiner *tj;
    char fullName[256];

    for (route = routeList; route != NULL; route = route->next)
    {
        struct joinerDtf *a = route->a;
        safef(fullName, sizeof(fullName), "%s.%s", a->database, a->table);
        tj = hashFindVal(tableHash, fullName);
        if (tj == NULL)
        {
            AllocVar(tj);
            tj->database = a->database;
            tj->table = a->table;
            slAddTail(pTfList, tj);
            hashAdd(tableHash, fullName, tj);
        }
        if (!inKeysOut(tj, route))
            refAdd(&tj->keysOut, route);
    }
}

static void addDtfListToBundle(struct hash *hash, struct tableJoiner **pList,
                               struct joinerDtf *dtfList, boolean addField)
/* Add table to hash/list if haven't seen it already.  Optionally add
 * field to table. */
{
    char fullName[256];
    struct joinerDtf *dtf, *dupe;
    struct tableJoiner *tj;
    for (dtf = dtfList; dtf != NULL; dtf = dtf->next)
    {
        safef(fullName, sizeof(fullName), "%s.%s", dtf->database, dtf->table);
        tj = hashFindVal(hash, fullName);
        if (tj == NULL)
        {
            AllocVar(tj);
            hashAdd(hash, fullName, tj);
            tj->database = dtf->database;
            tj->table = dtf->table;
            slAddHead(pList, tj);
        }
        if (addField)
        {
            dupe = joinerDtfClone(dtf);
            slAddTail(&tj->fieldList, dupe);
        }
    }
}

static void jrRowExpand(struct joinedTables *joined,
                        struct joinedRow *jr, char **row,
                        int fieldOffset, int fieldCount, int keyOffset, int keyCount)
/* Add some more to row. */
{
    int i;
    struct slName **keys = jr->keys + keyOffset;
    struct slName **fields = jr->fields + fieldOffset;
    struct lm *lm = joined->lm;
    struct slName *name;

    if (fieldCount + fieldOffset > joined->fieldCount)
        internalErr();
    if (keyCount + keyOffset > joined->keyCount)
        internalErr();
    for (i=0; i<fieldCount; ++i)
    {
        name = lmSlName(lm, row[i]);
        slAddTail(&fields[i], name);
    }
    row += fieldCount;
    for (i=0; i<keyCount; ++i)
    {
        name = lmSlName(lm, row[i]);
        slAddHead(&keys[i], name);
    }
}

struct gdfGene *wormGetSomeGdfGeneList(char *baseName, int baseNameSize, struct wormGdfCache *cache)
/* Get all gdfGenes that start with a given name. */
{
int snIx;
int maxIx;
struct snof *snof;
FILE *f;
struct gdfGene *list = NULL, *el;

wormCacheSomeGdf(cache);
snof = cache->snof;
f = cache->file;
if (!snofFindFirstStartingWith(snof, baseName, baseNameSize, &snIx))
    return NULL;

maxIx = snofElementCount(snof);
for (;snIx < maxIx; ++snIx)
    {
    long offset;
    char *geneName;

    snofNameOffsetAtIx(snof, snIx, &geneName, &offset);
    if (strncmp(geneName, baseName, baseNameSize) != 0)
        break;
    fseek(f, offset, SEEK_SET);
    el = gdfReadOneGene(f);
    slAddTail(&list, el);
    }
slReverse(&list);
return list;
}

static struct rqlParse *rqlParseOr(struct tokenizer *tkz)
/* Parse out and or or. */
{
struct rqlParse *p = rqlParseAnd(tkz);
struct rqlParse *parent = NULL;
for (;;)
    {
    char *tok = tokenizerNext(tkz);
    if (tok == NULL || !sameString(tok, "or"))
        {
	tokenizerReuse(tkz);
	return p;
	}
    else
        {
	if (parent == NULL)
	    {
	    p = rqlParseCoerce(p, rqlTypeBoolean);
	    AllocVar(parent);
	    parent->op = rqlOpOr;
	    parent->type = rqlTypeBoolean;
	    parent->children = p;
	    p = parent;
	    }
	struct rqlParse *r = rqlParseCoerce(rqlParseAnd(tkz), rqlTypeBoolean);
	slAddTail(&parent->children, r);
	}
    }
}

struct genScanGene *bundleGenes(struct genScanFeature *gsfList)
/* Bundle together features into genes.   Cannibalizes gsfList. */
{
struct genScanFeature *gsf, *nextGsf;
struct genScanGene *geneList = NULL, *gene = NULL;

for (gsf = gsfList; gsf != NULL; gsf = nextGsf)
    {
    nextGsf = gsf->next;
    if (gene == NULL || gene->id != gsf->geneId)
        {
	AllocVar(gene);
	slAddHead(&geneList, gene);
	gene->id = gsf->geneId;
	gene->strand = gsf->strand;
	gene->start = gsf->start;
	gene->end = gsf->end;
	}
    slAddTail(&gene->featureList, gsf);
    if (gsf->start < gene->start) gene->start = gsf->start;
    if (gsf->end > gene->end) gene->end = gsf->end;
    }
slReverse(&geneList);
return geneList;
}

void testPolyOut()
{
struct polygon *tri;
struct point *pt;
int i;

AllocVar(tri);
tri->id = 21060;
tri->pointCount = 4;
for (i=0; i<tri->pointCount; ++i)
    {
    AllocVar(pt);
    sprintf(pt->acc, "point-%c", 'A'+i);
    pt->x = 100 + i;
    pt->y = 200 + i;
    pt->z = 300 + i;
    slAddTail(&tri->points, pt);
    }
AllocArray(tri->persp,tri->pointCount);
for (i=0; i<tri->pointCount; ++i)
    {
    tri->persp[i].x = 50+i;
    tri->persp[i].y = 100+i;
    }
polygonCommaOut(tri, stdout);
printf("\n");
}

void setupTissueLibraryCache(struct sqlConnection *conn)
/* Hash all of the library and organism ids for this organism from
   the mrna table. The key to the hash is the accession and the
   value is a list of integers where the first is the library and
   the second is the tissue. */
{
struct sqlResult *sr = NULL;
char query[256];
struct slInt *tissue = NULL, *library = NULL;
char **row = NULL;
int i = 0;
tissLibHash = newHash(12);

/* Now load up all of the accessions for this organism. */
for(i = 0; i < numDbTables; i++)
    {
    sqlSafef(query, sizeof(query), "select library, tissue, acc from %s inner join gbCdnaInfo on qName = acc",
	  dbTables[i]);
    warn("%s", query);
    sr = sqlGetResult(conn, query);
    while((row = sqlNextRow(sr)) != NULL)
	{
	library = newSlInt(sqlSigned(row[0]));
	tissue = newSlInt(sqlSigned(row[1]));
	slAddTail(&library, tissue);
	hashAdd(tissLibHash, row[2], library);
	}
    sqlFreeResult(&sr);
    }

}

static struct gpFx *gpFxInIntron(struct variant *variant, struct txCoords *txc, int intronIx,
				 struct genePred *pred, boolean predIsNmd, char *altAllele,
				 struct lm *lm)
// Annotate a variant that overlaps an intron (and possibly splice region)
//#*** TODO: watch out for "introns" that are actually indels between tx seq and ref genome!
{
struct gpFx *effectsList = NULL;
boolean minusStrand = (pred->strand[0] == '-');
// If on - strand, flip intron number back to + strand for getting intron coords:
int intronPos = minusStrand ? (pred->exonCount - intronIx - 2) : intronIx;
int intronStart = pred->exonEnds[intronPos];
int intronEnd = pred->exonStarts[intronPos+1];
if (variant->chromEnd > intronStart && variant->chromStart < intronEnd)
    {
    enum soTerm soNumber = intron_variant;
    if (variant->chromEnd > intronStart && variant->chromStart < intronStart+2)
	// Within 2 bases of intron start(/end for '-'):
	soNumber = minusStrand ? splice_acceptor_variant : splice_donor_variant;
    if (variant->chromEnd > intronEnd-2 && variant->chromStart < intronEnd)
	// Within 2 bases of intron end(/start for '-'):
	soNumber = minusStrand ? splice_donor_variant : splice_acceptor_variant;
    else if ((variant->chromEnd > intronStart+3 && variant->chromStart < intronStart+8) ||
	     (variant->chromEnd > intronEnd-8 && variant->chromStart < intronEnd+3))
	// Within 3 to 8 bases of intron start or end:
	soNumber = splice_region_variant;
    if (predIsNmd)
	// This transcript is already subject to nonsense-mediated decay, so the effect
	// is probably not a big deal:
	soNumber = NMD_transcript_variant;
    struct gpFx *effects = gpFxNew(altAllele, pred->name, soNumber, intron, lm);
    effects->details.intron.intronNumber = intronIx;
    slAddTail(&effectsList, effects);
    }
return effectsList;
}

void pushPosString(struct speciesInfo *si)
{
flushPosString(si);

struct slName *newName = newSlName(si->posString);
slAddTail(&si->posStrings, newName);

freez(&si->posString);
}