void xpParseEndTag(struct xp *xp, char *tagName)
/* Call this after have seen </.  It will parse through
 * > and make sure that the tagName matches. */
{
struct dyString *dy = xp->endTag;
char c;

dyStringClear(dy);

/* Skip leading space. */
for (;;)
    {
    if ((c = xpGetChar(xp)) == 0)
	xpUnexpectedEof(xp);
    if (isspace(c))
	{
	if (c == '\n')
	    ++xp->lineIx;
	}
    else
	break;
    }

/* Read end tag. */
for (;;)
    {
    dyStringAppendC(dy, c);
    if ((c = xpGetChar(xp)) == 0)
	xpUnexpectedEof(xp);
    if (isspace(c))
	{
	if (c == '\n')
	    ++xp->lineIx;
	break;
	}
    if (c == '>')
	break;
    }

/* Skip until '>' */
while (c != '>')
    {
    dyStringAppendC(dy, c);
    if ((c = xpGetChar(xp)) == 0)
	xpUnexpectedEof(xp);
    if (isspace(c))
	{
	if (c == '\n')
	    ++xp->lineIx;
	}
    else if (c != '>')
	xpError(xp, "Unexpected characters past first word in /%s tag", dy->string);
    }

if (!sameString(dy->string, tagName))
    xpError(xp, "Mismatch between start tag %s and end tag %s",  tagName, dy->string);
}

struct dnaSeq *faReadWithDash(FILE *f)
/* Read in a fasta sequence that might include dashes. */
{
int c;
struct dnaSeq *seq;
struct dyString *dy;
char *line, *name;

/* Get first > */
for (;;)
    {
    if ((c = fgetc(f)) == -1)
	return NULL;
    if (c == '>')
        break;
    }
/* Get line into dy and then first word into name*/
dy = dyStringNew(0);
for (;;)
    {
    if ((c = fgetc(f)) == -1)
        errAbort("Unexpected end of file");
    if (c == '\n')
        break;
    dyStringAppendC(dy, c);
    }
line = dy->string;
name = cloneString(nextWord(&line));

/* Get up until next '>' into dy */
dyStringClear(dy);
for (;;)
    {
    if ((c = fgetc(f)) == -1)
	break;
    if (isdigit(c) || isspace(c))
        continue;
    if (c == '>')
	{
        ungetc(c, f);
	break;
	}
    dyStringAppendC(dy, c);
    }

/* Create seq. */
AllocVar(seq);
seq->name = name;
seq->dna = cloneString(dy->string);
seq->size = dy->stringSize;
verbose(2, "seq %s, %d bases\n", seq->name, seq->size);
dyStringFree(&dy);
return seq;
}

char *vcfGetSlashSepAllelesFromWords(char **words, struct dyString *dy,
				     boolean *retSkippedFirstBase)
/* Overwrite dy with a /-separated allele string from VCF words,
 * skipping the extra initial base that VCF requires for indel alleles if necessary.
 * Return dy->string for convenience. */
{
dyStringClear(dy);
// VCF reference allele gets its own column:
char *refAllele = words[3];
char *altAlleles = words[4];
// First determine whether there is an extra initial base that we need to skip:
boolean allStartSame = TRUE;
char *p;
while ((p = strchr(altAlleles, ',')) != NULL)
    {
    if (altAlleles[0] != refAllele[0])
	allStartSame = FALSE;
    altAlleles = p+1;
    }
if (altAlleles[0] != refAllele[0])
    allStartSame = FALSE;
int offset = allStartSame ? 1 : 0;
if (refAllele[offset] == '\0')
    dyStringAppendC(dy, '-');
else
    dyStringAppend(dy, refAllele+offset);
// VCF alternate alleles are comma-separated, make them /-separated:
altAlleles = words[4];
if (isNotEmpty(altAlleles) && differentString(altAlleles, "."))
    {
    // Now construct the string:
    while ((p = strchr(altAlleles, ',')) != NULL)
	{
	dyStringAppendC(dy, '/');
	int len = p - altAlleles - offset;
	if (len == 0)
	    dyStringAppendC(dy, '-');
	else
	    dyStringAppendN(dy, altAlleles+offset, len);
	altAlleles = p+1;
	}
    dyStringAppendC(dy, '/');
    int len = strlen(altAlleles) - offset;
    if (len == 0)
	dyStringAppendC(dy, '-');
    else
	dyStringAppendN(dy, altAlleles+offset, len);
    }
if (retSkippedFirstBase)
    *retSkippedFirstBase = offset;
return dy->string;
}

void dyStringAppendEscapeQuotes(struct dyString *dy, char *string,
	char quot, char esc)
/* Append escaped-for-quotation version of string to dy. */
{
char c;
char *s = string;
while ((c = *s++) != 0)
     {
     if (c == quot)
         dyStringAppendC(dy, esc);
     dyStringAppendC(dy, c);
     }
}

static void appendCgiVar(struct dyString *dy, char *name, char *value)
/* Append cgiVar with cgi-encoded value to dy. */
{
char *enc = NULL;
if (value == NULL)
    value = "";
enc = cgiEncode(value);
if (dy->stringSize != 0)
    dyStringAppendC(dy, '&');
dyStringAppend(dy, name);
dyStringAppendC(dy, '=');
dyStringAppend(dy, enc);
freez(&enc);
}

void dyStringQuoteString(struct dyString *dy, char quotChar, char *text)
/* Append quotChar-quoted text (with any internal occurrences of quotChar
 * \-escaped) onto end of dy. */
{
char c;

dyStringAppendC(dy, quotChar);
while ((c = *text++) != 0)
    {
    if (c == quotChar)
        dyStringAppendC(dy, '\\');
    dyStringAppendC(dy, c);
    }
dyStringAppendC(dy, quotChar);
}

static void dyStringAppendQuoted(struct dyString *dy, char *s)
/* Append s to dyString, surrounding with quotes and escaping
 * internal quotes. */
{
char c;
dyStringAppendC(dy, '"');
while ((c = *s++) != 0)
    {
    if (c == '"')
       dyStringAppendC(dy, '\\');
    dyStringAppendC(dy, c);
    }
dyStringAppendC(dy, '"');
dyStringAppendC(dy, ' ');
}

static void visiSearcherWeedResults(struct visiSearcher *searcher,
	struct sqlConnection *conn)
/* Get rid of images that are just partial matches, and also
 * images that are private.  This leaks a little memory - the
 * matches that are weeded out.*/
{
struct visiMatch *newList = NULL, *match, *next, key;
int wordCount = searcher->wordCount;
struct dyString *query = dyStringNew(0);
struct sqlResult *sr;
char **row;
int passCount = 0;

/* Construct query to fetch all non-private imageId's in matchList. */
dyStringAppend(query, "select image.id from image,submissionSet "
                      "where submissionSet.privateUser = 0 "
		      "and submissionSet.id = image.submissionSet "
		      "and image.id in (");
for (match = searcher->matchList; match != NULL; match = next)
    {
    next = match->next;
    if (bitCountRange(match->wordBits, 0, wordCount) == wordCount)
	{
	if (passCount != 0)
	    dyStringAppendC(query, ',');
	dyStringPrintf(query, "%d", match->imageId);
	++passCount;
	}
    }
dyStringAppendC(query, ')');

/* Execute query, and put corresponding images on newList. */
if (passCount > 0)
    {
    sr = sqlGetResult(conn, query->string);
    while ((row = sqlNextRow(sr)) != NULL)
	{
	key.imageId = sqlUnsigned(row[0]);
	match = rbTreeFind(searcher->tree, &key);
	if (match == NULL)
	    internalErr();
	slAddHead(&newList, match);
	}
    slReverse(&newList);
    }
searcher->matchList = newList;
dyStringFree(&query);
}

static void makeOrderedCommaFieldList(struct slName *orderList,
                                      struct joinerDtf *dtfList, struct dyString *dy)
/* Given list in order, and a subset of fields to use, make
 * a comma separated list of that subset in order. */
{
    struct joinerDtf *dtf;
    struct slName *order;
    boolean first = TRUE;
    struct hash *dtfHash = newHash(8);

    /* Build up hash of field names. */
    for (dtf = dtfList; dtf != NULL; dtf = dtf->next)
        hashAdd(dtfHash, dtf->field, NULL);
    for (order = orderList; order != NULL; order = order->next)
    {
        if (hashLookup(dtfHash, order->name))
        {
            if (first)
                first = FALSE;
            else
                dyStringAppendC(dy, ',');
            dyStringAppend(dy, order->name);
        }
    }
    hashFree(&dtfHash);
}

static void errCatchWarnHandler(char *format, va_list args)
/* Write an error to top of errCatchStack. */
{
struct errCatch **pErrCatchStack = getStack(), *errCatchStack = *pErrCatchStack;
dyStringVaPrintf(errCatchStack->message, format, args);
dyStringAppendC(errCatchStack->message, '\n');
}

char *hashToRaString(struct hash *hash)
/* Convert hash to string in ra format. */
{
struct hashEl *el, *list = hashElListHash(hash);
struct dyString *dy = dyStringNew(0);
slSort(&list, hashElCmp);
for (el = list; el != NULL; el = el->next)
   {
   dyStringAppend(dy, el->name);
   dyStringAppendC(dy, ' ');
   dyStringAppend(dy, el->val);
   dyStringAppendC(dy, '\n');
   }
hashElFreeList(&list);
return dyStringCannibalize(&dy);
}

static void hubConnectRemakeTrackHubVar(struct cart *cart)
/* Remake trackHub cart variable if need be from various check box vars. */
{
if (cartVarExists(cart, hgHubConnectRemakeTrackHub))
    {
    struct slPair *hubVarList = cartVarsWithPrefix(cart, hgHubConnectHubVarPrefix);
    int prefixLength = strlen(hgHubConnectHubVarPrefix);
    struct dyString *trackHubs = dyStringNew(0);
    struct slPair *hubVar;
    boolean firstOne = TRUE;
    for (hubVar = hubVarList; hubVar != NULL; hubVar = hubVar->next)
        {
	if (cartBoolean(cart, hubVar->name))
	    {
	    if (firstOne)
		firstOne = FALSE;
	    else
		dyStringAppendC(trackHubs, ' ');
	    dyStringAppend(trackHubs, hubVar->name + prefixLength);
	    }
	}
    slPairFreeList(&hubVarList);
    cartSetString(cart, hubConnectTrackHubsVarName, trackHubs->string);
    dyStringFree(&trackHubs);
    cartRemove(cart, hgHubConnectRemakeTrackHub);
    }
}

char *dlListFragWords(struct dlNode *head)
/* Return string containing all words in list pointed to by head. */
{
struct dyString *dy = dyStringNew(0);
dyStringAppendC(dy, '{');
struct dlNode *node;
for (node = head; !dlEnd(node); node = node->next)
    {
    if (node != head)
       dyStringAppendC(dy, ' ');
    struct wordInfo *info = node->val;
    dyStringAppend(dy, info->word);
    }
dyStringAppendC(dy, '}');
return dyStringCannibalize(&dy);
}

boolean raSkipLeadingEmptyLines(struct lineFile *lf, struct dyString *dy)
/* Skip leading empty lines and comments.  Returns FALSE at end of file.
 * Together with raNextTagVal you can construct your own raNextRecord....
 * If dy parameter is non-null, then the text parsed gets placed into dy. */
{
char *line;
/* Skip leading empty lines and comments. */
if (dy)
    dyStringClear(dy);
for (;;)
    {
    if (!lineFileNext(lf, &line, NULL))
       return FALSE;
    char *tag = skipLeadingSpaces(line);
    if (tag[0] == 0 || tag[0] == '#')
       {
       if (dy)
	   {
	   dyStringAppend(dy, line);
	   dyStringAppendC(dy, '\n');
	   }
	}
    else
       break;
    }
lineFileReuse(lf);
return TRUE;
}

void xpTextUntil(struct xp *xp, char *endPattern)
/* Stuff xp->text with everything up to endPattern. */
{
int endSize = strlen(endPattern);
int endPos = 0;
char c;
struct dyString *dy = xp->stack->text;
for (;;)
    {
    if ((c = xpGetChar(xp)) == 0)
	xpUnexpectedEof(xp);
    if (c == endPattern[endPos])
        {
	endPos += 1;
	if (endPos == endSize)
	    return;
	}
    else
        {
	if (endPos > 0)
	    dyStringAppendN(dy, endPattern, endPos);
	dyStringAppendC(dy, c);
	endPos = 0;
	}
    }
}

void makeTableDescriptions(char *database, char *asFile)
/* makeTableDescriptions - Add table descriptions to database.. */
{
struct sqlConnection *conn = sqlConnect(database);
struct lineFile *lf = lineFileOpen(asFile, TRUE);
FILE *f = hgCreateTabFile(".", "tableDescriptions");
/* Open a tab file with name corresponding to tableName in tmpDir. */
char *line;

/* struct asObject *asList = */ asParseFile(asFile);	/* Just to check syntax */

if (sqlTableExists(conn, "chromInfo"))
    errAbort("%s looks like a genome database, has chromInfo, aborting", 
    	database);

sqlRemakeTable(conn, "tableDescriptions",
   "NOSQLINJ CREATE TABLE tableDescriptions (\n"
   "  tableName varchar(255) not null,\n"
   "  autoSqlDef longblob not null,\n"
   "  gbdAnchor varchar(255) not null,\n"
   "  PRIMARY KEY(tableName(32))\n"
   ")" );

while (lineFileNextReal(lf, &line))
    {
    if (startsWith("table", line))
        {
	struct dyString *as = dyStringNew(0);
	char *name = trimSpaces(line + 6);	/* Skip over table. */
	char *escaped = NULL;

	fprintf(f, "%s\t", name);

	/* Putting lines into as. */
	for (;;)
	    {
	    char *s;
	    dyStringAppend(as, line);
	    dyStringAppendC(as, '\n');
	    s = skipLeadingSpaces(line);
	    if (s[0] == ')')
	        break;
	    if (!lineFileNext(lf, &line, NULL))
	        errAbort("Unexpected end of file, missing closing paren in %s",
			lf->fileName);
	    }
	escaped = needMem(2*as->stringSize+1);
	fprintf(f, "%s\t", sqlEscapeTabFileString2(escaped, as->string));
	fprintf(f, "\n");

	freez(&escaped);
	dyStringFree(&as);
	}
    else
        errAbort("Expecting table line %d of %s", lf->lineIx, lf->fileName);
    }
hgLoadTabFile(conn, ".", "tableDescriptions", &f);
}

char *associationCellVal(struct column *col, struct genePos *gp, 
	struct sqlConnection *conn)
/* Make comma separated list of matches to association table. */
{
char query[1024];
struct sqlResult *sr;
char **row;
boolean gotOne = FALSE;
struct dyString *dy = newDyString(512);
char *result = NULL;
char *key = (col->protKey 
    ? (kgVersion == KG_III ? lookupProtein(conn, gp->name) : gp->protein)
    : gp->name);
struct hash *uniqHash = NULL;

if (col->weedDupes) uniqHash = newHash(8);
safef(query, sizeof(query), col->queryOne, key);
sr = sqlGetResult(conn, query);
while ((row = sqlNextRow(sr)) != NULL)
    {
    char *s = row[0];
    boolean needQuote;
    if (uniqHash != NULL)
        {
	if (hashLookup(uniqHash, s))
	    continue;
	else
	    hashAdd(uniqHash, s, NULL);
	}
    needQuote = hasWhiteSpace(s);
    if (needQuote)
    	dyStringAppendC(dy, '\'');
    dyStringAppend(dy, s);
    if (needQuote)
    	dyStringAppendC(dy, '\'');
    dyStringAppend(dy, ",");
    gotOne = TRUE;
    }
sqlFreeResult(&sr);
if (gotOne)
    result = cloneString(dy->string);
dyStringFree(&dy);
freeHash(&uniqHash);
return result;
}

void dyStringAddList(struct dyString *dy, char *s)
/* Add s to end of string.  If string is non-empty
 * first add comma.  This make it relatively easy to
 * build up comma-separates lists. */
{
    if (dy->stringSize > 0)
        dyStringAppendC(dy, ',');
    dyStringAppend(dy, s);
}

static struct vcfFile *vcfFileHeaderFromLineFile(struct lineFile *lf, int maxErr)
/* Parse a VCF file into a vcfFile object.  If maxErr not zero, then
 * continue to parse until this number of error have been reached.  A maxErr
 * less than zero does not stop and reports all errors.
 * Set maxErr to VCF_IGNORE_ERRS for silence */
{
initVcfSpecInfoDefs();
initVcfSpecGtFormatDefs();
if (lf == NULL)
    return NULL;
struct vcfFile *vcff = vcfFileNew();
vcff->lf = lf;
vcff->fileOrUrl = vcfFileCloneStr(vcff, lf->fileName);
vcff->maxErr = (maxErr < 0) ? INT_MAX : maxErr;

struct dyString *dyHeader = dyStringNew(1024);
char *line = NULL;
// First, metadata lines beginning with "##":
while (lineFileNext(lf, &line, NULL) && startsWith("##", line))
    {
    dyStringAppend(dyHeader, line);
    dyStringAppendC(dyHeader, '\n');
    parseMetadataLine(vcff, line);
    }
slReverse(&(vcff->infoDefs));
slReverse(&(vcff->filterDefs));
slReverse(&(vcff->gtFormatDefs));
// Did we get the bare minimum VCF header with supported version?
if (vcff->majorVersion == 0)
    vcfFileErr(vcff, "missing ##fileformat= header line?  Assuming 4.1.");
if ((vcff->majorVersion != 4 || (vcff->minorVersion != 0 && vcff->minorVersion != 1)) &&
    (vcff->majorVersion != 3))
    vcfFileErr(vcff, "VCFv%d.%d not supported -- only v3.*, v4.0 or v4.1",
	       vcff->majorVersion, vcff->minorVersion);
// Next, one header line beginning with single "#" that names the columns:
if (line == NULL)
    // EOF after metadata
    return vcff;
dyStringAppend(dyHeader, line);
dyStringAppendC(dyHeader, '\n');
parseColumnHeaderRow(vcff, line);
vcff->headerString = dyStringCannibalize(&dyHeader);
return vcff;
}

boolean raNextTagVal(struct lineFile *lf, char **retTag, char **retVal, struct dyString *dyRecord)
// Read next line.  Return FALSE at end of file or blank line.  Otherwise fill in
// *retTag and *retVal and return TRUE.  If dy parameter is non-null, then the text parsed
// gets appended to dy. Continuation lines in RA file will be joined to produce tag and val,
// but dy will be filled with the unedited multiple lines containing the continuation chars.
// NOTE: retTag & retVal, if returned, point to static mem which will be overwritten on next call!
{
*retTag = NULL;
*retVal = NULL;

char *line, *raw = NULL;
int lineLen,rawLen;

// Don't bother with raw if it isn't used.
char **pRaw    = NULL;
int   *pRawLen = NULL;
if (dyRecord != NULL)
    {
    pRaw    = &raw;
    pRawLen = &rawLen;
    }

while (lineFileNextFull(lf, &line, &lineLen, pRaw, pRawLen)) // Joins continuation lines
    {
    char *clippedText = skipLeadingSpaces(line);
    if (*clippedText == 0)
        {
        if (dyRecord)
            lineFileReuse(lf);   // Just so don't loose leading space in dy.
        return FALSE;
        }

    // Append whatever line was read from file.
    if (dyRecord)
       {
        if (raw != NULL)
            dyStringAppendN(dyRecord, raw, rawLen);
       else
            dyStringAppendN(dyRecord, line, lineLen);
       dyStringAppendC(dyRecord,'\n');
       }

    // Skip comments
    if (*clippedText == '#')
       {
       if (startsWith("#EOF", clippedText))
           return FALSE;
       else
           continue;
       }
    *retTag = nextWord(&line);
    *retVal = trimSpaces(line);
    return TRUE;
    }
return FALSE;
}