void
ngram_search::xor_block(string_buffer& sb, unsigned int val)
{
  for (string_buffer::iterator i = sb.begin(); i != sb.end(); ++i) {
    (*i) ^= val;
  }
}

void
append_uint32(string_buffer& buf, uint32_t v)
{
  char *const wp = buf.make_space(64);
  const int len = snprintf(wp, 64, "%lu", static_cast<unsigned long>(v));
  if (len > 0) {
    buf.space_wrote(len);
  }
}

template <typename T> void
ngram_search::serialize_nat(string_buffer& sb, T value)
{
  do {
    const unsigned char b7 = value & 0x7fU;
    value >>= 7;
    const unsigned char cf = (value != 0) ? 0x80U : 0;
    const unsigned char b = b7 | cf;
    sb.append(reinterpret_cast<const char *>(&b), 1);
  } while (value != 0);
}

 append_string_buffer(string_buffer& b) : b(b) { b.reserve(100); }

size_t
hstcpcli::request_buf_append(const char *start, const char *finish)
{
/*
  if (num_req_sent > 0 || num_req_rcvd > 0) {
*/
  if (num_req_rcvd > 0) {
    close();
    set_error(-1, "request_buf_append: protocol out of sync");
    return 0;
  }
  const char *nl = start;
  size_t num_req = 0;
  while ((nl = memchr_char(nl, '\n', finish - nl))) {
    if (nl == finish)
      break;
    num_req++;
    nl++;
  }
  num_req++;
  writebuf.append(start, finish);
  if (*(finish - 1) != '\n')
    writebuf.append_literal("\n");
  num_req_bufd += num_req;
  return num_req;
}

int
hstcpcli::request_send()
{
  if (error_code < 0) {
    return error_code;
  }
  clear_error();
  if (fd.get() < 0) {
    close();
    return set_error(-1, "write: closed");
  }
  if (num_req_bufd == 0 || num_req_sent > 0 || num_req_rcvd > 0) {
    close();
    return set_error(-1, "request_send: protocol out of sync");
  }
  const size_t wrlen = writebuf.size();
  const ssize_t r = send(fd.get(), writebuf.begin(), wrlen, MSG_NOSIGNAL);
  if (r <= 0) {
    close();
    return set_error(-1, r < 0 ? "write: failed" : "write: eof");
  }
  writebuf.erase_front(r);
  if (static_cast<size_t>(r) != wrlen) {
    close();
    return set_error(-1, "write: incomplete");
  }
  num_req_sent = num_req_bufd;
  num_req_bufd = 0;
  DBG(fprintf(stderr, "REQSEND 0\n"));
  return 0;
}

const string_ref *
hstcpcli::get_next_row()
{
  if (num_flds == 0) {
    DBG(fprintf(stderr, "GNR NF 0\n"));
    return 0;
  }
  if (flds.size() < num_flds) {
    flds.resize(num_flds);
  }
  char *start = readbuf.begin() + cur_row_offset;
  char *const finish = readbuf.begin() + response_end_offset - 1;
  if (start >= finish) { /* start[0] == nl */
    DBG(fprintf(stderr, "GNR FIN 0 %p %p\n", start, finish));
    return 0;
  }
  for (size_t i = 0; i < num_flds; ++i) {
    skip_one(start, finish);
    char *const fld_begin = start;
    read_token(start, finish);
    char *const fld_end = start;
    char *wp = fld_begin;
    if (is_null_expression(fld_begin, fld_end)) {
      /* null */
      flds[i] = string_ref();
    } else {
      unescape_string(wp, fld_begin, fld_end); /* in-place */
      flds[i] = string_ref(fld_begin, wp);
    }
  }
  cur_row_offset = start - readbuf.begin();
  return &flds[0];
}

void
hstcpcli::request_reset()
{
  if (num_req_bufd) {
    writebuf.erase_front(writebuf.size());
    num_req_bufd = 0;
  }
}

int
hstcpcli::read_more()
{
  const size_t block_size = 4096; // FIXME
  char *const wp = readbuf.make_space(block_size);
  const ssize_t rlen = read(fd.get(), wp, block_size);
  if (rlen <= 0) {
    if (rlen < 0) {
      error_str = "read: failed";
    } else {
      error_str = "read: eof";
    }
    return rlen;
  }
  readbuf.space_wrote(rlen);
  return rlen;
}

 dtoken read_id() {
     while (!eos() && m_curr_char != '(' && m_curr_char != ')' && 
            m_curr_char != '#' && m_curr_char != ',' && (m_parsing_domains || m_curr_char != '.') && 
            m_curr_char != ':' && m_curr_char != '=' && !iswspace(m_curr_char) ) {
         save_and_next();
     }
     return m_reserved_symbols.string2dtoken(m_buffer.c_str());
 }

 // read an id of the form '|'.*'|' 
 dtoken read_bid() {
     while (!eos() && m_curr_char != '|') {
         save_and_next();
     }
     if (m_curr_char == '|') {
         next();
     }
     return m_reserved_symbols.string2dtoken(m_buffer.c_str());
 }

 bool lookahead_newline() {
     while (m_curr_char == ' ') {
         save_and_next();
     }
     if (m_curr_char == '\n') {
         next();
         m_line++;
         m_buffer.reset();
         return true;
     }
     if (m_curr_char == '#') {
         m_buffer.reset();
         m_prev_char = 0;
         read_comment();
         return true;
     }
     return false;
 }

void
hstcpcli::request_buf_auth(const char *secret, const char *typ)
{
  if (num_req_sent > 0 || num_req_rcvd > 0) {
    close();
    set_error(-1, "request_buf_auth: protocol out of sync");
    return;
  }
  if (typ == 0) {
    typ = "1";
  }
  const string_ref typ_ref(typ, strlen(typ));
  const string_ref secret_ref(secret, strlen(secret));
  writebuf.append_literal("A\t");
  writebuf.append(typ_ref.begin(), typ_ref.end());
  writebuf.append_literal("\t");
  writebuf.append(secret_ref.begin(), secret_ref.end());
  writebuf.append_literal("\n");
  ++num_req_bufd;
}

int
hstcpcli::read_more()
{
  const size_t block_size = 4096; // FIXME
  char *const wp = readbuf.make_space(block_size);
  int rlen;
  errno = 0;
  while ((rlen = read(fd.get(), wp, block_size)) <= 0) {
    errno_buf = errno;
    if (rlen < 0) {
      if (errno == EINTR || errno == EAGAIN)
      {
        errno = 0;
        continue;
      }
      error_str = String("read: failed", &my_charset_bin);
    } else {
      error_str = String("read: eof", &my_charset_bin);
    }
    return rlen;
  }
  readbuf.space_wrote(rlen);
  return rlen;
}

int
hstcpcli::get_result(hstresult& result)
{
/*
  readbuf.swap(result.readbuf);
*/
  char *const wp = result.readbuf.make_space(response_end_offset);
  memcpy(wp, readbuf.begin(), response_end_offset);
  result.readbuf.space_wrote(response_end_offset);
  result.response_end_offset = response_end_offset;
  result.num_flds = num_flds;
  result.cur_row_offset = cur_row_offset;
  if (result.flds.max_element < num_flds)
  {
    if (allocate_dynamic(&result.flds, num_flds))
      return set_error(-1, "out of memory");
  }
  result.flds.elements = num_flds;
  return 0;
}

 const char * get_token_data() const {
     return m_buffer.c_str();
 }

 dtoken next_token() {
     for(;;) {
         if (eos()) {
             return TK_EOS;
         }
         
         m_buffer.reset();
         switch (m_curr_char) {
         case '#': // comment
             read_comment();
             break;
         case '\n':
             next();
             m_line++;
             return TK_NEWLINE;
         case '\\':
             // here we ignore a newline if it is preceded by a backslash.
             // We need to take care, since anywhere else backshlash is used
             // as a regular character
             next();
             save_char('\\');
             if (lookahead_newline()) {
                 break;
             }
             return read_id();
         case '(':
             m_tok_pos = m_pos;
             next();
             return TK_LP;
         case ')':
             m_tok_pos = m_pos;
             next();
             return TK_RP;               
         case ',':
             m_tok_pos = m_pos;
             next();
             return TK_COMMA; 
         case '=':
             m_tok_pos = m_pos;
             next();
             return TK_EQ; 
         case '!':
             m_tok_pos = m_pos;
             next();
             if(m_curr_char == '=') {
                 next();
                 return TK_NEQ;
             }
             return TK_NEG; 
         case ':':
             m_tok_pos = m_pos;
             next();
             if (m_curr_char == '-') {
                 next();
                 return TK_LEFT_ARROW;
             }
             return TK_COLON;
         case '\"':
             return read_string();
         case '|':
             next();
             return read_bid();
         default:
             if (iswspace(m_curr_char)) {
                 next();
                 break;
             }
             else if (iswdigit(m_curr_char)) {
                 m_tok_pos = m_pos;
                 save_and_next();
                 return read_num();
             }
             else {
                 char old = m_curr_char;
                 m_tok_pos = m_pos;
                 save_and_next();
                 if (old == '-' && iswdigit(m_curr_char)) {
                     return read_num();
                 }
                 else {
                     return read_id();
                 }
             }
         }
     }
 }

void
hstcpcli::request_buf_exec_generic(size_t pst_id, const string_ref& op,
  const string_ref *kvs, size_t kvslen, uint32_t limit, uint32_t skip,
  const string_ref& mod_op, const string_ref *mvs, size_t mvslen,
  const hstcpcli_filter *fils, size_t filslen, int invalues_keypart,
  const string_ref *invalues, size_t invalueslen)
{
  if (num_req_sent > 0 || num_req_rcvd > 0) {
    close();
    set_error(-1, "request_buf_exec_generic: protocol out of sync");
    return;
  }
  append_uint32(writebuf, pst_id); // FIXME size_t ?
  writebuf.append_literal("\t");
  writebuf.append(op.begin(), op.end());
  writebuf.append_literal("\t");
  append_uint32(writebuf, kvslen); // FIXME size_t ?
  for (size_t i = 0; i < kvslen; ++i) {
    const string_ref& kv = kvs[i];
    append_delim_value(writebuf, kv.begin(), kv.end());
  }
  if (limit != 0 || skip != 0 || invalues_keypart >= 0 ||
    mod_op.size() != 0 || filslen != 0) {
    /* has more option */
    writebuf.append_literal("\t");
    append_uint32(writebuf, limit); // FIXME size_t ?
    if (skip != 0 || invalues_keypart >= 0 ||
      mod_op.size() != 0 || filslen != 0) {
      writebuf.append_literal("\t");
      append_uint32(writebuf, skip); // FIXME size_t ?
    }
    if (invalues_keypart >= 0) {
      writebuf.append_literal("\[email protected]\t");
      append_uint32(writebuf, invalues_keypart);
      writebuf.append_literal("\t");
      append_uint32(writebuf, invalueslen);
      for (size_t i = 0; i < invalueslen; ++i) {
	const string_ref& s = invalues[i];
	append_delim_value(writebuf, s.begin(), s.end());
      }
    }
    for (size_t i = 0; i < filslen; ++i) {
      const hstcpcli_filter& f = fils[i];
      writebuf.append_literal("\t");
      writebuf.append(f.filter_type.begin(), f.filter_type.end());
      writebuf.append_literal("\t");
      writebuf.append(f.op.begin(), f.op.end());
      writebuf.append_literal("\t");
      append_uint32(writebuf, f.ff_offset);
      append_delim_value(writebuf, f.val.begin(), f.val.end());
    }
    if (mod_op.size() != 0) {
      writebuf.append_literal("\t");
      writebuf.append(mod_op.begin(), mod_op.end());
      for (size_t i = 0; i < mvslen; ++i) {
	const string_ref& mv = mvs[i];
	append_delim_value(writebuf, mv.begin(), mv.end());
      }
    }
  }
  writebuf.append_literal("\n");
  ++num_req_bufd;
}

void
hstcpcli::request_buf_open_index(size_t pst_id, const char *dbn,
  const char *tbl, const char *idx, const char *retflds, const char *filflds)
{
  if (num_req_sent > 0 || num_req_rcvd > 0) {
    close();
    set_error(-1, "request_buf_open_index: protocol out of sync");
    return;
  }
  const string_ref dbn_ref(dbn, strlen(dbn));
  const string_ref tbl_ref(tbl, strlen(tbl));
  const string_ref idx_ref(idx, strlen(idx));
  const string_ref rfs_ref(retflds, strlen(retflds));
  writebuf.append_literal("P\t");
  append_uint32(writebuf, pst_id); // FIXME size_t ?
  writebuf.append_literal("\t");
  writebuf.append(dbn_ref.begin(), dbn_ref.end());
  writebuf.append_literal("\t");
  writebuf.append(tbl_ref.begin(), tbl_ref.end());
  writebuf.append_literal("\t");
  writebuf.append(idx_ref.begin(), idx_ref.end());
  writebuf.append_literal("\t");
  writebuf.append(rfs_ref.begin(), rfs_ref.end());
  if (filflds != 0) {
    const string_ref fls_ref(filflds, strlen(filflds));
    writebuf.append_literal("\t");
    writebuf.append(fls_ref.begin(), fls_ref.end());
  }
  writebuf.append_literal("\n");
  ++num_req_bufd;
}