void c4_HandlerSeq::Prepare(const t4_byte **ptr_, bool selfDesc_)
{
    if (ptr_ != 0) {
        d4_dbgdef(t4_i32 sias =)c4_Column::PullValue(*ptr_);
        d4_assert(sias == 0); // not yet

        if (selfDesc_) {
            t4_i32 n = c4_Column::PullValue(*ptr_);
            if (n > 0) {
                c4_String s = "[" + c4_String((const char *) * ptr_, n) + "]";
                const char *desc = s;

                c4_Field *f = d4_new c4_Field(desc);
                d4_assert(! *desc);

                Restructure(*f, false);
                *ptr_ += n;
            }
        }

        int rows = (int)c4_Column::PullValue(*ptr_);
        if (rows > 0) {
            SetNumRows(rows);

            for (int i = 0; i < NumFields(); ++i) {
                NthHandler(i).Define(rows, ptr_);
            }
        }
    }

c4_GroupByViewer::c4_GroupByViewer (c4_Sequence& seq_, const c4_View& keys_,
                           const c4_Property& result_)
  : _parent (&seq_), _keys (keys_), _result (result_)
{
  _sorted = _parent.SortOn(_keys);
  int n = _sorted.GetSize();

  c4_Bytes temp;
  t4_byte* buf = temp.SetBufferClear(n);

  int groups = 0;
  if (n > 0)
  {
    ++buf[0]; // the first entry is always a transition
    groups = 1 + ScanTransitions(1, n, buf, _sorted.Project(_keys));
  }

    // set up a map pointing to each transition
  _map.SetSize(groups + 1);
  int j = 0;

  for (int i = 0; i < n; ++i)
    if (buf[i])
      _map.SetAt(j++, i);

    // also append an entry to point just past the end
  _map.SetAt(j, n);

  d4_assert(_map.GetAt(0) == 0);
  d4_assert(j == groups);
}

int c4_GroupByViewer::ScanTransitions(int lo_, int hi_, t4_byte *flags_, const
  c4_View &match_)const {
  d4_assert(lo_ > 0);

  int m = hi_ - lo_;
  d4_assert(m >= 0);

  // done if nothing left or if entire range is identical
  if (m == 0 || match_[lo_ - 1] == match_[hi_ - 1])
    return 0;

  // range has a transition, done if it is exactly of size one
  if (m == 1) {
    ++(flags_[lo_]);
    return 1;
  }

  // use binary splitting if the range has enough entries
  if (m >= 5)
    return ScanTransitions(lo_, lo_ + m / 2, flags_, match_) + ScanTransitions
      (lo_ + m / 2, hi_, flags_, match_);

  // else use a normal linear scan
  int n = 0;

  for (int i = lo_; i < hi_; ++i)
  if (match_[i] != match_[i - 1]) {
    ++(flags_[i]);
    ++n;
  }

  return n;
}

void c4_FormatV::SetupAllSubviews()
{
  d4_assert(!_inited);
  _inited = true;
  
  if (_data.ColSize() > 0) {
    c4_Bytes temp;
    _data.FetchBytes(0, _data.ColSize(), temp, true);
    const t4_byte* ptr = temp.Contents();
    
    for (int r = 0; r < _subSeqs.GetSize(); ++r) {
      // don't materialize subview if it is empty
      // duplicates code which is in c4_HandlerSeq::Prepare
      const t4_byte* p2 = ptr;
      d4_dbgdef(t4_i32 sias =)
	c4_Column::PullValue(p2);
      d4_assert(sias == 0); // not yet

      if (c4_Column::PullValue(p2) > 0)
	At(r).Prepare(&ptr, false);
      else
	ptr = p2;
    }

    d4_assert(ptr == temp.Contents() + temp.Size());
  }

void c4_FormatB::Define(int, const t4_byte **ptr_) {
  d4_assert(_memos.GetSize() == 0);

  if (ptr_ != 0) {
    _data.PullLocation(*ptr_);
    if (_data.ColSize() > 0)
      _sizeCol.PullLocation(*ptr_);
    _memoCol.PullLocation(*ptr_);
  }

  // everything below this point could be delayed until use
  // in that case, watch out that column space use is properly tracked

  InitOffsets(_sizeCol);

  if (_memoCol.ColSize() > 0) {
    c4_Bytes walk;
    _memoCol.FetchBytes(0, _memoCol.ColSize(), walk, true);

    const t4_byte *p = walk.Contents();

    for (int row = 0; p < walk.Contents() + walk.Size(); ++row) {
      row += c4_Column::PullValue(p);
      d4_assert(row < _memos.GetSize());

      c4_Column *mc = d4_new c4_Column(_data.Persist());
      d4_assert(mc != 0);
      _memos.SetAt(row, mc);

      mc->PullLocation(p);
    }

    d4_assert(p == walk.Contents() + walk.Size());
  }
}

void c4_FormatB::Remove(int index_, int count_) {
  _recalc = true;

  t4_i32 off = Offset(index_);
  t4_i32 n = Offset(index_ + count_) - off;
  d4_assert(n >= 0);

  // remove the columns, if present
  for (int i = 0; i < count_; ++i)
    delete (c4_Column*)_memos.GetAt(index_ + i);
  _memos.RemoveAt(index_, count_);

  if (n > 0)
    _data.Shrink(off, n);

  _offsets.RemoveAt(index_, count_);

  d4_assert(index_ < _offsets.GetSize());

  // adjust all following entries
  while (index_ < _offsets.GetSize())
    _offsets.ElementAt(index_++) -= n;

  d4_assert((t4_i32)_offsets.GetAt(index_ - 1) == _data.ColSize());
  d4_assert(index_ <= _memos.GetSize() + 1);
}

void c4_Persist::LoadAll() {
  c4_Column walk(this);
  if (!LoadIt(walk))
    return ;

  if (_strategy._rootLen < 0) {
    _oldSeek = _strategy._rootPos;
    _oldBuf = d4_new t4_byte[512];
    _oldCurr = _oldLimit = _oldBuf;

    t4_i32 n = FetchOldValue();
    d4_assert(n == 0);
    n = FetchOldValue();
    d4_assert(n > 0);

    c4_Bytes temp;
    t4_byte *buf = temp.SetBuffer(n);
    d4_dbgdef(int n2 = )OldRead(buf, n);
    d4_assert(n2 == n);

    c4_String s = "[" + c4_String((const char*)buf, n) + "]";
    const char *desc = s;

    c4_Field *f = d4_new c4_Field(desc);
    d4_assert(! *desc);

    //?_root->DefineRoot();
    _root->Restructure(*f, false);

    _root->OldPrepare();

    // don't touch data inside while converting the file
    if (_strategy.FileSize() >= 0)
      OccupySpace(1, _strategy.FileSize());
  } else {

void c4_FormatB::InitOffsets(c4_ColOfInts& sizes_)
{
  int rows = Owner().NumRows();

  if (sizes_.RowCount() != rows) {
    sizes_.SetRowCount(rows);
  }

  _memos.SetSize(rows);
  _offsets.SetSize(rows + 1);

  if (_data.ColSize() > 0) {
    t4_i32 total = 0;

    for (int r = 0; r < rows; ++r) {
      int n = sizes_.GetInt(r);
      d4_assert(n >= 0);
      total += n;
      _offsets.SetAt(r + 1, total);
    }

    d4_assert(total == _data.ColSize());
  }
  
}

/// Insert one or more rows into this sequence
void c4_Sequence::InsertAt(int index_, c4_Cursor newElem_, int count_)
{
    d4_assert(newElem_._seq != 0);

    c4_Notifier change(this);
    if (GetDependencies()) {
        change.StartInsertAt(index_, newElem_, count_);
    }

    SetNumRows(NumRows() + count_);

    c4_Bytes data;

    for (int i = 0; i < newElem_._seq->NumHandlers(); ++i) {
        c4_Handler &h = newElem_._seq->NthHandler(i);

        // added 06-12-1999 to do index remapping for derived seq's
        const c4_Sequence *hc = newElem_._seq->HandlerContext(i);
        int ri = newElem_._seq->RemapIndex(newElem_._index, hc);

        int colNum = PropIndex(h.Property());
        d4_assert(colNum >= 0);

        if (h.Property().Type() == 'V') {
            // If inserting from self: Make sure we get a copy of the bytes,
            // so we don't get an invalid pointer if the memory get realloc'ed
            h.GetBytes(ri, data, newElem_._seq == this);

            // special treatment for subviews, insert empty, then overwrite
            // changed 19990904 - probably fixes a long-standing limitation
            c4_Bytes temp;
            h.ClearBytes(temp);

            c4_Handler &h2 = NthHandler(colNum);
            h2.Insert(index_, temp, count_);

            for (int j = 0; j < count_; ++j) {
                h2.Set(index_ + j, data);
            }
        } else {
            h.GetBytes(ri, data);
            NthHandler(colNum).Insert(index_, data, count_);
        }
    }

    // if number of props in dest is larger after adding, clear the rest
    // this way, new props get copied and undefined props get cleared
    if (newElem_._seq->NumHandlers() < NumHandlers()) {
        for (int j = 0; j < NumHandlers(); ++j) {
            c4_Handler &h = NthHandler(j);

            // if the property does not appear in the source
            if (newElem_._seq->PropIndex(h.PropId()) < 0) {
                h.ClearBytes(data);
                h.Insert(index_, data, count_);
            }
        }
    }
}

void c4_CustomSeq::DoSet(int row_, int col_, const c4_Bytes& buf_)
{
  d4_assert(_inited);

  d4_dbgdef(const bool f =)
    _viewer->SetItem(row_, col_, buf_);
  d4_assert(f);
}

c4_Property::c4_Property(const c4_Property &prop_): _id(prop_.GetId()), _type
  (prop_.Type()) {
  c4_ThreadLock::Hold lock;

  d4_assert(sPropCounts != 0);
  d4_assert(sPropCounts->GetAt(_id) > 0);

  Refs( + 1);
}

c4_Sequence::~c4_Sequence() {
  d4_assert(_refCount == 0);

  d4_assert(!_dependencies); // there can be no dependencies left

  ClearCache();

  delete _tempBuf;
}

c4_FileMark::c4_FileMark(t4_i32 pos_, bool flipped_, bool extend_) {
  d4_assert(sizeof *this == 8);
  *(short*)_data = flipped_ ? kReverseFormat : kStorageFormat;
  _data[2] = extend_ ? 0x0A : 0x1A;
  _data[3] = 0;
  t4_byte *p = _data + 4;
  for (int i = 24; i >= 0; i -= 8)
    *p++ = (t4_byte)(pos_ >> i);
  d4_assert(p == _data + sizeof _data);
}

/// Assignment from a reference to a row.
c4_Row &c4_Row::operator = (const c4_RowRef &rowRef_) {
  d4_assert(_cursor._seq != 0);

  if (_cursor !=  &rowRef_) {
    d4_assert(_cursor._index == 0);
    _cursor._seq->SetAt(0, &rowRef_);
  }

  return  *this;
}

void c4_HandlerSeq::DefineRoot()
{
    d4_assert(_field == 0);
    d4_assert(_parent == 0);

    SetNumRows(1);

    const char *desc = "[]";
    _field = d4_new c4_Field(desc);
    d4_assert(! *desc);

    _parent = this;
}

/// Define the complete view structure of the storage
void c4_Storage::SetStructure(const char *description_) {
  d4_assert(description_ != 0);

  if (description_ != Description()) {
    c4_String s = "[" + c4_String(description_) + "]";
    description_ = s;

    c4_Field *field = d4_new c4_Field(description_);
    d4_assert(! *description_);

    d4_assert(field != 0);
    Persist()->Root().Restructure(*field, false);
  }
}

void c4_Column::PullLocation(const t4_byte * &ptr_) {
  d4_assert(_segments.GetSize() == 0);

  _size = PullValue(ptr_);
  _position = 0;
  if (_size > 0) {
    _position = PullValue(ptr_);
    if (_position > 0) {
      d4_assert(_persist != 0);
      _persist->OccupySpace(_position, _size);
    }
  }

  _dirty = false;
}

void c4_FormatB::SetOne(int index_, const c4_Bytes &xbuf_, bool ignoreMemos_) {
  // this fixes bug in 2.4.0 when copying string from higher row
  // TODO: this fix is very conservative, figure out when to copy
  // (can probably look at pointer to see whether it's from us)
  int sz = xbuf_.Size();
  c4_Bytes buf_(xbuf_.Contents(), sz, 0 < sz && sz <= c4_Column::kSegMax);

  c4_Column *cp = &_data;
  t4_i32 start = Offset(index_);
  int len = Offset(index_ + 1) - start;

  if (!ignoreMemos_ && _memos.GetAt(index_) != 0)
    len = ItemLenOffCol(index_, start, cp);

  int m = buf_.Size();
  int n = m - len;

  if (n > 0)
    cp->Grow(start, n);
  else if (n < 0)
    cp->Shrink(start,  - n);
  else if (m == 0)
    return ;
  // no size change and no contents

  _recalc = true;

  cp->StoreBytes(start, buf_);

  if (n && cp ==  &_data) {
    // if size has changed
    int k = _offsets.GetSize() - 1;

    // if filling in an empty entry at end: extend offsets first
    if (m > 0 && index_ >= k) {
      _offsets.InsertAt(k, _offsets.GetAt(k), index_ - k + 1);

      k = index_ + 1;
      d4_assert(k == _offsets.GetSize() - 1);
    }

    // adjust following entry offsets
    while (++index_ <= k)
      _offsets.ElementAt(index_) += n;
  }

  d4_assert((t4_i32)_offsets.GetAt(_offsets.GetSize() - 1) == _data.ColSize());
}

c4_HandlerSeq::~c4_HandlerSeq()
{
    const bool rootLevel = _parent == this;
    c4_Persist *pers = _persist;

    if (rootLevel && pers != 0) {
        pers->DoAutoCommit();
    }

    DetachFromParent();
    DetachFromStorage(true);

    for (int i = 0; i < NumHandlers(); ++i) {
        delete  &NthHandler(i);
    }
    _handlers.SetSize(0);

    ClearCache();

    if (rootLevel) {
        delete _field;

        d4_assert(pers != 0);
        delete pers;
    }
}