static int gridfs_insert_file( gridfs *gfs, const char *name,
                                const bson_oid_t id, gridfs_offset length,
                                const char *contenttype ) {
    bson command;
    bson ret;
    bson res;
    bson_iterator it;
    int result;

    /* Check run md5 */
    bson_init( &command );
    bson_append_oid( &command, "filemd5", &id );
    bson_append_string( &command, "root", gfs->prefix );
    bson_finish( &command );
    assert( mongo_run_command( gfs->client, gfs->dbname, &command, &res ) == MONGO_OK );
    bson_destroy( &command );

    /* Create and insert BSON for file metadata */
    bson_init( &ret );
    bson_append_oid( &ret, "_id", &id );
    if ( name != NULL && *name != '\0' ) {
        bson_append_string( &ret, "filename", name );
    }
    bson_append_long( &ret, "length", length );
    bson_append_int( &ret, "chunkSize", DEFAULT_CHUNK_SIZE );
    bson_append_date( &ret, "uploadDate", ( bson_date_t )1000*time( NULL ) );
    bson_find( &it, &res, "md5" );
    bson_append_string( &ret, "md5", bson_iterator_string( &it ) );
    bson_destroy( &res );
    if ( contenttype != NULL && *contenttype != '\0' ) {
        bson_append_string( &ret, "contentType", contenttype );
    }
    bson_finish( &ret );
    result = mongo_insert( gfs->client, gfs->files_ns, &ret );
    bson_destroy( &ret );

    return result;
}

static void
test_insert (void)
{
   mongoc_collection_t *collection;
   mongoc_client_t *client;
   bson_context_t *context;
   bson_error_t error;
   bool r;
   bson_oid_t oid;
   unsigned i;
   bson_t b;

   client = mongoc_client_new(gTestUri);
   ASSERT (client);

   collection = mongoc_client_get_collection(client, "test", "test");
   ASSERT (collection);

   mongoc_collection_drop(collection, &error);

   context = bson_context_new(BSON_CONTEXT_NONE);
   ASSERT (context);

   for (i = 0; i < 10; i++) {
      bson_init(&b);
      bson_oid_init(&oid, context);
      bson_append_oid(&b, "_id", 3, &oid);
      bson_append_utf8(&b, "hello", 5, "/world", 5);
      r = mongoc_collection_insert(collection, MONGOC_INSERT_NONE, &b, NULL,
                                   &error);
      if (!r) {
         MONGOC_WARNING("%s\n", error.message);
      }
      ASSERT (r);
      bson_destroy(&b);
   }

   bson_init (&b);
   BSON_APPEND_INT32 (&b, "$hello", 1);
   r = mongoc_collection_insert (collection, MONGOC_INSERT_NONE, &b, NULL,
                                 &error);
   ASSERT (!r);
   ASSERT (error.domain == MONGOC_ERROR_BSON);
   ASSERT (error.code == MONGOC_ERROR_BSON_INVALID);
   bson_destroy (&b);

   mongoc_collection_destroy(collection);
   bson_context_destroy(context);
   mongoc_client_destroy(client);
}

static void
test_save (void)
{
   mongoc_collection_t *collection;
   mongoc_database_t *database;
   mongoc_client_t *client;
   bson_context_t *context;
   bson_error_t error;
   bool r;
   bson_oid_t oid;
   unsigned i;
   bson_t b;

   client = mongoc_client_new(gTestUri);
   ASSERT (client);

   database = get_test_database (client);
   ASSERT (database);

   collection = get_test_collection (client, "test_save");
   ASSERT (collection);

   mongoc_collection_drop (collection, &error);

   context = bson_context_new(BSON_CONTEXT_NONE);
   ASSERT (context);

   for (i = 0; i < 10; i++) {
      bson_init(&b);
      bson_oid_init(&oid, context);
      bson_append_oid(&b, "_id", 3, &oid);
      bson_append_utf8(&b, "hello", 5, "/world", 5);
      r = mongoc_collection_save(collection, &b, NULL, &error);
      if (!r) {
         MONGOC_WARNING("%s\n", error.message);
      }
      ASSERT (r);
      bson_destroy(&b);
   }

   bson_destroy (&b);

   r = mongoc_collection_drop (collection, &error);
   ASSERT (r);

   mongoc_collection_destroy(collection);
   mongoc_database_destroy(database);
   bson_context_destroy(context);
   mongoc_client_destroy(client);
}

void gridfs_remove_filename( gridfs *gfs, const char *filename ) {
    bson query;
    mongo_cursor *files;
    bson file;
    bson_iterator it;
    bson_oid_t id;
    bson b;

    bson_init( &query );
    bson_append_string( &query, "filename", filename );
    bson_finish( &query );
    files = mongo_find( gfs->client, gfs->files_ns, &query, NULL, 0, 0, 0 );
    bson_destroy( &query );

    /* Remove each file and it's chunks from files named filename */
    while ( mongo_cursor_next( files ) == MONGO_OK ) {
        file = files->current;
        bson_find( &it, &file, "_id" );
        id = *bson_iterator_oid( &it );

        /* Remove the file with the specified id */
        bson_init( &b );
        bson_append_oid( &b, "_id", &id );
        bson_finish( &b );
        mongo_remove( gfs->client, gfs->files_ns, &b );
        bson_destroy( &b );

        /* Remove all chunks from the file with the specified id */
        bson_init( &b );
        bson_append_oid( &b, "files_id", &id );
        bson_finish( &b );
        mongo_remove( gfs->client, gfs->chunks_ns, &b );
        bson_destroy( &b );
    }

    mongo_cursor_destroy( files );
}

static void
test_bson_as_json (void)
{
   bson_oid_t oid;
   bson_t *b;
   bson_t *b2;
   char *str;
   size_t len;
   int i;

   bson_oid_init_from_string(&oid, "123412341234abcdabcdabcd");

   b = bson_new();
   assert(bson_append_utf8(b, "utf8", -1, "bar", -1));
   assert(bson_append_int32(b, "int32", -1, 1234));
   assert(bson_append_int64(b, "int64", -1, 4321));
   assert(bson_append_double(b, "double", -1, 123.4));
   assert(bson_append_undefined(b, "undefined", -1));
   assert(bson_append_null(b, "null", -1));
   assert(bson_append_oid(b, "oid", -1, &oid));
   assert(bson_append_bool(b, "true", -1, true));
   assert(bson_append_bool(b, "false", -1, false));
   assert(bson_append_time_t(b, "date", -1, time(NULL)));
   assert(bson_append_timestamp(b, "timestamp", -1, (uint32_t)time(NULL), 1234));
   assert(bson_append_regex(b, "regex", -1, "^abcd", "xi"));
   assert(bson_append_dbpointer(b, "dbpointer", -1, "mycollection", &oid));
   assert(bson_append_minkey(b, "minkey", -1));
   assert(bson_append_maxkey(b, "maxkey", -1));
   assert(bson_append_symbol(b, "symbol", -1, "var a = {};", -1));

   b2 = bson_new();
   assert(bson_append_int32(b2, "0", -1, 60));
   assert(bson_append_document(b, "document", -1, b2));
   assert(bson_append_array(b, "array", -1, b2));

   {
      const uint8_t binary[] = { 0, 1, 2, 3, 4 };
      assert(bson_append_binary(b, "binary", -1, BSON_SUBTYPE_BINARY,
                                binary, sizeof binary));
   }

   for (i = 0; i < 1000; i++) {
      str = bson_as_json(b, &len);
      bson_free(str);
   }

   bson_destroy(b);
   bson_destroy(b2);
}

 void InitWith(bson *obj)
 {
     bson_iterator it;
     bson_iterator_init(&it, obj);
     
     while (bson_iterator_next(&it))
     {
         const char * key = bson_iterator_key(&it);
         bson_type type = bson_iterator_type(&it);
         
         switch (type)
         {
             case BSON_STRING:
                 bson_append_string(object, key, bson_iterator_string(&it));
                 break;
                 
             case BSON_INT:
                 bson_append_int(object, key, bson_iterator_int(&it));
                 break;
                 
             case BSON_LONG:
                 bson_append_long(object, key, bson_iterator_long(&it));
                 break;
                 
             case BSON_DOUBLE:
                 bson_append_double(object, key, bson_iterator_double(&it));
                 break;
                 
             case BSON_OBJECT:
             {
                 bson sub;
                 
                 bson_iterator_subobject(&it, &sub);
                 bson_append_bson(object, key, &sub);
                 break;
             }
                 
             case BSON_OID:
                 bson_append_oid(object, key, bson_iterator_oid(&it));
                 break;
                 
                 
             default:
                 DVASSERT(false);
                 Logger::Error("[MongodbObjectInternalData::InitWith] Not implemented type: %d", type);
                 break;
         }
     }
 }

static void
test_bson_append_oid (void)
{
   bson_oid_t oid;
   bson_t *b;
   bson_t *b2;

   bson_oid_init_from_string(&oid, "1234567890abcdef1234abcd");

   b = bson_new();
   assert(bson_append_oid(b, "oid", -1, &oid));
   b2 = get_bson("test22.bson");
   assert_bson_equal(b, b2);
   bson_destroy(b);
   bson_destroy(b2);
}

static bson * chunk_new(bson_oid_t id, int chunkNumber,
                        const char * data, int len)

{
    bson * b;
    bson_buffer buf;

    b = (bson *)malloc(sizeof(bson));
    if (b == NULL) return NULL;

    bson_buffer_init(&buf);
    bson_append_oid(&buf, "files_id", &id);
    bson_append_int(&buf, "n", chunkNumber);
    bson_append_binary(&buf, "data", 2, data, len);
    bson_from_buffer(b, &buf);
    return  b;
}

bson *
test_bson_generate_full (void)
{
  bson *b, *d, *a, *scope;
  guint8 oid[] = "1234567890ab";

  a = bson_new ();
  bson_append_int32 (a, "0", 32);
  bson_append_int64 (a, "1", (gint64)-42);
  bson_finish (a);

  d = bson_new ();
  bson_append_string (d, "name", "sub-document", -1);
  bson_append_int32 (d, "answer", 42);
  bson_finish (d);

  scope = bson_new ();
  bson_append_string (scope, "v", "hello world", -1);
  bson_finish (scope);

  b = bson_new ();
  bson_append_double (b, "double", 3.14);
  bson_append_string (b, "str", "hello world", -1);
  bson_append_document (b, "doc", d);
  bson_append_array (b, "array", a);
  bson_append_binary (b, "binary0", BSON_BINARY_SUBTYPE_GENERIC,
                      (guint8 *)"foo\0bar", 7);
  bson_append_oid (b, "_id", oid);
  bson_append_boolean (b, "TRUE", FALSE);
  bson_append_utc_datetime (b, "date", 1294860709000);
  bson_append_timestamp (b, "ts", 1294860709000);
  bson_append_null (b, "null");
  bson_append_regex (b, "foobar", "s/foo.*bar/", "i");
  bson_append_javascript (b, "alert", "alert (\"hello world!\");", -1);
  bson_append_symbol (b, "sex", "Marilyn Monroe", -1);
  bson_append_javascript_w_scope (b, "print", "alert (v);", -1, scope);
  bson_append_int32 (b, "int32", 32);
  bson_append_int64 (b, "int64", (gint64)-42);
  bson_finish (b);

  bson_free (d);
  bson_free (a);
  bson_free (scope);

  return b;
}

uint8_t
XTDBInsert(XTDBHandle* handle,bson* newVal) {
    _S_FN(insert);
    handle->gen++;
    bson_oid_t oid;
    bson_oid_gen(&oid);
    BinaryStr key,val;
    uint8_t ret;
    bson copyObj;

    handle->lastOp = XTDB_OP_INSERT;
    key.data = oid.bytes;
    key.len  = sizeof(oid.bytes);
    if (bson_copy(&copyObj,newVal)) {
        handle->error = XTDB_INVALID_BSON;
        return False;
    }
    bson_unfinish_object(&copyObj);
    if (bson_append_oid(&copyObj,"_id",&oid)) {
        handle->error = XTDB_NO_MEM;
        bson_destroy(&copyObj);
        return False;
    }
    if (bson_finish(&copyObj)) {
        handle->error = XTDB_NO_MEM;
        bson_destroy(&copyObj);
        return False;
    }
    //bson_print(&copyObj);
    BsonToBStr(&copyObj,&val);

    if (XTDBInsertToIndex(handle,&key,&val) == False) {
        assert(0);
        return False;
    }

    ret = DBSet(handle->mainDB,&key,&val,False);
    _E_FN(insert);
    if (!ret) {
        handle->error = DBGetLastError(handle->mainDB);
    } else {
        memcpy(&handle->status.insertStatus.newOid,&oid,sizeof(oid));
    }
    return ret;
}

int commit_inode(struct inode * e) {
    bson cond, doc;
    mongo * conn = get_conn();
    char istr[4];
    struct dirent * cde = e->dirents;
    int res;

    bson_init(&doc);
    bson_append_start_object(&doc, "$set");
    bson_append_start_array(&doc, "dirents");
    res = 0;
    while(cde) {
        bson_numstr(istr, res++);
        bson_append_string(&doc, istr, cde->path);
        cde = cde->next;
    }
    bson_append_finish_array(&doc);

    bson_append_int(&doc, "mode", e->mode);
    bson_append_long(&doc, "owner", e->owner);
    bson_append_long(&doc, "group", e->group);
    bson_append_long(&doc, "size", e->size);
    bson_append_time_t(&doc, "created", e->created);
    bson_append_time_t(&doc, "modified", e->modified);
    if(e->data && e->datalen > 0)
        bson_append_string_n(&doc, "data", e->data, e->datalen);
    bson_append_finish_object(&doc);
    bson_finish(&doc);

    bson_init(&cond);
    bson_append_oid(&cond, "_id", &e->oid);
    bson_finish(&cond);

    res = mongo_update(conn, inodes_name, &cond, &doc,
        MONGO_UPDATE_UPSERT, NULL);
    bson_destroy(&cond);
    bson_destroy(&doc);
    if(res != MONGO_OK) {
        fprintf(stderr, "Error committing inode %s\n",
            mongo_get_server_err_string(conn));
        return -EIO;
    }
    return 0;
}

MONGO_EXPORT mongo_cursor *gridfile_get_chunks( gridfile *gfile, int start, int size ) {
    bson_iterator it;
    bson_oid_t id;
    bson gte;
    bson query;
    bson orderby;
    bson command;
    mongo_cursor *cursor;

    bson_find( &it, gfile->meta, "_id" );
    id = *bson_iterator_oid( &it );

    bson_init( &query );
    bson_append_oid( &query, "files_id", &id );
    if ( size == 1 ) {
        bson_append_int( &query, "n", start );
    }
    else {
        bson_init( &gte );
        bson_append_int( &gte, "$gte", start );
        bson_finish( &gte );
        bson_append_bson( &query, "n", &gte );
        bson_destroy( &gte );
    }
    bson_finish( &query );

    bson_init( &orderby );
    bson_append_int( &orderby, "n", 1 );
    bson_finish( &orderby );

    bson_init( &command );
    bson_append_bson( &command, "query", &query );
    bson_append_bson( &command, "orderby", &orderby );
    bson_finish( &command );

    cursor = mongo_find( gfile->gfs->client, gfile->gfs->chunks_ns,
                         &command, NULL, size, 0, 0 );

    bson_destroy( &command );
    bson_destroy( &query );
    bson_destroy( &orderby );

    return cursor;
}

uint8_t XTDBBsonAppendOid(XTDBHandle* handle,bson* in , bson* out,bson_oid_t * oid) {

    if (bson_copy(out,in)) {
        handle->error = XTDB_INVALID_BSON;
        return False;
    }
    bson_unfinish_object(out);
    if (bson_append_oid(out,"_id",oid)) {
        handle->error = XTDB_NO_MEM;
        bson_destroy(out);
        return False;
    }
    if (bson_finish(out)) {
        handle->error = XTDB_NO_MEM;
        bson_destroy(out);
        return False;
    }
    return True;
}

static void
test_has_collection (void)
{
   mongoc_collection_t *collection;
   mongoc_database_t *database;
   mongoc_client_t *client;
   bson_error_t error;
   char *name;
   bool r;
   bson_oid_t oid;
   bson_t b;

   client = mongoc_client_new (gTestUri);
   assert (client);

   name = gen_collection_name ("has_collection");
   collection = mongoc_client_get_collection (client, "test", name);
   assert (collection);

   database = mongoc_client_get_database (client, "test");
   assert (database);

   bson_init (&b);
   bson_oid_init (&oid, NULL);
   bson_append_oid (&b, "_id", 3, &oid);
   bson_append_utf8 (&b, "hello", 5, "world", 5);
   r = mongoc_collection_insert (collection, MONGOC_INSERT_NONE, &b, NULL,
                                 &error);
   if (!r) {
      MONGOC_WARNING ("%s\n", error.message);
   }
   assert (r);
   bson_destroy (&b);

   r = mongoc_database_has_collection (database, name, &error);
   assert (!error.domain);
   assert (r);

   bson_free (name);
   mongoc_database_destroy (database);
   mongoc_collection_destroy (collection);
   mongoc_client_destroy (client);
}

static void
test_oid (void)
{
   bson_t bcon, expected;
   bson_oid_t oid;

   bson_init (&bcon);
   bson_init (&expected);

   bson_oid_init (&oid, NULL);

   bson_append_oid (&expected, "foo", -1, &oid);
   BCON_APPEND (&bcon, "foo", BCON_OID (&oid));

   bson_eq_bson (&bcon, &expected);

   bson_destroy (&bcon);
   bson_destroy (&expected);
}

bson gridfile_get_chunk( gridfile *gfile, int n ) {
    bson query;
    bson out;
    bson_iterator it;
    bson_oid_t id;

    bson_init( &query );
    bson_find( &it, gfile->meta, "_id" );
    id = *bson_iterator_oid( &it );
    bson_append_oid( &query, "files_id", &id );
    bson_append_int( &query, "n", n );
    bson_finish( &query );

    assert( mongo_find_one( gfile->gfs->client,
                            gfile->gfs->chunks_ns,
                            &query, NULL, &out ) == MONGO_OK );

    bson_destroy( &query );
    return out;
}

mongo_cursor* gridfile_get_chunks(gridfile* gfile, int start, int size)

{
  bson_iterator it;
  bson_oid_t id;
  bson_buffer gte_buf;
  bson gte_bson;
  bson_buffer query_buf;
  bson query_bson;
  bson_buffer orderby_buf;
  bson orderby_bson;
  bson_buffer command_buf;
  bson command_bson;

  bson_find(&it, gfile->meta, "_id");
  id = *bson_iterator_oid(&it);

  bson_buffer_init(&query_buf);
  bson_append_oid(&query_buf, "files_id", &id);
  if (size == 1) {
    bson_append_int(&query_buf, "n", start);
  } else {
    bson_buffer_init(&gte_buf);
    bson_append_int(&gte_buf, "$gte", start);
    bson_from_buffer(&gte_bson, &gte_buf);
    bson_append_bson(&query_buf, "n", &gte_bson);
  }
  bson_from_buffer(&query_bson, &query_buf);

  bson_buffer_init(&orderby_buf);
  bson_append_int(&orderby_buf, "n", 1);
  bson_from_buffer(&orderby_bson, &orderby_buf);

  bson_buffer_init(&command_buf);
  bson_append_bson(&command_buf, "query", &query_bson);
  bson_append_bson(&command_buf, "orderby", &orderby_bson);
  bson_from_buffer(&command_bson, &command_buf);

  return mongo_find(gfile->gfs->client, gfile->gfs->chunks_ns,
        &command_bson, NULL, size, 0, 0);
}

bson gridfile_get_chunk(gridfile* gfile, int n)

{
    bson query;
    bson out;
    bson_buffer buf;
    bson_iterator it;
    bson_oid_t id;

    bson_buffer_init(&buf);
    bson_find(&it, gfile->meta, "_id");
    id = *bson_iterator_oid(&it);
    bson_append_oid(&buf, "files_id", &id);
    bson_append_int(&buf, "n", n);
    bson_from_buffer(&query, &buf);

    assert(mongo_find_one(gfile->gfs->client,
                          gfile->gfs->chunks_ns,
                          &query, NULL, &out));
    return out;
}

int do_trunc(struct inode * e, off_t off) {
    bson cond;
    int res;
    mongo * conn = get_conn();

    if(off > e->size) {
        e->size = off;
        return 0;
    }

    pthread_mutex_lock(&e->wr_lock);
    if(e->wr_extent) {
        if(off < 0 && (res = serialize_extent(e, e->wr_extent)) != 0)
            return res;
        e->wr_extent->nnodes = 0;
    }
    e->wr_age = time(NULL);
    pthread_mutex_unlock(&e->wr_lock);

    bson_init(&cond);
    bson_append_oid(&cond, "inode", &e->oid);
    if(off > 0) {
        bson_append_start_object(&cond, "start");
        bson_append_long(&cond, "$gte", off);
        bson_append_finish_object(&cond);
    }
    bson_finish(&cond);

    res = mongo_remove(conn, extents_name, &cond, NULL);
    bson_destroy(&cond);
    if(res != 0) {
        fprintf(stderr, "Error removing extents in do_truncate\n");
        return -EIO;
    }

    e->size = off;

    return 0;
}

static void
download_task (perf_test_t *test)
{
   download_test_t *gridfs_test;
   bson_t query = BSON_INITIALIZER;
   mongoc_gridfs_file_t *file;
   mongoc_iovec_t iov;
   bson_error_t error;
   ssize_t read_sz;

   gridfs_test = (download_test_t *) test;

   bson_append_oid (&query, "_id", 3, &gridfs_test->file_id);
   file = mongoc_gridfs_find_one (gridfs_test->base.gridfs, &query, &error);
   if (!file) {
      MONGOC_ERROR ("gridfs_find_one: %s\n", error.message);
      abort ();
   }

   /* overwrite the buffer we used for _upload_big_file */
   iov.iov_base = (void *) gridfs_test->base.data;
   iov.iov_len = gridfs_test->base.data_sz;
   read_sz = mongoc_gridfs_file_readv (file, &iov, 1, gridfs_test->base.data_sz,
                                       0);

   if (read_sz != gridfs_test->base.data_sz) {
      if (mongoc_gridfs_file_error (file, &error)) {
         MONGOC_ERROR ("file_readv: %s\n", error.message);
      } else {
         MONGOC_ERROR ("file_readv: unknown error\n");
      }

      abort ();
   }

   mongoc_gridfs_file_destroy (file);
   bson_destroy (&query);
}