int main(int argc, char *argv[]) {
  IMP::base::setup_from_argv(argc, argv, "Test of base caches in C++");
  IMP_NEW(IMP::kernel::Model, m, ());
  IMP::kernel::ParticleIndexes pis;
  IMP::algebra::BoundingBox3D bb = IMP::algebra::get_unit_bounding_box_d<3>();
  for (unsigned int i = 0; i < num_particles; ++i) {
    pis.push_back(m->add_particle("P%1%"));
    IMP::core::XYZR::setup_particle(
        m, pis.back(),
        IMP::algebra::Sphere3D(IMP::algebra::get_random_vector_in(bb), radius));
  }
  IMP_NEW(IMP::core::GridClosePairsFinder, gcpf, ());
  gcpf->set_distance(.1);
  IMP::base::PointerMember<IMP::core::ClosePairsFinder> mcpf =
      IMP::misc::create_metric_close_pairs_finder(LowerBound(), UpperBound());
  mcpf->set_distance(.1);

  IMP::kernel::ParticleIndexPairs gcp = gcpf->get_close_pairs(m, pis);
  canonicalize(gcp);
  IMP::kernel::ParticleIndexPairs mcp = mcpf->get_close_pairs(m, pis);
  canonicalize(mcp);
  std::sort(gcp.begin(), gcp.end());
  std::sort(mcp.begin(), mcp.end());
  std::cout << "Lists are " << gcp << " and " << mcp << std::endl;
  IMP::kernel::ParticleIndexPairs out;
  std::set_intersection(gcp.begin(), gcp.end(), mcp.begin(), mcp.end(),
                        std::back_inserter(out));
  IMP_TEST_EQUAL(out.size(), mcp.size());
  IMP_TEST_EQUAL(out.size(), gcp.size());
  return 0;
}

int link(char *oldname, char *newname) {
  struct fs *oldfs;
  struct fs *newfs;
  char *oldrest;
  char *newrest;
  int rc;
  char oldpath[MAXPATH];
  char newpath[MAXPATH];

  rc = canonicalize(oldname, oldpath);
  if (rc < 0) return rc;

  rc = fslookup(oldpath, 0, &oldfs, &oldrest);
  if (rc < 0) return rc;

  rc = canonicalize(newname, newpath);
  if (rc < 0) return rc;

  rc = fslookup(newpath, 0, &newfs, &newrest);
  if (rc < 0) return rc;

  if (oldfs != newfs) return -EXDEV;

  if (!oldfs->ops->link) return -ENOSYS;
  oldfs->locks++;
  if (lock_fs(oldfs, FSOP_LINK) < 0) {
    oldfs->locks--;
    return -ETIMEOUT;
  }
  rc = oldfs->ops->link(oldfs, oldrest, newrest);
  unlock_fs(oldfs, FSOP_LINK);
  oldfs->locks--;
  return rc;
}

char *
dtools_resolve_path (const char *base, const char *path)
{
  const char *base_rest;
  char *new_path, *res;

  g_return_val_if_fail (base != NULL, NULL);
  g_return_val_if_fail (dtools_path_is_full (base), NULL);

  if (path == NULL)
    return g_strdup (base);

  if (dtools_path_is_full (path))
    return canonicalize (path);

  if (dtools_path_is_absolute (path))
    {
      char *bus = dtools_get_bus (base, &base_rest);
      new_path = g_strconcat (bus, path, NULL);
      g_free (bus);
    }
  else
    new_path = g_build_path ("/", base, path, NULL);

  res = canonicalize (new_path);
  g_free (new_path);
  return res;

}

/* Like tensor_compress, but also compress into one dimension any
   group of dimensions that form a contiguous block of indices with
   some stride.  (This can safely be done for transform vector sizes.) */
tensor *X(tensor_compress_contiguous)(const tensor *sz)
{
     int i, rnk;
     tensor *sz2, *x;

     if (X(tensor_sz)(sz) == 0) 
	  return X(mktensor)(RNK_MINFTY);

     sz2 = really_compress(sz);
     A(FINITE_RNK(sz2->rnk));

     if (sz2->rnk <= 1) { /* nothing to compress. */ 
	  if (0) {
	       /* this call is redundant, because "sz->rnk <= 1" implies
		  that the tensor is already canonical, but I am writing
		  it explicitly because "logically" we need to canonicalize
		  the tensor before returning. */
	       canonicalize(sz2);
	  }
          return sz2;
     }

     /* sort in descending order of |istride|, so that compressible
	dimensions appear contigously */
     qsort(sz2->dims, (unsigned)sz2->rnk, sizeof(iodim),
		(int (*)(const void *, const void *))compare_by_istride);

     /* compute what the rank will be after compression */
     for (i = rnk = 1; i < sz2->rnk; ++i)
          if (!strides_contig(sz2->dims + i - 1, sz2->dims + i))
               ++rnk;

     /* merge adjacent dimensions whenever possible */
     x = X(mktensor)(rnk);
     x->dims[0] = sz2->dims[0];
     for (i = rnk = 1; i < sz2->rnk; ++i) {
          if (strides_contig(sz2->dims + i - 1, sz2->dims + i)) {
               x->dims[rnk - 1].n *= sz2->dims[i].n;
               x->dims[rnk - 1].is = sz2->dims[i].is;
               x->dims[rnk - 1].os = sz2->dims[i].os;
          } else {
               A(rnk < x->rnk);
               x->dims[rnk++] = sz2->dims[i];
          }
     }

     X(tensor_destroy)(sz2);

     /* reduce to canonical form */
     canonicalize(x);
     return x;
}

static inline bool sb_unlinkat_pre_check(WRAPPER_ARGS_PROTO)
{
	char canonic[SB_PATH_MAX];

	save_errno();

	if (-1 == canonicalize(pathname, canonic))
		/* see comments in check_syscall() */
		if (ENAMETOOLONG != errno)
			goto error;

	/* XXX: Hack to make sure sandboxed process cannot remove
	 * a device node, bug #79836. */
	if (0 == strcmp(canonic, "/dev/null") ||
	    0 == strcmp(canonic, "/dev/zero"))
	{
		errno = EACCES;
		goto error;
	}

	restore_errno();

	return true;

 error:
	if (is_env_on(ENV_SANDBOX_DEBUG))
		SB_EINFO("EARLY FAIL", "  %s(%s): %s\n",
			STRING_NAME, pathname, strerror(errno));
	return false;
}

int utime(char *name, struct utimbuf *times) {
  struct fs *fs;
  char *rest;
  int rc;
  char path[MAXPATH];

  rc = canonicalize(name, path);
  if (rc < 0) return rc;

  rc = fslookup(path, 0, &fs, &rest);
  if (rc < 0) return rc;
  
  if (!times) return -EINVAL;

  if (!fs->ops->utime) return -ENOSYS;
  fs->locks++;
  if (lock_fs(fs, FSOP_UTIME) < 0) {
    fs->locks--;
    return -ETIMEOUT;
  }
  rc = fs->ops->utime(fs, rest, times);
  unlock_fs(fs, FSOP_UTIME);
  fs->locks--;
  return rc;
}

bool sb_unlinkat_pre_check(const char *func, const char *pathname, int dirfd)
{
	char canonic[SB_PATH_MAX];

	save_errno();

	/* XXX: need to check pathname with dirfd */
	if (-1 == canonicalize(pathname, canonic))
		/* see comments in check_syscall() */
		if (ENAMETOOLONG != errno)
			goto error;

	/* XXX: Hack to make sure sandboxed process cannot remove
	 * a device node, bug #79836. */
	if (0 == strcmp(canonic, "/dev/null") ||
	    0 == strcmp(canonic, "/dev/zero"))
	{
		errno = EACCES;
		goto error;
	}

	restore_errno();

	return true;

 error:
	if (is_env_on(ENV_SANDBOX_DEBUG))
		SB_EINFO("EARLY FAIL", "  %s(%s): %s\n",
			func, pathname, strerror(errno));
	return false;
}

BOOL
TSHPath::Canonicalize(LPTSTR pszBuf, LPCTSTR pszPath)
{
  static TModuleProc2<BOOL,LPTSTR,LPCTSTR>
         canonicalize(GetModule(), CanonicalizeStr);
  return canonicalize(pszBuf, pszPath);
}

bool sb_mkdirat_pre_check(const char *func, const char *pathname, int dirfd)
{
	char canonic[SB_PATH_MAX];

	save_errno();

	/* XXX: need to check pathname with dirfd */
	if (-1 == canonicalize(pathname, canonic))
		/* see comments in check_syscall() */
		if (ENAMETOOLONG != errno) {
			if (is_env_on(ENV_SANDBOX_DEBUG))
				SB_EINFO("EARLY FAIL", "  %s(%s) @ canonicalize: %s\n",
					func, pathname, strerror(errno));
			return false;
		}

	/* XXX: Hack to prevent errors if the directory exist, and are
	 * not writable - we rather return EEXIST than fail.  This can
	 * occur if doing something like `mkdir -p /`.  We certainly do
	 * not want to pass this attempt up to the higher levels as those
	 * will trigger a sandbox violation.
	 */
	struct stat st;
	if (0 == lstat(canonic, &st)) {
		if (is_env_on(ENV_SANDBOX_DEBUG))
			SB_EINFO("EARLY FAIL", "  %s(%s[%s]) @ lstat: %s\n",
				func, pathname, canonic, strerror(errno));
		errno = EEXIST;
		return false;
	}

	restore_errno();

	return true;
}

UTL_WString::UTL_WString(UTL_WString *s)
{
   char *b;

   if (s == NULL)
   {
      p_str = c_str = NULL;
      alloced = len = 0;
   }
   else
   {
      b = s->get_string();

      if (b == NULL)
      {
         p_str = c_str = NULL;
         alloced = len = 0;
      }
      else
      {
         len = strlen(b);
         alloced = len + 1;
         p_str = new char [alloced];
         c_str = new char [alloced];
         os_strcpy(p_str, b);
         canonicalize();
      }
   }
}

bool sb_mkdirat_pre_check(const char *func, const char *pathname, int dirfd)
{
	char canonic[SB_PATH_MAX];
	char dirfd_path[SB_PATH_MAX];

	save_errno();

	/* Expand the dirfd path first */
	switch (resolve_dirfd_path(dirfd, pathname, dirfd_path, sizeof(dirfd_path))) {
		case -1:
			sb_debug_dyn("EARLY FAIL: %s(%s) @ resolve_dirfd_path: %s\n",
				func, pathname, strerror(errno));
			return false;
		case 0:
			pathname = dirfd_path;
			break;
	}

	/* Then break down any relative/symlink paths */
	if (-1 == canonicalize(pathname, canonic))
		/* see comments in check_syscall() */
		if (ENAMETOOLONG != errno) {
			sb_debug_dyn("EARLY FAIL: %s(%s) @ canonicalize: %s\n",
				func, pathname, strerror(errno));
			return false;
		}

	/* XXX: Hack to prevent errors if the directory exist, and are
	 * not writable - we rather return EEXIST than fail.  This can
	 * occur if doing something like `mkdir -p /`.  We certainly do
	 * not want to pass this attempt up to the higher levels as those
	 * will trigger a sandbox violation.
	 */
	struct stat st;
	if (0 == lstat(canonic, &st)) {
		int new_errno;
		sb_debug_dyn("EARLY FAIL: %s(%s[%s]) @ lstat: %s\n",
			func, pathname, canonic, strerror(errno));

		new_errno = EEXIST;

		/* Hmm, is this a broken symlink we're trying to extend ? */
		if (S_ISLNK(st.st_mode) && stat(pathname, &st) != 0) {
			/* XXX: This awful hack should probably be turned into a
			 * common func that does a better job.  For now, we have
			 * enough crap to catch gnulib tests #297026.
			 */
			char *parent = strrchr(pathname, '/');
			if (parent && (strcmp(parent, "/.") == 0 || strcmp(parent, "/..") == 0))
				new_errno = ENOENT;
		}

		errno = new_errno;
		return false;
	}

	restore_errno();

	return true;
}

int chdir(char *name) {
  struct fs *fs;
  char *rest;
  int rc;
  char path[MAXPATH];
  char newdir[MAXPATH];
  struct stat64 buffer;

  rc = canonicalize(name, path);
  if (rc < 0) return rc;
  strcpy(newdir, path);

  rc = fslookup(path, 0, &fs, &rest);
  if (rc < 0) return rc;
  
  if (fs->ops->stat) {
    fs->locks++;
    if (lock_fs(fs, FSOP_STAT) < 0) {
      fs->locks--;
      return -ETIMEOUT;
    }
    rc = fs->ops->stat(fs, rest, &buffer);
    unlock_fs(fs, FSOP_STAT);
    fs->locks--;

    if (rc < 0) return rc;
    if ((buffer.st_mode & S_IFMT) != S_IFDIR) return -ENOTDIR;
  }

  strcpy(self()->curdir, newdir);

  return 0;
}

// Creates canonical codewords
vector<string> create_codewords(double *freqs, int n_freqs)
{
    typedef pair<Node<int>*, string> NodeCode;
    vector<string> codewords(n_freqs);
    queue<NodeCode> q;
    Node<int>* root = create_tree(freqs, n_freqs);
    q.push(NodeCode(root, ""));
    while(!q.empty())
    {
        NodeCode nc = q.front();
        q.pop();
        if(nc.first->is_leaf())
        {
            codewords[nc.first->data] = nc.second;
        }
        else
        {
            q.push(NodeCode(nc.first->left, nc.second + "0"));
            q.push(NodeCode(nc.first->right, nc.second + "1"));
        }
    }

    auto cws = new pair<int, string>[n_freqs]; // max n_freqs
    for(int i = 0; i < n_freqs; ++i)
    {
        cws[i] = pair<int, string>(i, codewords[i]);
    }

    vector<string> canonical = canonicalize(cws, n_freqs);

    delete[] cws;

    return canonical;
}

static inline v2
get_min_diff(v2 a, v2 b)
{
    v2 result;
    v2 diff = a - b;
    result = canonicalize(diff);
    return result;
}

    WITH_PLATFORM_STRING(env, pathname, path) {
	char canonicalPath[JVM_MAXPATHLEN];
	if (canonicalize(JVM_NativePath((char *)path),
			 canonicalPath, JVM_MAXPATHLEN) < 0) {
	    JNU_ThrowIOExceptionWithLastError(env, "Bad pathname");
	} else {
	    rv = JNU_NewStringPlatform(env, canonicalPath);
	}
    } END_PLATFORM_STRING(env, path);

/*
 * based on similar function in util-linux-2.12r/mount/mount.c
 */
static void
update_mtab_entry(const char *spec, const char *node, const char *type,
		  const char *opts, int freq, int pass, int addnew)
{
	struct my_mntent mnt;

	mnt.mnt_fsname = canonicalize (spec);
	mnt.mnt_dir = canonicalize (node);
	mnt.mnt_type = xstrdup(type);
	mnt.mnt_opts = xstrdup(opts);
	mnt.mnt_freq = freq;
	mnt.mnt_passno = pass;

	/* We get chatty now rather than after the update to mtab since the
	   mount succeeded, even if the write to /etc/mtab should fail.  */
	if (verbose)
		print_one (&mnt);

	if (!addnew)
		update_mtab (mnt.mnt_dir, &mnt);
	else {
		mntFILE *mfp;

		lock_mtab();
		mfp = my_setmntent(MOUNTED, "a+");
		if (mfp == NULL || mfp->mntent_fp == NULL) {
			int errsv = errno;
			error(_("%s: can't open %s, %s"),
			      progname, MOUNTED, strerror(errsv));
		} else {
			if ((my_addmntent (mfp, &mnt)) == 1) {
				int errsv = errno;
				error(_("%s: error writing %s, %s"),
				      progname, MOUNTED, strerror(errsv));
			}
		}
		my_endmntent(mfp);
		unlock_mtab();
	}
	my_free(mnt.mnt_fsname);
	my_free(mnt.mnt_dir);
	my_free(mnt.mnt_type);
	my_free(mnt.mnt_opts);
}

    void run() {
        // Run the update.
        {
            Client::WriteContext ctx(&_txn, ns());
            Client& c = cc();
            CurOp& curOp = *c.curop();
            OpDebug* opDebug = &curOp.debug();
            UpdateDriver driver( (UpdateDriver::Options()) );
            Database* db = ctx.ctx().db();

            // Collection should be empty.
            ASSERT_EQUALS(0U, count(BSONObj()));

            UpdateRequest request(&_txn, nsString());
            UpdateLifecycleImpl updateLifecycle(false, nsString());
            request.setLifecycle(&updateLifecycle);

            // Update is the upsert {_id: 0, x: 1}, {$set: {y: 2}}.
            BSONObj query = fromjson("{_id: 0, x: 1}");
            BSONObj updates = fromjson("{$set: {y: 2}}");

            request.setUpsert();
            request.setQuery(query);
            request.setUpdates(updates);

            ASSERT_OK(driver.parse(request.getUpdates(), request.isMulti()));

            // Setup update params.
            UpdateStageParams params(&request, &driver, opDebug);
            scoped_ptr<CanonicalQuery> cq(canonicalize(query));
            params.canonicalQuery = cq.get();

            scoped_ptr<WorkingSet> ws(new WorkingSet());
            auto_ptr<EOFStage> eofStage(new EOFStage());

            scoped_ptr<UpdateStage> updateStage(
                new UpdateStage(params, ws.get(), db, eofStage.release()));

            runUpdate(updateStage.get());
            ctx.commit();
        }

        // Verify the contents of the resulting collection.
        {
            Client::ReadContext ctx(&_txn, ns());
            Collection* collection = ctx.ctx().db()->getCollection(&_txn, ns());

            vector<BSONObj> objs;
            getCollContents(collection, &objs);

            // Expect a single document, {_id: 0, x: 1, y: 2}.
            ASSERT_EQUALS(1U, objs.size());
            ASSERT_EQUALS(objs[0], fromjson("{_id: 0, x: 1, y: 2}"));
        }
    }

/*
 * Code based on similar function in util-linux-2.12p/mount/mount.c
 *
 */
static void update_mtab_entry(char *spec, char *node, char *type, char *opts,
			      int flags, int freq, int pass)
{
	struct my_mntent mnt;

	mnt.mnt_fsname = canonicalize (spec);
	mnt.mnt_dir = canonicalize (node);
	mnt.mnt_type = type;
	mnt.mnt_opts = opts;
	mnt.mnt_freq = freq;
	mnt.mnt_passno = pass;
      
	/* We get chatty now rather than after the update to mtab since the
	   mount succeeded, even if the write to /etc/mtab should fail.  */
	if (verbose)
		print_one (&mnt);

	if (!nomtab && mtab_is_writable()) {
		if (flags & MS_REMOUNT)
			update_mtab (mnt.mnt_dir, &mnt);
		else {
			mntFILE *mfp;

			lock_mtab();

			mfp = my_setmntent(MOUNTED, "a+");
			if (mfp == NULL || mfp->mntent_fp == NULL) {
				com_err(progname, OCFS2_ET_IO, "%s, %s",
					MOUNTED, strerror(errno));
			} else {
				if ((my_addmntent (mfp, &mnt)) == 1) {
					com_err(progname, OCFS2_ET_IO, "%s, %s",
						MOUNTED, strerror(errno));
				}
			}
			my_endmntent(mfp);
			unlock_mtab();
		}
	}
	my_free(mnt.mnt_fsname);
	my_free(mnt.mnt_dir);
}

PhpFile *FileRepository::readFile(const std::string &name, const struct stat &s) {
  vector<StaticStatementPtr> sts;
  const char *canoname = canonicalize(name);
  StatementPtr stmt = Parser::parseFile(canoname, sts);
  if (stmt) {
    uint lock = hash_string(canoname) & 127;
    PhpFile *p = new PhpFile(stmt, sts, s_locks[lock], s);
    return p;
  }
  return NULL;
}

// Get the canonical representation from a WString
char *
UTL_WString::get_canonical_rep()
{
   if (c_str == NULL)
   {
      c_str = new char [alloced];
      canonicalize();
   }

   return c_str;
}