/*
 * util_parse_add_part -- (internal) add a new part file to the replica info
 */
static int
util_parse_add_part(struct pool_set *set, const char *path, size_t filesize)
{
	LOG(3, "set %p path %s filesize %zu", set, path, filesize);

	ASSERTne(set, NULL);

	struct pool_replica *rep = set->replica[set->nreplicas - 1];
	ASSERTne(rep, NULL);

	/* XXX - pre-allocate space for X parts, and reallocate every X parts */
	rep = Realloc(rep, sizeof (struct pool_replica) +
			(rep->nparts + 1) * sizeof (struct pool_set_part));
	if (rep == NULL) {
		ERR("!Realloc");
		return -1;
	}
	set->replica[set->nreplicas - 1] = rep;

	unsigned p = rep->nparts++;

	rep->part[p].path = path;
	rep->part[p].filesize = filesize;
	rep->part[p].fd = -1;
	rep->part[p].addr = NULL;
	rep->part[p].created = 0;

	return 0;
}

/*
 * util_parse_add_replica -- (internal) add a new replica to the pool set info
 */
static int
util_parse_add_replica(struct pool_set **setp)
{
	LOG(3, "setp %p", setp);

	ASSERTne(setp, NULL);

	struct pool_set *set = *setp;
	ASSERTne(set, NULL);

	set = Realloc(set, sizeof (struct pool_set) +
			(set->nreplicas + 1) * sizeof (struct pool_replica *));
	if (set == NULL) {
		ERR("!Realloc");
		return -1;
	}
	*setp = set;

	struct pool_replica *rep;
	rep = Malloc(sizeof (struct pool_replica));
	if (rep == NULL) {
		ERR("!Malloc");
		return -1;
	}

	unsigned r = set->nreplicas++;

	set->replica[r] = rep;

	rep->nparts = 0;
	rep->repsize = 0;

	return 0;
}

/*
 * util_parse_add_remote_replica -- (internal) add a new remote replica
 *                                  to the pool set info
 */
static int
util_parse_add_remote_replica(struct pool_set **setp, char *node_addr,
				char *pool_desc)
{
	LOG(3, "setp %p node_addr %s pool_desc %s", setp, node_addr, pool_desc);

	ASSERTne(setp, NULL);
	ASSERTne(node_addr, NULL);
	ASSERTne(pool_desc, NULL);

	int ret = util_parse_add_replica(setp);
	if (ret != 0)
		return ret;

	/* a remote replica has one 'fake' part */
	ret = util_parse_add_part(*setp, NULL, 0);
	if (ret != 0)
		return ret;

	struct pool_set *set = *setp;
	struct pool_replica *rep = set->replica[set->nreplicas - 1];
	ASSERTne(rep, NULL);

	rep->remote = Zalloc(sizeof(struct remote_replica));
	if (rep->remote == NULL) {
		ERR("!Malloc");
		return -1;
	}
	rep->remote->node_addr = node_addr;
	rep->remote->pool_desc = pool_desc;
	set->remote = 1;

	return 0;
}

/*
 * constructor_zrealloc -- (internal) constructor for pmemobj_zrealloc
 */
static void
constructor_zrealloc(PMEMobjpool *pop, void *ptr, void *arg)
{
	LOG(3, "pop %p ptr %p arg %p", pop, ptr, arg);

	ASSERTne(ptr, NULL);
	ASSERTne(arg, NULL);

	struct carg_realloc *carg = arg;
	struct oob_header *pobj = OOB_HEADER_FROM_PTR(ptr);

	if (ptr != carg->ptr) {
		size_t cpy_size = carg->new_size > carg->old_size ?
			carg->old_size : carg->new_size;

		pop->memcpy_persist(pop, ptr, carg->ptr, cpy_size);

		pobj->data.internal_type = TYPE_ALLOCATED;
		pobj->data.user_type = carg->user_type;
		pop->persist(pop, &pobj->data.internal_type,
		/* there's no padding between these, so we can add sizes */
			sizeof (pobj->data.internal_type) +
			sizeof (pobj->data.user_type));

		VALGRIND_DO_MAKE_MEM_NOACCESS(pop, &pobj->data.padding,
				sizeof (pobj->data.padding));
	}

	if (carg->new_size > carg->old_size) {
		size_t grow_len = carg->new_size - carg->old_size;
		void *new_data_ptr = (void *)((uintptr_t)ptr + carg->old_size);

		pop->memset_persist(pop, new_data_ptr, 0, grow_len);
	}
}

/*
 * constructor_zrealloc_root -- (internal) constructor for pmemobj_root
 */
static void
constructor_zrealloc_root(PMEMobjpool *pop, void *ptr,
	size_t usable_size, void *arg)
{
	LOG(3, "pop %p ptr %p arg %p", pop, ptr, arg);

	ASSERTne(ptr, NULL);
	ASSERTne(arg, NULL);

	struct carg_realloc *carg = arg;

	VALGRIND_ADD_TO_TX(OOB_HEADER_FROM_PTR(ptr),
		usable_size + OBJ_OOB_SIZE);

	constructor_realloc(pop, ptr, usable_size, arg);

	/* activate the padding redzone */
	VALGRIND_DO_MAKE_MEM_NOACCESS(pop,
		&OOB_HEADER_FROM_PTR(ptr)->data.padding,
		sizeof (OOB_HEADER_FROM_PTR(ptr)->data.padding));

	if (carg->constructor)
		carg->constructor(pop, ptr, carg->arg);

	VALGRIND_REMOVE_FROM_TX(OOB_HEADER_FROM_PTR(ptr),
		carg->new_size + OBJ_OOB_SIZE);
}

static void *
do_test(void *arg)
{
	int **bufs = malloc(NBUFS * sizeof (void *));
	ASSERTne(bufs, NULL);

	size_t *sizes = malloc(NBUFS * sizeof (size_t));
	ASSERTne(sizes, NULL);

	for (int j = 0; j < NBUFS; j++) {
		sizes[j] = sizeof (int) + 64 * (rand() % 100);
		bufs[j] = malloc(sizes[j]);
		ASSERTne(bufs[j], NULL);
	}

	for (int j = 0; j < NBUFS; j++) {
		ASSERT(malloc_usable_size(bufs[j]) >= sizes[j]);
		free(bufs[j]);
	}

	free(sizes);
	free(bufs);

	return NULL;
}

/*
 * constructor_alloc_bytype -- (internal) constructor for obj_alloc_construct
 */
static void
constructor_alloc_bytype(PMEMobjpool *pop, void *ptr,
	size_t usable_size, void *arg)
{
	LOG(3, "pop %p ptr %p arg %p", pop, ptr, arg);

	ASSERTne(ptr, NULL);
	ASSERTne(arg, NULL);

	struct oob_header *pobj = OOB_HEADER_FROM_PTR(ptr);
	struct carg_bytype *carg = arg;

	pobj->data.internal_type = TYPE_ALLOCATED;
	pobj->data.user_type = carg->user_type;
	pop->flush(pop, &pobj->data.internal_type,
		/* there's no padding between these, so we can add sizes */
		sizeof (pobj->data.internal_type) +
		sizeof (pobj->data.user_type));

	if (carg->zero_init)
		pop->memset_persist(pop, ptr, 0, usable_size);
	else
		pop->drain(pop);

	VALGRIND_DO_MAKE_MEM_NOACCESS(pop, &pobj->data.padding,
			sizeof (pobj->data.padding));

	if (carg->constructor)
		carg->constructor(pop, ptr, carg->arg);
}

int
main(int argc, char *argv[])
{
	char *dir = NULL;
	void *mem_pool = NULL;
	VMEM *vmp;

	START(argc, argv, "vmem_check");

	if (argc == 2) {
		dir = argv[1];
	} else if (argc > 2) {
		FATAL("usage: %s [directory]", argv[0]);
	}

	if (dir == NULL) {
		/* allocate memory for function vmem_create_in_region() */
		mem_pool = MMAP(NULL, VMEM_MIN_POOL*2, PROT_READ|PROT_WRITE,
					MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);

		vmp = vmem_create_in_region(mem_pool, VMEM_MIN_POOL);
		if (vmp == NULL)
			FATAL("!vmem_create_in_region");
	} else {
		vmp = vmem_create(dir, VMEM_MIN_POOL);
		if (vmp == NULL)
			FATAL("!vmem_create");
	}

	ASSERTeq(1, vmem_check(vmp));

	/* create pool in this same memory region */
	if (dir == NULL) {
		unsigned long Pagesize = (unsigned long) sysconf(_SC_PAGESIZE);
		void *mem_pool2 = (void *)(((uintptr_t)mem_pool +
			VMEM_MIN_POOL/2) & ~(Pagesize-1));

		VMEM *vmp2 = vmem_create_in_region(mem_pool2,
			VMEM_MIN_POOL);

		if (vmp2 == NULL)
			FATAL("!vmem_create_in_region");

		/* detect memory range collision */
		ASSERTne(1, vmem_check(vmp));
		ASSERTne(1, vmem_check(vmp2));

		vmem_delete(vmp2);

		ASSERTne(1, vmem_check(vmp2));
	}

	vmem_delete(vmp);

	/* for vmem_create() memory unmapped after delete pool */
	if (!dir)
		ASSERTne(1, vmem_check(vmp));

	DONE(NULL);
}

/*
 * lane_init -- (internal) initializes a single lane runtime variables
 */
static int
lane_init(PMEMobjpool *pop, struct lane *lane, struct lane_layout *layout,
		pthread_mutex_t *mtx, pthread_mutexattr_t *attr)
{
	ASSERTne(lane, NULL);
	ASSERTne(mtx, NULL);
	ASSERTne(attr, NULL);

	int err;

	util_mutex_init(mtx, attr);

	lane->lock = mtx;

	int i;
	for (i = 0; i < MAX_LANE_SECTION; ++i) {
		lane->sections[i].runtime = NULL;
		lane->sections[i].layout = &layout->sections[i];
		err = Section_ops[i]->construct(pop, &lane->sections[i]);
		if (err != 0) {
			ERR("!lane_construct_ops %d", i);
			goto error_section_construct;
		}
	}

	return 0;

error_section_construct:
	for (i = i - 1; i >= 0; --i)
		Section_ops[i]->destruct(pop, &lane->sections[i]);

	util_mutex_destroy(lane->lock);
	return err;
}

/*
 * constructor_alloc_root -- (internal) constructor for obj_alloc_root
 */
static void
constructor_alloc_root(PMEMobjpool *pop, void *ptr, void *arg)
{
	LOG(3, "pop %p ptr %p arg %p", pop, ptr, arg);

	ASSERTne(ptr, NULL);
	ASSERTne(arg, NULL);

	struct oob_header *ro = OOB_HEADER_FROM_PTR(ptr);
	struct carg_root *carg = arg;

	/* temporarily add atomic root allocation to pmemcheck transaction */
	VALGRIND_ADD_TO_TX(ro, OBJ_OOB_SIZE + carg->size);

	if (carg->constructor)
		carg->constructor(pop, ptr, carg->arg);
	else
		pop->memset_persist(pop, ptr, 0, carg->size);

	ro->data.internal_type = TYPE_ALLOCATED;
	ro->data.user_type = POBJ_ROOT_TYPE_NUM;
	ro->size = carg->size;

	VALGRIND_REMOVE_FROM_TX(ro, OBJ_OOB_SIZE + carg->size);

	pop->persist(pop, &ro->size,
		/* there's no padding between these, so we can add sizes */
		sizeof (ro->size) + sizeof (ro->data.internal_type) +
		sizeof (ro->data.user_type));

	VALGRIND_DO_MAKE_MEM_NOACCESS(pop, &ro->data.padding,
			sizeof (ro->data.padding));
}

int
main(int argc, char *argv[])
{
	char *dir = NULL;
	void *mem_pool = NULL;
	VMEM *vmp;

	START(argc, argv, "vmem_check");

	if (argc == 2) {
		dir = argv[1];
	} else if (argc > 2) {
		FATAL("usage: %s [directory]", argv[0]);
	}

	if (dir == NULL) {
		/* allocate memory for function vmem_create_in_region() */
		mem_pool = MMAP_ANON_ALIGNED(VMEM_MIN_POOL * 2, 4 << 20);

		vmp = vmem_create_in_region(mem_pool, VMEM_MIN_POOL);
		if (vmp == NULL)
			FATAL("!vmem_create_in_region");
	} else {
		vmp = vmem_create(dir, VMEM_MIN_POOL);
		if (vmp == NULL)
			FATAL("!vmem_create");
	}

	ASSERTeq(1, vmem_check(vmp));

	/* create pool in this same memory region */
	if (dir == NULL) {
		void *mem_pool2 = (void *)(((uintptr_t)mem_pool +
			VMEM_MIN_POOL / 2) & ~(Ut_pagesize - 1));

		VMEM *vmp2 = vmem_create_in_region(mem_pool2,
			VMEM_MIN_POOL);

		if (vmp2 == NULL)
			FATAL("!vmem_create_in_region");

		/* detect memory range collision */
		ASSERTne(1, vmem_check(vmp));
		ASSERTne(1, vmem_check(vmp2));

		vmem_delete(vmp2);

		ASSERTne(1, vmem_check(vmp2));
	}

	vmem_delete(vmp);

	/* for vmem_create() memory unmapped after delete pool */
	if (!dir)
		ASSERTne(1, vmem_check(vmp));

	DONE(NULL);
}

int
main(int argc, char *argv[])
{
	const int test_value = 123456;
	char *dir = NULL;
	void *mem_pool = NULL;
	VMEM *vmp;

	START(argc, argv, "vmem_realloc");

	if (argc == 2) {
		dir = argv[1];
	} else if (argc > 2) {
		FATAL("usage: %s [directory]", argv[0]);
	}

	if (dir == NULL) {
		/* allocate memory for function vmem_create_in_region() */
		mem_pool = MMAP(NULL, VMEM_MIN_POOL, PROT_READ|PROT_WRITE,
					MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);

		vmp = vmem_create_in_region(mem_pool, VMEM_MIN_POOL);
		if (vmp == NULL)
			FATAL("!vmem_create_in_region");
	} else {
		vmp = vmem_create(dir, VMEM_MIN_POOL);
		if (vmp == NULL)
			FATAL("!vmem_create");
	}

	int *test = vmem_realloc(vmp, NULL, sizeof (int));
	ASSERTne(test, NULL);

	test[0] = test_value;
	ASSERTeq(test[0], test_value);

	/* check that pointer came from mem_pool */
	if (dir == NULL) {
		ASSERTrange(test, mem_pool, VMEM_MIN_POOL);
	}

	test = vmem_realloc(vmp, test, sizeof (int) * 10);
	ASSERTne(test, NULL);
	ASSERTeq(test[0], test_value);
	test[1] = test_value;
	test[9] = test_value;

	/* check that pointer came from mem_pool */
	if (dir == NULL) {
		ASSERTrange(test, mem_pool, VMEM_MIN_POOL);
	}

	vmem_free(vmp, test);

	vmem_delete(vmp);

	DONE(NULL);
}

/*
 * alloc_class_find_min_frag -- searches for an existing allocation
 * class that will provide the smallest internal fragmentation for the given
 * size.
 */
static struct alloc_class *
alloc_class_find_min_frag(struct alloc_class_collection *ac, size_t n)
{
	LOG(10, NULL);

	struct alloc_class *best_c = NULL;
	size_t lowest_waste = SIZE_MAX;

	ASSERTne(n, 0);

	/*
	 * Start from the largest buckets in order to minimize unit size of
	 * allocated memory blocks.
	 */
	for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) {
		struct alloc_class *c = ac->aclasses[i];

		/* can't use alloc classes /w no headers by default */
		if (c == NULL || c->header_type == HEADER_NONE)
			continue;

		size_t real_size = n + header_type_to_size[c->header_type];

		size_t units = CALC_SIZE_IDX(c->unit_size, real_size);

		/* can't exceed the maximum allowed run unit max */
		if (c->type == CLASS_RUN && units > RUN_UNIT_MAX_ALLOC)
			continue;

		if (c->unit_size * units == real_size)
			return c;

		size_t waste = (c->unit_size * units) - real_size;

		/*
		 * If we assume that the allocation class is only ever going to
		 * be used with exactly one size, the effective internal
		 * fragmentation would be increased by the leftover
		 * memory at the end of the run.
		 */
		if (c->type == CLASS_RUN) {
			size_t wasted_units = c->run.nallocs % units;
			size_t wasted_bytes = wasted_units * c->unit_size;
			size_t waste_avg_per_unit = wasted_bytes /
				c->run.nallocs;

			waste += waste_avg_per_unit;
		}

		if (best_c == NULL || lowest_waste > waste) {
			best_c = c;
			lowest_waste = waste;
		}
	}

	ASSERTne(best_c, NULL);
	return best_c;
}

int
main(int argc, char *argv[])
{
	void *handle = NULL;
	void *ptr;

	START(argc, argv, "vmmalloc_init");

	if (argc > 2)
		FATAL("usage: %s [d|l]", argv[0]);

	if (argc == 2) {
		switch (argv[1][0]) {
		case 'd':
			OUT("deep binding");
			handle = dlopen("./libtest.so",
				RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND);
			break;
		case 'l':
			OUT("lazy binding");
			handle = dlopen("./libtest.so", RTLD_LAZY);
			break;
		default:
			FATAL("usage: %s [d|l]", argv[0]);
		}

		if (handle == NULL)
			OUT("dlopen: %s", dlerror());
		ASSERTne(handle, NULL);

		Falloc = dlsym(handle, "falloc");
		ASSERTne(Falloc, NULL);
	}

	ptr = malloc(4321);
	free(ptr);

	if (argc == 2) {
		/*
		 * NOTE: falloc calls malloc internally.
		 * If libtest is loaded with RTLD_DEEPBIND flag, then it will
		 * use its own lookup scope in preference to global symbols
		 * from already loaded (LD_PRELOAD) libvmmalloc.  So, falloc
		 * will call the stock libc's malloc.
		 * However, since we override the malloc hooks, a call to libc's
		 * malloc will be redirected to libvmmalloc anyway, and the
		 * memory can be safely reclaimed using libvmmalloc's free.
		 */
		ptr = Falloc(4321, 0xaa);
		free(ptr);
	}

	DONE(NULL);
}

/*
 * constructor_zalloc -- (internal) constructor for pmemobj_zalloc
 */
static void
constructor_zalloc(PMEMobjpool *pop, void *ptr, void *arg)
{
	LOG(3, "pop %p ptr %p arg %p", pop, ptr, arg);

	ASSERTne(ptr, NULL);
	ASSERTne(arg, NULL);

	struct carg_alloc *carg = arg;

	pop->memset_persist(pop, ptr, 0, carg->size);
}

/*
 * constructor_strdup -- (internal) constructor of pmemobj_strndup
 */
static void
constructor_strdup(PMEMobjpool *pop, void *ptr, void *arg)
{
	LOG(3, "pop %p ptr %p arg %p", pop, ptr, arg);

	ASSERTne(ptr, NULL);
	ASSERTne(arg, NULL);

	struct carg_strdup *carg = arg;

	/* copy string */
	pop->memcpy_persist(pop, ptr, carg->s, carg->size);
}

/*
 * alloc_class_find_min_frag -- searches for an existing allocation
 * class that will provide the smallest internal fragmentation for the given
 * size.
 */
static struct alloc_class *
alloc_class_find_min_frag(struct alloc_class_collection *ac, size_t n)
{
	LOG(10, NULL);

	struct alloc_class *best_c = NULL;
	size_t best_frag_d = SIZE_MAX;
	size_t best_frag_r = SIZE_MAX;

	ASSERTne(n, 0);

	/*
	 * Start from the largest buckets in order to minimize unit size of
	 * allocated memory blocks.
	 */
	for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) {
		struct alloc_class *c = ac->aclasses[i];

		/* can't use alloc classes /w no headers by default */
		if (c == NULL || c->header_type == HEADER_NONE)
			continue;

		size_t real_size = n + header_type_to_size[c->header_type];

		size_t units = CALC_SIZE_IDX(c->unit_size, real_size);

		/* can't exceed the maximum allowed run unit max */
		if (units > RUN_UNIT_MAX_ALLOC)
			continue;

		if (c->unit_size * units == real_size)
			return c;

		ASSERT(c->unit_size * units > real_size);

		size_t frag_d = (c->unit_size * units) / real_size;
		size_t frag_r = (c->unit_size * units) % real_size;
		if (best_c == NULL || frag_d < best_frag_d ||
			(frag_d == best_frag_d && frag_r < best_frag_r)) {
			best_c = c;
			best_frag_d = frag_d;
			best_frag_r = frag_r;
		}
	}

	ASSERTne(best_c, NULL);
	return best_c;
}

/*
 * pool_test -- test pool
 *
 * This function creates a memory pool in a file (if dir is not NULL),
 * or in RAM (if dir is NULL) and allocates memory for the test.
 */
void
pool_test(const char *dir)
{
	VMEM *vmp = NULL;

	if (dir != NULL) {
		vmp = vmem_pool_create(dir, VMEM_MIN_POOL);
	} else {
		vmp = vmem_pool_create_in_region(mem_pool, VMEM_MIN_POOL);
	}

	if (expect_create_pool == 0) {
		ASSERTeq(vmp, NULL);
		DONE(NULL);
	} else {
		if (vmp == NULL) {
			if (dir == NULL) {
				FATAL("!vmem_pool_create_in_region");
			} else {
				FATAL("!vmem_pool_create");
			}
		}
	}

	char *test = vmem_malloc(vmp, strlen(TEST_STRING_VALUE) + 1);
	ASSERTne(test, NULL);

	strcpy(test, TEST_STRING_VALUE);
	ASSERTeq(strcmp(test, TEST_STRING_VALUE), 0);

	vmem_free(vmp, test);

	vmem_pool_delete(vmp);
}

/*
 * alloc_class_assign_by_size -- (internal) chooses the allocation class that
 *	best approximates the provided size
 */
static struct alloc_class *
alloc_class_assign_by_size(struct alloc_class_collection *ac,
	size_t size)
{
	LOG(10, NULL);

	size_t class_map_index = SIZE_TO_CLASS_MAP_INDEX(size,
		ac->granularity);

	struct alloc_class *c = alloc_class_find_min_frag(ac,
		class_map_index * ac->granularity);
	ASSERTne(c, NULL);

	/*
	 * We don't lock this array because locking this section here and then
	 * bailing out if someone else was faster would be still slower than
	 * just calculating the class and failing to assign the variable.
	 * We are using a compare and swap so that helgrind/drd don't complain.
	 */
	util_bool_compare_and_swap64(
		&ac->class_map_by_alloc_size[class_map_index],
		MAX_ALLOCATION_CLASSES, c->id);

	return c;
}

/*
 * util_file_create -- create a new memory pool file
 */
int
util_file_create(const char *path, size_t size, size_t minsize)
{
	LOG(3, "path %s size %zu minsize %zu", path, size, minsize);

	ASSERTne(size, 0);

	if (size < minsize) {
		ERR("size %zu smaller than %zu", size, minsize);
		errno = EINVAL;
		return -1;
	}

	if (((off_t)size) < 0) {
		ERR("invalid size (%zu) for off_t", size);
		errno = EFBIG;
		return -1;
	}

	int fd;
	int mode;
	int flags = O_RDWR | O_CREAT | O_EXCL;
#ifndef _WIN32
	mode = 0;
#else
	mode = S_IWRITE | S_IREAD;
	flags |= O_BINARY;
#endif

	/*
	 * Create file without any permission. It will be granted once
	 * initialization completes.
	 */
	if ((fd = open(path, flags, mode)) < 0) {
		ERR("!open %s", path);
		return -1;
	}

	if ((errno = posix_fallocate(fd, 0, (off_t)size)) != 0) {
		ERR("!posix_fallocate");
		goto err;
	}

	/* for windows we can't flock until after we fallocate */
	if (flock(fd, LOCK_EX | LOCK_NB) < 0) {
		ERR("!flock");
		goto err;
	}

	return fd;

err:
	LOG(4, "error clean up");
	int oerrno = errno;
	if (fd != -1)
		(void) close(fd);
	unlink(path);
	errno = oerrno;
	return -1;
}