static int
tika_get_http_client_url(struct mail_user *user, struct http_url **http_url_r)
{
    struct fts_parser_tika_user *tuser = TIKA_USER_CONTEXT(user);
    struct http_client_settings http_set;
    const char *url, *error;

    url = mail_user_plugin_getenv(user, "fts_tika");
    if (url == NULL) {
        /* fts_tika disabled */
        return -1;
    }

    if (tuser != NULL) {
        *http_url_r = tuser->http_url;
        return *http_url_r == NULL ? -1 : 0;
    }

    tuser = p_new(user->pool, struct fts_parser_tika_user, 1);
    MODULE_CONTEXT_SET(user, fts_parser_tika_user_module, tuser);

    if (http_url_parse(url, NULL, 0, user->pool,
                       &tuser->http_url, &error) < 0) {
        i_error("fts_tika: Failed to parse HTTP url %s: %s", url, error);
        return -1;
    }

    if (tika_http_client == NULL) {
        memset(&http_set, 0, sizeof(http_set));
        http_set.max_idle_time_msecs = 100;
        http_set.max_parallel_connections = 1;
        http_set.max_pipelined_requests = 1;
        http_set.max_redirects = 1;
        http_set.max_attempts = 3;
        http_set.connect_timeout_msecs = 5*1000;
        http_set.request_timeout_msecs = 60*1000;
        http_set.debug = user->mail_debug;
        tika_http_client = http_client_init(&http_set);
    }
    *http_url_r = tuser->http_url;
    return 0;
}

static int
imap_hibernate_client_input_args(struct connection *conn,
				 const char *const *args, int fd, pool_t pool)
{
	struct imap_hibernate_client *client =
		(struct imap_hibernate_client *)conn;
	struct imap_client_state state;
	const char *error;

	if (imap_hibernate_client_parse_input(args, pool, &state, &error) < 0) {
		i_error("Failed to parse client input: %s", error);
		o_stream_send_str(conn->output, t_strdup_printf(
			"-Failed to parse client input: %s\n", error));
		return -1;
	}
	client->imap_client = imap_client_create(fd, &state);
	/* the transferred imap client fd is now counted as the client. */
	client->imap_client_created = TRUE;
	return state.have_notify_fd ? 0 : 1;
}

static bool
snarf_box_find(struct mail_user *user, struct mailbox_list **list_r,
	       const char **name_r)
{
	struct mail_namespace *snarf_ns;
	const char *snarf_name;

	snarf_name = mail_user_plugin_getenv(user, "snarf");
	if (snarf_name == NULL)
		return FALSE;
	if (!uni_utf8_str_is_valid(snarf_name)) {
		i_error("snarf: Mailbox name not UTF-8: %s", snarf_name);
		return FALSE;
	}

	snarf_ns = mail_namespace_find(user->namespaces, snarf_name);
	*list_r = snarf_ns->list;
	*name_r = snarf_name;
	return TRUE;
}

void client_destroy(struct client **_client)
{
	struct client *client = *_client;

	*_client = NULL;

	DLLIST_REMOVE(&clients, client);
	if (client->io != NULL)
		io_remove(&client->io);
	i_stream_destroy(&client->input);
	o_stream_destroy(&client->output);
	if (close(client->fd) < 0)
		i_error("close(client) failed: %m");

	client_unref_iters(client);
	pool_unref(&client->cmd_pool);
	i_free(client);

	master_service_client_connection_destroyed(master_service);
}

static void login_connection_input(struct login_connection *conn)
{
	struct ostream *output;
	unsigned char buf[4096];
	ssize_t ret;

	ret = read(conn->fd, buf, sizeof(buf));
	if (ret <= 0) {
		if (ret < 0) {
			if (errno == EAGAIN)
				return;
			if (errno != ECONNRESET)
				i_error("read(login connection) failed: %m");
		}
		login_connection_deinit(&conn);
		return;
	}
	output = auth_connection_get_output(conn->auth);
	o_stream_nsend(output, buf, ret);
}

static struct master_login_auth_request *
master_login_auth_lookup_request(struct master_login_auth *auth,
				 unsigned int id)
{
	struct master_login_auth_request *request;

	request = hash_table_lookup(auth->requests, POINTER_CAST(id));
	if (request == NULL) {
		i_error("Auth server sent reply with unknown ID %u", id);
		return NULL;
	}
	master_login_auth_request_remove(auth, request);
	if (request->aborted) {
		request->callback(NULL, MASTER_AUTH_ERRMSG_INTERNAL_FAILURE,
				  request->context);
		i_free(request);
		return NULL;
	}
	return request;
}

static int
master_login_auth_connect(struct master_login_auth *auth)
{
	int fd;

	i_assert(auth->fd == -1);

	fd = net_connect_unix_with_retries(auth->auth_socket_path, 1000);
	if (fd == -1) {
		i_error("net_connect_unix(%s) failed: %m",
			auth->auth_socket_path);
		return -1;
	}
	auth->fd = fd;
	auth->input = i_stream_create_fd(fd, AUTH_MAX_INBUF_SIZE, FALSE);
	auth->output = o_stream_create_fd(fd, (size_t)-1, FALSE);
	o_stream_set_no_error_handling(auth->output, TRUE);
	auth->io = io_add(fd, IO_READ, master_login_auth_input, auth);
	return 0;
}

static void lmtp_client_dns_done(const struct dns_lookup_result *result,
				 struct lmtp_client *client)
{
	client->dns_lookup = NULL;

	if (result->ret != 0) {
		i_error("lmtp client: DNS lookup of %s failed: %s",
			client->host, result->error);
		if (client->running) {
			lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE
					 " (DNS lookup)");
		}
	} else {
		client->ip = result->ips[0];
		if (lmtp_client_connect(client) < 0 && client->running) {
			lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE
					 " (connect)");
		}
	}
}

const char *sysinfo_get(const char *mail_location)
{
	const char *distro = "", *fs, *uname_info = "";
#ifdef HAVE_SYS_UTSNAME_H
	struct utsname u;

	if (uname(&u) < 0)
		i_error("uname() failed: %m");
	else {
		uname_info = t_strdup_printf("%s %s %s",
					     u.sysname, u.release, u.machine);
	}
	if (strcmp(u.sysname, "Linux") == 0)
		distro = distro_get();
#endif
	fs = filesystem_get(mail_location);
	if (*uname_info == '\0' && *distro == '\0' && *fs == '\0')
		return "";
	return t_strdup_printf("OS: %s %s %s %s %s", u.sysname, u.release, u.machine, distro, fs);
}

static int
cmd_fts_optimize_run(struct doveadm_mail_cmd_context *ctx,
		     struct mail_user *user)
{
	const char *ns_prefix = ctx->args[0];
	struct mail_namespace *ns;
	struct fts_backend *backend;

	if (fts_namespace_find(user, ns_prefix, &ns) < 0) {
		doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND);
		return -1;
	}
	backend = fts_list_backend(ns->list);
	if (fts_backend_optimize(backend) < 0) {
		i_error("fts optimize failed");
		doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP);
		return -1;
	}
	return 0;
}

int fts_indexer_deinit(struct fts_indexer_context **_ctx)
{
	struct fts_indexer_context *ctx = *_ctx;
	int ret = ctx->failed ? -1 : 0;

	*_ctx = NULL;

	i_stream_destroy(&ctx->input);
	if (close(ctx->fd) < 0)
		i_error("close(%s) failed: %m", ctx->path);
	if (ctx->notified) {
		/* we notified at least once */
		ctx->box->storage->callbacks.
			notify_ok(ctx->box, "Mailbox indexing finished",
				  ctx->box->storage->callback_context);
	}
	i_free(ctx->path);
	i_free(ctx);
	return ret;
}

static int fetch_stream_send_direct(struct imap_fetch_context *ctx)
{
	off_t ret;

	o_stream_set_max_buffer_size(ctx->client->output, 0);
	ret = o_stream_send_istream(ctx->client->output, ctx->cur_input);
	o_stream_set_max_buffer_size(ctx->client->output, (size_t)-1);

	if (ret < 0)
		return -1;

	ctx->cur_offset += ret;

	if (ctx->cur_append_eoh && ctx->cur_offset + 2 == ctx->cur_size) {
		/* Netscape missing EOH workaround. */
		if (o_stream_send(ctx->client->output, "\r\n", 2) < 0)
			return -1;
		ctx->cur_offset += 2;
		ctx->cur_append_eoh = FALSE;
	}

	if (ctx->cur_offset != ctx->cur_size) {
		/* unfinished */
		if (!i_stream_have_bytes_left(ctx->cur_input)) {
			/* Input stream gave less data than expected */
			i_error("FETCH %s for mailbox %s UID %u "
				"got too little data (copying): "
				"%"PRIuUOFF_T" vs %"PRIuUOFF_T,
				ctx->cur_name, mailbox_get_vname(ctx->mail->box),
				ctx->mail->uid, ctx->cur_offset, ctx->cur_size);
			mail_set_cache_corrupted(ctx->mail,
						 ctx->cur_size_field);
			client_disconnect(ctx->client, "FETCH failed");
			return -1;
		}

		o_stream_set_flush_pending(ctx->client->output, TRUE);
		return 0;
	}
	return 1;
}

static const struct mailbox_info *
quota_mailbox_iter_next(struct quota_mailbox_iter *iter)
{
	struct mail_namespace *const *namespaces;
	const struct mailbox_info *info;
	unsigned int count;

	if (iter->iter == NULL) {
		namespaces = array_get(&iter->root->quota->namespaces, &count);
		if (iter->ns_idx >= count)
			return NULL;

		iter->ns = namespaces[iter->ns_idx++];
		iter->iter = mailbox_list_iter_init(iter->ns->list, "*",
			MAILBOX_LIST_ITER_SKIP_ALIASES |
			MAILBOX_LIST_ITER_RETURN_NO_FLAGS |
			MAILBOX_LIST_ITER_NO_AUTO_BOXES);
	}
	while ((info = mailbox_list_iter_next(iter->iter)) != NULL) {
		if ((info->flags & (MAILBOX_NONEXISTENT |
				    MAILBOX_NOSELECT)) == 0)
			return info;
	}
	if (mailbox_list_iter_deinit(&iter->iter) < 0) {
		i_error("quota: Listing namespace '%s' failed: %s",
			iter->ns->prefix,
			mailbox_list_get_last_error(iter->ns->list, NULL));
		iter->failed = TRUE;
	}
	if (iter->ns->prefix_len > 0 &&
	    (iter->ns->prefix_len != 6 ||
	     strncasecmp(iter->ns->prefix, "INBOX", 5) != 0)) {
		/* if the namespace prefix itself exists, count it also */
		iter->info.ns = iter->ns;
		iter->info.vname = t_strndup(iter->ns->prefix,
					     iter->ns->prefix_len-1);
		return &iter->info;
	}
	/* try the next namespace */
	return quota_mailbox_iter_next(iter);
}

static struct fs_file *
fs_sis_file_init(struct fs *_fs, const char *path,
		 enum fs_open_mode mode, enum fs_open_flags flags)
{
	struct sis_fs *fs = (struct sis_fs *)_fs;
	struct sis_fs_file *file;
	const char *dir, *hash;

	file = i_new(struct sis_fs_file, 1);
	file->file.fs = _fs;
	file->file.path = i_strdup(path);
	file->fs = fs;
	file->open_mode = mode;
	if (mode == FS_OPEN_MODE_APPEND) {
		fs_set_error(_fs, "APPEND mode not supported");
		return &file->file;
	}

	if (fs_sis_path_parse(_fs, path, &dir, &hash) < 0) {
		fs_set_error(_fs, "Invalid path");
		return &file->file;
	}

	/* if hashes/<hash> already exists, open it */
	file->hash_path = i_strdup_printf("%s/"HASH_DIR_NAME"/%s", dir, hash);
	file->hash_file = fs_file_init(_fs->parent, file->hash_path,
				       FS_OPEN_MODE_READONLY);

	file->hash_input = fs_read_stream(file->hash_file, IO_BLOCK_SIZE);
	if (i_stream_read(file->hash_input) == -1) {
		/* doesn't exist */
		if (errno != ENOENT) {
			i_error("fs-sis: Couldn't read hash file %s: %m",
				file->hash_path);
		}
		i_stream_destroy(&file->hash_input);
	}

	file->super = fs_file_init(_fs->parent, path, mode | flags);
	return &file->file;
}

int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
				  const struct dsync_mailbox *remote_dsync_box)
{
	enum dsync_mailbox_exporter_flags exporter_flags = 0;
	uint32_t last_common_uid, highest_wanted_uid;
	uint64_t last_common_modseq, last_common_pvt_modseq;

	i_assert(brain->log_scan == NULL);
	i_assert(brain->box_exporter == NULL);

	last_common_uid = brain->mailbox_state.last_common_uid;
	last_common_modseq = brain->mailbox_state.last_common_modseq;
	last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq;
	highest_wanted_uid = last_common_uid == 0 ?
		(uint32_t)-1 : last_common_uid;
	if (dsync_transaction_log_scan_init(brain->box->view,
					    brain->box->view_pvt,
					    highest_wanted_uid,
					    last_common_modseq,
					    last_common_pvt_modseq,
					    &brain->log_scan) < 0) {
		i_error("Failed to read transaction log for mailbox %s",
			mailbox_get_vname(brain->box));
		brain->failed = TRUE;
		return -1;
	}

	if (!brain->mail_requests)
		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS;
	if (remote_dsync_box->have_save_guids &&
	    (brain->local_dsync_box.have_save_guids ||
	     (brain->backup_send && brain->local_dsync_box.have_guids)))
		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;

	brain->box_exporter = brain->backup_recv ? NULL :
		dsync_mailbox_export_init(brain->box, brain->log_scan,
					  last_common_uid,
					  exporter_flags);
	dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box);
	return 0;
}

static int i_stream_seekable_write_failed(struct seekable_istream *sstream)
{
	struct istream_private *stream = &sstream->istream;
	void *data;

	i_assert(sstream->fd != -1);

	stream->max_buffer_size = (size_t)-1;
	data = i_stream_alloc(stream, sstream->write_peak);

	if (pread_full(sstream->fd, data, sstream->write_peak, 0) < 0) {
		i_error("istream-seekable: read(%s) failed: %m", sstream->temp_path);
		memarea_unref(&stream->memarea);
		return -1;
	}
	i_stream_destroy(&sstream->fd_input);
	i_close_fd(&sstream->fd);

	i_free_and_null(sstream->temp_path);
	return 0;
}

static void pop3c_client_connected(struct pop3c_client *client)
{
	int err;

	err = net_geterror(client->fd);
	if (err != 0) {
		i_error("pop3c(%s): connect(%s, %u) failed: %s",
			client->set.host, net_ip2addr(&client->ip),
			client->set.port, strerror(err));
		pop3c_client_disconnect(client);
		return;
	}
	io_remove(&client->io);
	client->io = io_add(client->fd, IO_READ,
			    pop3c_client_prelogin_input, client);

	if (client->set.ssl_mode == POP3C_CLIENT_SSL_MODE_IMMEDIATE) {
		if (pop3c_client_ssl_init(client) < 0)
			pop3c_client_disconnect(client);
	}
}

static void pop3c_client_dot_input(struct pop3c_client *client)
{
	ssize_t ret;

	if (client->to != NULL)
		timeout_reset(client->to);
	while ((ret = i_stream_read(client->dot_input)) > 0 || ret == -2) {
		i_stream_skip(client->dot_input,
			      i_stream_get_data_size(client->dot_input));
	}
	if (ret != 0) {
		i_assert(ret == -1);
		if (client->dot_input->stream_errno != 0) {
			i_error("pop3c(%s): Server disconnected unexpectedly",
				client->set.host);
			pop3c_client_disconnect(client);
		}
		if (client->running)
			io_loop_stop(current_ioloop);
	}
}

static int
dsync_mailbox_tree_add_node(struct dsync_mailbox_tree *tree,
			    const struct mailbox_info *info,
			    struct dsync_mailbox_node **node_r)
{
	struct dsync_mailbox_node *node;

	node = dsync_mailbox_tree_get(tree, info->vname);
	if (node->ns == info->ns)
		;
	else if (node->ns == NULL) {
		i_assert(tree->root.ns == NULL);
		node->ns = info->ns;
	} else {
		i_error("Mailbox '%s' exists in two namespaces: '%s' and '%s'",
			info->vname, node->ns->prefix, info->ns->prefix);
		return -1;
	}
	*node_r = node;
	return 0;
}

static bool
snarf_box_find(struct mail_user *user, struct mailbox_list **list_r,
	       const char **name_r)
{
	struct mail_namespace *snarf_ns;
	const char *snarf_name;

	snarf_name = mail_user_plugin_getenv(user, "snarf");
	if (snarf_name == NULL)
		return FALSE;

	snarf_ns = mail_namespace_find(user->namespaces, &snarf_name);
	if (snarf_ns == NULL) {
		i_error("snarf: Namespace not found for mailbox: %s",
			snarf_name);
		return FALSE;
	}
	*list_r = snarf_ns->list;
	*name_r = snarf_name;
	return TRUE;
}