/*
 * Copyright 2025 Vsevolod Stakhov
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * Implementation of map files handling
 */

#include "config.h"
#include "map.h"
#include "map_private.h"
#include "libserver/http/http_connection.h"
#include "libserver/http/http_private.h"
#include "rspamd.h"
#include "contrib/libev/ev.h"
#include "contrib/uthash/utlist.h"
#include "libutil/str_util.h"
#include "libcryptobox/cryptobox.h"

#include <sodium.h>

#include <worker_util.h>

#ifdef SYS_ZSTD
#include "zstd.h"
#else
#include "contrib/zstd/zstd.h"
#endif

#undef MAP_DEBUG_REFS
#ifdef MAP_DEBUG_REFS
#define MAP_RETAIN(x, t)                                                                                                             \
	do {                                                                                                                             \
		msg_err(G_GNUC_PRETTY_FUNCTION ": " t ": retain ref %p, refcount: %d -> %d", (x), (x)->ref.refcount, (x)->ref.refcount + 1); \
		REF_RETAIN(x);                                                                                                               \
	} while (0)

#define MAP_RELEASE(x, t)                                                                                                             \
	do {                                                                                                                              \
		msg_err(G_GNUC_PRETTY_FUNCTION ": " t ": release ref %p, refcount: %d -> %d", (x), (x)->ref.refcount, (x)->ref.refcount - 1); \
		REF_RELEASE(x);                                                                                                               \
	} while (0)
#else
#define MAP_RETAIN(x, t) REF_RETAIN(x)
#define MAP_RELEASE(x, t) REF_RELEASE(x)
#endif

enum rspamd_map_periodic_opts {
	RSPAMD_MAP_SCHEDULE_NORMAL = 0,
	RSPAMD_MAP_SCHEDULE_ERROR = (1u << 0u),
	RSPAMD_MAP_SCHEDULE_LOCKED = (1u << 1u),
	RSPAMD_MAP_SCHEDULE_INIT = (1u << 2u),
};

static void free_http_cbdata_common(struct http_callback_data *cbd,
									gboolean plan_new);
static void free_http_cbdata_dtor(gpointer p);
static void free_http_cbdata(struct http_callback_data *cbd);
static void rspamd_map_process_periodic(struct map_periodic_cbdata *cbd);
static void rspamd_map_schedule_periodic(struct rspamd_map *map, int how);
static gboolean read_map_file_chunks(struct rspamd_map *map,
									 struct map_cb_data *cbdata,
									 const char *fname,
									 gsize len,
									 goffset off);
static gboolean rspamd_map_save_http_cached_file(struct rspamd_map *map,
												 struct rspamd_map_backend *bk,
												 struct http_map_data *htdata,
												 const unsigned char *data,
												 gsize len);
static gboolean rspamd_map_update_http_cached_file(struct rspamd_map *map,
												   struct rspamd_map_backend *bk,
												   struct http_map_data *htdata);

static inline void
rspamd_map_cache_file_path(struct rspamd_map_backend *bk,
						   const char *cache_dir,
						   char *path, gsize pathlen)
{
	unsigned char digest[rspamd_cryptobox_HASHBYTES];

	rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0);
	rspamd_snprintf(path, pathlen, "%s%c%*xs.map", cache_dir,
					G_DIR_SEPARATOR, 20, digest);
}

unsigned int rspamd_map_log_id = (unsigned int) -1;
RSPAMD_CONSTRUCTOR(rspamd_map_log_init)
{
	rspamd_map_log_id = rspamd_logger_add_debug_module("map");
}

/**
 * Write HTTP request with proper cache validation headers
 * Uses ETags (If-None-Match) and Last-Modified (If-Modified-Since) for conditional requests
 */
static void
write_http_request(struct http_callback_data *cbd)
{
	char datebuf[128];
	struct rspamd_http_message *msg;

	msg = rspamd_http_new_message(HTTP_REQUEST);
	if (cbd->check) {
		msg->method = HTTP_HEAD;
	}

	msg->url = rspamd_fstring_append(msg->url,
									 cbd->data->path, strlen(cbd->data->path));

	if (cbd->check) {
		if (cbd->data->last_modified != 0) {
			rspamd_http_date_format(datebuf, sizeof(datebuf),
									cbd->data->last_modified);
			rspamd_http_message_add_header(msg, "If-Modified-Since",
										   datebuf);
		}
		if (cbd->data->etag) {
			rspamd_http_message_add_header_len(msg, "If-None-Match",
											   cbd->data->etag->str,
											   cbd->data->etag->len);
		}
	}

	msg->url = rspamd_fstring_append(msg->url, cbd->data->rest,
									 strlen(cbd->data->rest));

	if (cbd->data->userinfo) {
		rspamd_http_message_add_header(msg, "Authorization",
									   cbd->data->userinfo);
	}

	MAP_RETAIN(cbd, "http_callback_data");
	rspamd_http_connection_write_message(cbd->conn,
										 msg,
										 cbd->data->host,
										 NULL,
										 cbd,
										 cbd->timeout);
}

/**
 * Callback for destroying HTTP callback data
 */
static void
free_http_cbdata_common(struct http_callback_data *cbd, gboolean plan_new)
{
	struct map_periodic_cbdata *periodic = cbd->periodic;

	if (cbd->shmem_data) {
		rspamd_http_message_shmem_unref(cbd->shmem_data);
	}

	if (cbd->pk) {
		rspamd_pubkey_unref(cbd->pk);
	}

	if (cbd->conn) {
		rspamd_http_connection_unref(cbd->conn);
		cbd->conn = NULL;
	}

	if (cbd->addrs) {
		rspamd_inet_addr_t *addr;
		unsigned int i;

		PTR_ARRAY_FOREACH(cbd->addrs, i, addr)
		{
			rspamd_inet_address_free(addr);
		}

		g_ptr_array_free(cbd->addrs, TRUE);
	}


	MAP_RELEASE(cbd->bk, "rspamd_map_backend");

	if (periodic) {
		/* Detached in case of HTTP error */
		MAP_RELEASE(periodic, "periodic");
	}

	g_free(cbd);
}

static void
free_http_cbdata(struct http_callback_data *cbd)
{
	cbd->map->tmp_dtor = NULL;
	cbd->map->tmp_dtor_data = NULL;

	free_http_cbdata_common(cbd, TRUE);
}

static void
free_http_cbdata_dtor(gpointer p)
{
	struct http_callback_data *cbd = p;
	struct rspamd_map *map;

	map = cbd->map;
	if (cbd->stage == http_map_http_conn) {
		REF_RELEASE(cbd);
	}
	else {
		/* We cannot terminate DNS requests sent */
		cbd->stage = http_map_terminated;
	}

	msg_warn_map("%s: "
				 "connection with http server is terminated: worker is stopping",
				 map->name);
}

/*
 * HTTP callbacks
 */
static void
http_map_error(struct rspamd_http_connection *conn,
			   GError *err)
{
	struct http_callback_data *cbd = conn->ud;
	struct rspamd_map *map;

	map = cbd->map;

	if (cbd->periodic) {
		cbd->periodic->errored = TRUE;
		msg_err_map("error reading %s(%s): "
					"connection with http server terminated incorrectly: %e",
					cbd->bk->uri,
					cbd->addr ? rspamd_inet_address_to_string_pretty(cbd->addr) : "",
					err);

		rspamd_map_process_periodic(cbd->periodic);
	}

	MAP_RELEASE(cbd, "http_callback_data");
}

static void
rspamd_map_cache_cb(struct ev_loop *loop, ev_timer *w, int revents)
{
	struct rspamd_http_map_cached_cbdata *cache_cbd = (struct rspamd_http_map_cached_cbdata *)
														  w->data;
	struct rspamd_map *map;
	struct http_map_data *data;

	map = cache_cbd->map;
	data = cache_cbd->data;

	if (cache_cbd->gen != cache_cbd->data->gen) {
		/* We have another update, so this cache element is obviously expired */
		/*
		 * Important!: we do not set cache availability to zero here, as there
		 * might be fresh cache
		 */
		msg_info_map("cached data is now expired (gen mismatch %L != %L) for %s; shm name=%s; refcount=%d",
					 cache_cbd->gen, cache_cbd->data->gen, map->name, cache_cbd->shm->shm_name,
					 cache_cbd->shm->ref.refcount);
		MAP_RELEASE(cache_cbd->shm, "rspamd_http_map_cached_cbdata");
		ev_timer_stop(loop, &cache_cbd->timeout);
		g_free(cache_cbd);
	}
	else if (cache_cbd->data->last_checked >= cache_cbd->last_checked) {
		/*
		 * We checked map but we have not found anything more recent,
		 * reschedule cache check
		 */
		if (cache_cbd->map->poll_timeout >
			rspamd_get_calendar_ticks() - cache_cbd->data->last_checked) {
			w->repeat = cache_cbd->map->poll_timeout -
						(rspamd_get_calendar_ticks() - cache_cbd->data->last_checked);
		}
		else {
			w->repeat = cache_cbd->map->poll_timeout;
		}

		if (w->repeat < 0) {
			msg_info_map("cached data for %s has skewed check time: %d last checked, "
						 "%d poll timeout, %.2f diff; shm name=%s; refcount=%d",
						 map->name, (int) cache_cbd->data->last_checked,
						 (int) cache_cbd->map->poll_timeout,
						 (rspamd_get_calendar_ticks() - cache_cbd->data->last_checked),
						 cache_cbd->shm->shm_name,
						 cache_cbd->shm->ref.refcount);
			w->repeat = 0.0;
		}

		cache_cbd->last_checked = cache_cbd->data->last_checked;
		msg_debug_map("cached data is up to date for %s", map->name);
		ev_timer_again(loop, &cache_cbd->timeout);
	}
	else {
		data->cur_cache_cbd = NULL;
		g_atomic_int_set(&data->cache->available, 0);
		msg_info_map("cached data is now expired for %s; shm name=%s; refcount=%d",
					 map->name,
					 cache_cbd->shm->shm_name,
					 cache_cbd->shm->ref.refcount);
		MAP_RELEASE(cache_cbd->shm, "rspamd_http_map_cached_cbdata");
		ev_timer_stop(loop, &cache_cbd->timeout);
		g_free(cache_cbd);
	}
}

/**
 * Calculate next check time with proper priority for different cache validation mechanisms
 * Enforces server-controlled refresh intervals to prevent aggressive client polling
 * @param now current time
 * @param expires time from cache expiration header (0 if not present)
 * @param map_check_interval base polling interval in seconds
 * @param has_etag whether we have ETag for conditional requests
 * @param has_last_modified whether we have Last-Modified for conditional requests
 * @return next check time
 */
static inline time_t
rspamd_http_map_process_next_check(struct rspamd_map *map,
								   struct rspamd_map_backend *bk,
								   time_t now,
								   time_t expires,
								   time_t map_check_interval,
								   gboolean has_etag,
								   gboolean has_last_modified)
{
	static const time_t max_expires_interval = 8 * 3600;   /* 8 hours maximum */
	static const time_t min_no_expires_interval = 10 * 60; /* 10 minutes minimum when no expires */
	static const time_t liberal_mult = 10;                 /* Multiplier for liberal interval */
	static const time_t max_valid_time = 365 * 24 * 3600;  /* 1 year max for overflow protection */
	time_t next_check;

	/*
	 * Goal: Respect server-provided expiration while preventing abuse
	 * Server controls refresh rate via Expires header, client cannot override aggressively
	 */

	/* Sanity check: detect overflow or absurdly invalid expires values */
	if (expires > now) {
		/* Check if expires would be more than max_valid_time in future without overflow */
		if (expires > (time_t) (now + max_valid_time) || (expires - now) > max_valid_time) {
			msg_warn_map("expires header is absurdly far in future (overflow?) for %s, ignoring",
						 bk->uri);
			expires = 0; /* Treat as no expires header */
		}
	}

	if (expires > now) {
		/* Server provided an Expires header */
		time_t expires_interval = expires - now;

		if (expires_interval > max_expires_interval) {
			/*
			 * Absurdly high expiration (> 8 hours)
			 * Use min(map_check_interval * 10, 8 hours) with overflow protection
			 */
			time_t liberal_interval;
			if (map_check_interval > max_expires_interval / liberal_mult) {
				liberal_interval = max_expires_interval;
			}
			else {
				liberal_interval = map_check_interval * liberal_mult;
			}

			if (liberal_interval > max_expires_interval) {
				next_check = now + max_expires_interval;
				msg_info_map("expires header too high (%d hours) for %s, capping to 8 hours",
							 (int) (expires_interval / 3600), bk->uri);
			}
			else {
				next_check = now + liberal_interval;
				msg_info_map("expires header very high (%d hours) for %s, using liberal interval %d seconds",
							 (int) (expires_interval / 3600), bk->uri, (int) liberal_interval);
			}
		}
		else if (expires_interval < map_check_interval) {
			/*
			 * Server wants faster refresh than configured interval
			 * Respect the configured minimum to prevent abuse
			 */
			next_check = now + map_check_interval;
			msg_debug_map("expires header (%d sec) less than map_check_interval (%d sec) for %s, "
						  "using map_check_interval",
						  (int) expires_interval, (int) map_check_interval, bk->uri);
		}
		else {
			/*
			 * Reasonable expires header (between map_check_interval and 8 hours)
			 * Use it as-is
			 */
			next_check = expires;
		}
	}
	else {
		/*
		 * No expires header (or expired)
		 * Differentiate based on cache validation capabilities
		 */
		if (has_etag || has_last_modified) {
			/*
			 * We have cache validation (ETag or Last-Modified)
			 * Conditional requests are cheap (304 responses), use respectful 4x multiplier
			 */
			static const time_t respectful_mult = 4;
			time_t respectful_interval;

			/* Overflow protection for multiplication */
			if (map_check_interval > max_expires_interval / respectful_mult) {
				respectful_interval = max_expires_interval;
			}
			else {
				respectful_interval = map_check_interval * respectful_mult;
			}

			if (respectful_interval < min_no_expires_interval) {
				next_check = now + min_no_expires_interval;
				msg_info_map("no expires but has cache validation for %s, "
							 "using 10 minute minimum",
							 bk->uri);
			}
			else {
				next_check = now + respectful_interval;
				msg_debug_map("no expires but has cache validation for %s, "
							  "using %dx interval (%d sec)",
							  bk->uri, (int) respectful_mult, (int) respectful_interval);
			}
		}
		else {
			/*
			 * No cache validation available
			 * Must re-download entire map, enforce strict minimum
			 */
			if (map_check_interval < min_no_expires_interval) {
				next_check = now + min_no_expires_interval;
				msg_info_map("no cache validation and low map_check_interval (%d sec) for %s, "
							 "enforcing 10 minute minimum",
							 (int) map_check_interval, bk->uri);
			}
			else {
				next_check = now + map_check_interval;
			}
		}
	}

	return next_check;
}

static gboolean
rspamd_map_secretbox_decrypt_buf(struct rspamd_map_backend *bk,
								 const unsigned char *in,
								 gsize inlen,
								 unsigned char **out,
								 gsize *outlen)
{
	/* Logging-aware helper */
	struct rspamd_map *map = bk ? bk->map : NULL;

	if (!bk->has_secretbox_key) {
		msg_err_map("%s: secretbox key is not configured", bk->uri);
		return FALSE;
	}

	if (inlen < crypto_secretbox_NONCEBYTES + crypto_secretbox_MACBYTES) {
		msg_err_map("%s: too short buffer for secretbox: %z bytes (need >= %d)",
					bk->uri, inlen, (int) (crypto_secretbox_NONCEBYTES + crypto_secretbox_MACBYTES));
		return FALSE;
	}

	const unsigned char *nonce = in;
	const unsigned char *ct = in + crypto_secretbox_NONCEBYTES;
	gsize clen = inlen - crypto_secretbox_NONCEBYTES;

	if (clen < crypto_secretbox_MACBYTES) {
		msg_err_map("%s: invalid ciphertext length for secretbox: %z (< MAC %d)",
					bk->uri, clen, (int) crypto_secretbox_MACBYTES);
		return FALSE;
	}

	gsize ptlen = clen - crypto_secretbox_MACBYTES;
	unsigned char *pt = g_malloc(ptlen);

	msg_debug_map("%s: attempting secretbox decrypt; nonce_len=%d ct_len=%z pt_len=%z",
				  bk->uri, (int) crypto_secretbox_NONCEBYTES, clen, ptlen);

	if (crypto_secretbox_open_easy(pt, ct, clen, nonce, bk->secretbox_key) != 0) {
		msg_err_map("%s: secretbox authentication failed (nonce_len=%d, ct_len=%z)",
					bk->uri, (int) crypto_secretbox_NONCEBYTES, clen);
		g_free(pt);
		return FALSE;
	}

	*out = pt;
	*outlen = ptlen;
	return TRUE;
}

static gboolean
rspamd_map_decode_secretbox_key(const char *in, gsize inlen, unsigned char out[32])
{
	/* Be compatible with lua_secretbox: derive a 32-byte key via crypto_generichash
	 * from the provided secret (which can be hex/base64/raw). */
	if (in == NULL || inlen == 0) {
		return FALSE;
	}

	unsigned char *raw = NULL;
	gsize rawlen = 0;
	bool allocated = false;

	/* Detect hex (only hex chars, even length) */
	bool maybe_hex = (inlen % 2 == 0);
	for (gsize i = 0; i < inlen && maybe_hex; i++) {
		char c = in[i];
		if (!g_ascii_isxdigit((int) c)) {
			maybe_hex = false;
		}
	}

	if (maybe_hex) {
		gsize tmp_len = inlen / 2 + 1;
		raw = g_malloc(tmp_len);
		gssize dec = rspamd_decode_hex_buf(in, inlen, raw, tmp_len);
		if (dec > 0) {
			rawlen = (gsize) dec;
			allocated = true;
		}
		else {
			g_free(raw);
			raw = NULL;
		}
	}

	if (raw == NULL && rspamd_cryptobox_base64_is_valid(in, inlen)) {
		/* Try base64 */
		/* Worst-case allocate */
		gsize tmp_len = (inlen / 4 + 1) * 3 + 8;
		raw = g_malloc(tmp_len);
		if (rspamd_cryptobox_base64_decode(in, inlen, raw, &rawlen)) {
			allocated = true;
		}
		else {
			g_free(raw);
			raw = NULL;
			rawlen = 0;
		}
	}

	if (raw == NULL) {
		/* Treat as raw string */
		raw = (unsigned char *) in;
		rawlen = inlen;
	}

	/* Derive 32-byte key */
	crypto_generichash(out, crypto_secretbox_KEYBYTES, raw, rawlen, NULL, 0);

	if (allocated) {
		rspamd_explicit_memzero(raw, rawlen);
		g_free(raw);
	}

	return TRUE;
}

static inline gboolean
rspamd_map_payload_is_zstd(const unsigned char *p, gsize len)
{
	/* Zstandard frame magic number: 0x28B52FFD at start of frame */
	if (len >= 4) {
		const unsigned char zstd_magic[4] = {0x28u, 0xB5u, 0x2Fu, 0xFDu};
		if (memcmp(p, zstd_magic, 4) == 0) {
			return TRUE;
		}
	}

	return FALSE;
}

static void
rspamd_map_try_load_secretbox_key(struct rspamd_config *cfg,
								  struct rspamd_map_backend *bk)
{
	const ucl_object_t *maps_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "maps");
	if (maps_obj == NULL || ucl_object_type(maps_obj) != UCL_OBJECT) {
		const ucl_object_t *opts_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "options");
		if (opts_obj && ucl_object_type(opts_obj) == UCL_OBJECT) {
			maps_obj = ucl_object_lookup(opts_obj, "maps");
		}
	}

	if (maps_obj && ucl_object_type(maps_obj) == UCL_OBJECT) {
		const ucl_object_t *src = ucl_object_lookup(maps_obj, bk->uri);
		if (!src) src = maps_obj;
		const ucl_object_t *kobj = ucl_object_lookup_any(src, "secretbox_key", "secretbox-key", "enc_key", NULL);
		if (kobj && ucl_object_type(kobj) == UCL_STRING) {
			gsize klen = 0;
			const char *k = ucl_object_tolstring(kobj, &klen);
			if (rspamd_map_decode_secretbox_key(k, klen, bk->secretbox_key)) {
				bk->has_secretbox_key = TRUE;
				msg_info_config("loaded secretbox key late for %s", bk->uri);
			}
		}
	}
}
static int
http_map_finish(struct rspamd_http_connection *conn,
				struct rspamd_http_message *msg)
{
	struct http_callback_data *cbd = conn->ud;
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;
	struct http_map_data *data;
	struct rspamd_http_map_cached_cbdata *cache_cbd;
	const rspamd_ftok_t *expires_hdr, *etag_hdr;
	char next_check_date[128];
	unsigned char *in = NULL;
	gsize dlen = 0;

	map = cbd->map;
	bk = cbd->bk;
	data = bk->data.hd;

	if (msg->code == 200) {

		if (cbd->check) {
			msg_info_map("need to reread map from %s (reply code 200); "
						 "date timestamp: %z, last modified: %z",
						 cbd->bk->uri, (size_t) msg->date, (size_t) msg->last_modified);
			cbd->periodic->need_modify = TRUE;
			/* Reset the whole chain */
			cbd->periodic->cur_backend = 0;
			/* Reset cache, old cached data will be cleaned on timeout */
			g_atomic_int_set(&data->cache->available, 0);
			g_atomic_int_set(&map->shared->loaded, 0);
			data->cur_cache_cbd = NULL;

			rspamd_map_process_periodic(cbd->periodic);
			MAP_RELEASE(cbd, "http_callback_data");

			return 0;
		}

		/* This code is executed when we are actually reading a map */
		cbd->data->last_checked = msg->date;

		if (msg->last_modified) {
			cbd->data->last_modified = msg->last_modified;
		}
		else {
			cbd->data->last_modified = msg->date;
		}


		/* Unsigned version - just open file */
		cbd->shmem_data = rspamd_http_message_shmem_ref(msg);
		cbd->data_len = msg->body_buf.len;

		if (cbd->data_len == 0) {
			msg_err_map("cannot read empty map");
			goto err;
		}

		g_assert(cbd->shmem_data != NULL);

		in = rspamd_shmem_xmap(cbd->shmem_data->shm_name, PROT_READ, &dlen);

		if (in == NULL) {
			msg_err_map("cannot read tempfile %s: %s",
						cbd->shmem_data->shm_name,
						strerror(errno));
			goto err;
		}

		/* Check for expires + etag */
		double cached_timeout = map->poll_timeout * 2;

		expires_hdr = rspamd_http_message_find_header(msg, "Expires");
		etag_hdr = rspamd_http_message_find_header(msg, "ETag");

		if (expires_hdr) {
			time_t hdate;

			hdate = rspamd_http_parse_date(expires_hdr->begin, expires_hdr->len);

			if (hdate != (time_t) -1 && hdate > msg->date) {
				map->next_check = rspamd_http_map_process_next_check(map, bk, msg->date, hdate,
																	 (time_t) map->poll_timeout,
																	 etag_hdr != NULL,
																	 msg->last_modified != 0);
				cached_timeout = map->next_check - msg->date;
			}
			else {
				msg_info_map("invalid expires header: %T, ignore it", expires_hdr);
				/* Treat invalid expires as no expires header */
				map->next_check = rspamd_http_map_process_next_check(map, bk, msg->date, 0,
																	 (time_t) map->poll_timeout,
																	 etag_hdr != NULL,
																	 msg->last_modified != 0);
			}
		}
		else if (etag_hdr != NULL || msg->last_modified != 0) {
			/* No expires header, but we have ETag or Last-Modified */
			map->next_check = rspamd_http_map_process_next_check(map, bk, msg->date, 0,
																 (time_t) map->poll_timeout,
																 etag_hdr != NULL,
																 msg->last_modified != 0);
		}

		if (etag_hdr) {
			if (cbd->data->etag) {
				/* Remove old etag */
				rspamd_fstring_free(cbd->data->etag);
			}

			cbd->data->etag = rspamd_fstring_new_init(etag_hdr->begin,
													  etag_hdr->len);
		}
		else {
			if (cbd->data->etag) {
				/* Remove and clear old etag */
				rspamd_fstring_free(cbd->data->etag);
				cbd->data->etag = NULL;
			}
		}

		MAP_RETAIN(cbd->shmem_data, "shmem_data");
		cbd->data->gen++;

		/* Store cached data: raw HTTP body in SHM */
		rspamd_strlcpy(data->cache->shmem_name, cbd->shmem_data->shm_name,
					   sizeof(data->cache->shmem_name));
		data->cache->len = cbd->data_len;
		data->cache->last_modified = cbd->data->last_modified;
		cache_cbd = g_malloc0(sizeof(*cache_cbd));
		cache_cbd->shm = cbd->shmem_data;
		cache_cbd->event_loop = cbd->event_loop;
		cache_cbd->map = map;
		cache_cbd->data = cbd->data;
		cache_cbd->last_checked = cbd->data->last_checked;
		cache_cbd->gen = cbd->data->gen;
		MAP_RETAIN(cache_cbd->shm, "shmem_data");
		msg_info_map("stored map data in a shared memory cache: %s",
					 cache_cbd->shm->shm_name);

		ev_timer_init(&cache_cbd->timeout, rspamd_map_cache_cb, cached_timeout,
					  0.0);
		ev_timer_start(cbd->event_loop, &cache_cbd->timeout);
		cache_cbd->timeout.data = cache_cbd;
		data->cur_cache_cbd = cache_cbd;

		if (map->next_check) {
			rspamd_http_date_format(next_check_date, sizeof(next_check_date),
									map->next_check);
		}
		else {
			rspamd_http_date_format(next_check_date, sizeof(next_check_date),
									rspamd_get_calendar_ticks() + map->poll_timeout);
		}


		/* Prepare payload: decrypt (if needed) then optionally decompress */
		unsigned char *payload = NULL;
		gsize payload_len = 0;
		unsigned char *final_out = NULL;

		if (cbd->bk->is_encrypted) {
			if (!rspamd_map_secretbox_decrypt_buf(cbd->bk, in, dlen, &payload, &payload_len)) {
				msg_err_map("%s(%s): cannot decrypt data", cbd->bk->uri,
							rspamd_inet_address_to_string_pretty(cbd->addr));
				MAP_RELEASE(cbd->shmem_data, "shmem_data");
				goto err;
			}
		}
		else {
			/* Use mapped buffer directly */
			payload = (unsigned char *) in;
			payload_len = dlen;
		}

		/* If compressed flag is set OR payload looks like zstd, decompress */
		if (cbd->bk->is_compressed || rspamd_map_payload_is_zstd(payload, payload_len)) {
			ZSTD_DStream *zstream;
			ZSTD_inBuffer zin;
			ZSTD_outBuffer zout;
			gsize outlen, r;

			zstream = ZSTD_createDStream();
			ZSTD_initDStream(zstream);

			zin.pos = 0;
			zin.src = payload;
			zin.size = payload_len;

			if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
				outlen = ZSTD_DStreamOutSize();
			}

			final_out = g_malloc(outlen);

			zout.dst = final_out;
			zout.pos = 0;
			zout.size = outlen;

			while (zin.pos < zin.size) {
				r = ZSTD_decompressStream(zstream, &zout, &zin);

				if (ZSTD_isError(r)) {
					msg_err_map("%s(%s): cannot decompress data: %s",
								cbd->bk->uri,
								rspamd_inet_address_to_string_pretty(cbd->addr),
								ZSTD_getErrorName(r));
					ZSTD_freeDStream(zstream);
					g_free(final_out);
					if (cbd->bk->is_encrypted && payload && payload != (unsigned char *) in) {
						rspamd_explicit_memzero(payload, payload_len);
						g_free(payload);
					}
					MAP_RELEASE(cbd->shmem_data, "shmem_data");
					goto err;
				}

				if (zout.pos == zout.size) {
					/* We need to extend output buffer */
					zout.size = zout.size * 2 + 1.0;
					final_out = g_realloc(zout.dst, zout.size);
					zout.dst = final_out;
				}
			}

			ZSTD_freeDStream(zstream);
			msg_info_map("%s(%s): read map data %z bytes compressed, "
						 "%z uncompressed, next check at %s",
						 cbd->bk->uri,
						 rspamd_inet_address_to_string_pretty(cbd->addr),
						 payload_len, zout.pos, next_check_date);
			if (!rspamd_map_save_http_cached_file(map, bk, cbd->data, final_out, zout.pos)) {
				msg_err_map("%s: failed to save cache file", bk->uri);
				g_free(final_out);
				MAP_RELEASE(cbd->shmem_data, "shmem_data");
				goto err;
			}

			if (map->no_file_read) {
				/* Pass cache file path; consumer mmaps at page-aligned offset */
				char cache_path[PATH_MAX];
				rspamd_map_cache_file_path(bk, map->cfg->maps_cache_dir,
										   cache_path, sizeof(cache_path));
				map->no_file_read_offset = RSPAMD_MAP_CACHE_HEADER_SIZE;
				map->read_callback(cache_path, strlen(cache_path),
								   &cbd->periodic->cbdata, TRUE);
			}
			else {
				map->read_callback(final_out, zout.pos, &cbd->periodic->cbdata, TRUE);
			}
			g_free(final_out);
		}
		else {
			msg_info_map("%s(%s): read map data %z bytes, next check at %s",
						 cbd->bk->uri,
						 rspamd_inet_address_to_string_pretty(cbd->addr),
						 payload_len, next_check_date);

			if (!rspamd_map_save_http_cached_file(map, bk, cbd->data, payload, payload_len)) {
				msg_err_map("%s: failed to save cache file", bk->uri);
				MAP_RELEASE(cbd->shmem_data, "shmem_data");
				goto err;
			}

			if (map->no_file_read) {
				/* Pass cache file path; consumer mmaps at page-aligned offset */
				char cache_path[PATH_MAX];
				rspamd_map_cache_file_path(bk, map->cfg->maps_cache_dir,
										   cache_path, sizeof(cache_path));
				map->no_file_read_offset = RSPAMD_MAP_CACHE_HEADER_SIZE;
				map->read_callback(cache_path, strlen(cache_path),
								   &cbd->periodic->cbdata, TRUE);
			}
			else {
				map->read_callback(payload, payload_len, &cbd->periodic->cbdata, TRUE);
			}
		}

		MAP_RELEASE(cbd->shmem_data, "shmem_data");

		cbd->periodic->cur_backend++;
		if (cbd->bk->is_encrypted && payload && payload != (unsigned char *) in) {
			rspamd_explicit_memzero(payload, payload_len);
			g_free(payload);
		}
		munmap(in, dlen);

		/* Announce for other processes */
		g_atomic_int_set(&data->cache->available, 1);
		g_atomic_int_set(&map->shared->loaded, 1);
		g_atomic_int_set(&map->shared->cached, 1);

		rspamd_map_process_periodic(cbd->periodic);
	}
	else if (msg->code == 304 && cbd->check) {
		cbd->data->last_checked = msg->date;

		if (msg->last_modified) {
			cbd->data->last_modified = msg->last_modified;
		}
		else {
			cbd->data->last_modified = msg->date;
		}

		expires_hdr = rspamd_http_message_find_header(msg, "Expires");
		bool has_expires = (expires_hdr != NULL);

		if (expires_hdr) {
			time_t hdate;

			hdate = rspamd_http_parse_date(expires_hdr->begin, expires_hdr->len);
			if (hdate != (time_t) -1 && hdate > msg->date) {
				map->next_check = rspamd_http_map_process_next_check(map, bk, msg->date, hdate,
																	 (time_t) map->poll_timeout,
																	 cbd->data->etag != NULL,
																	 msg->last_modified != 0);
			}
			else {
				msg_info_map("invalid expires header: %T, ignore it", expires_hdr);
				/* Treat invalid expires as no expires header */
				map->next_check = rspamd_http_map_process_next_check(map, bk, msg->date, 0,
																	 (time_t) map->poll_timeout,
																	 cbd->data->etag != NULL,
																	 msg->last_modified != 0);
				has_expires = false;
			}
		}
		else if (cbd->data->etag != NULL || msg->last_modified != 0) {
			/* No expires header, but we have ETag or Last-Modified */
			map->next_check = rspamd_http_map_process_next_check(map, bk, msg->date, 0,
																 (time_t) map->poll_timeout,
																 cbd->data->etag != NULL,
																 msg->last_modified != 0);
		}

		etag_hdr = rspamd_http_message_find_header(msg, "ETag");

		if (etag_hdr) {
			if (cbd->data->etag) {
				/* Remove old etag */
				rspamd_fstring_free(cbd->data->etag);
				cbd->data->etag = rspamd_fstring_new_init(etag_hdr->begin,
														  etag_hdr->len);
			}
		}

		if (has_expires) {
			rspamd_http_date_format(next_check_date, sizeof(next_check_date),
									map->next_check);
			msg_info_map("data is not modified for server %s (%s), next check at %s "
						 "(http cache based: %T)",
						 cbd->data->host,
						 bk->uri,
						 next_check_date,
						 expires_hdr);
		}
		else {
			rspamd_http_date_format(next_check_date, sizeof(next_check_date),
									map->next_check);
			msg_info_map("data is not modified for server %s (%s), next check at %s "
						 "(timer based)",
						 cbd->data->host,
						 bk->uri,
						 next_check_date);
		}

		rspamd_map_update_http_cached_file(map, bk, cbd->data);
		cbd->periodic->cur_backend++;
		rspamd_map_process_periodic(cbd->periodic);
	}
	else {
		msg_info_map("cannot load map %s from %s: HTTP error %d",
					 bk->uri, cbd->data->host, msg->code);
		goto err;
	}

	MAP_RELEASE(cbd, "http_callback_data");
	return 0;

err:
	cbd->periodic->errored = 1;
	rspamd_map_process_periodic(cbd->periodic);
	MAP_RELEASE(cbd, "http_callback_data");

	return 0;
}

static gboolean
read_map_file_chunks(struct rspamd_map *map, struct map_cb_data *cbdata,
					 const char *fname, gsize len, goffset off)
{
	int fd;
	gssize r, avail;
	gsize buflen = 1024 * 1024;
	char *pos, *bytes;

	fd = rspamd_file_xopen(fname, O_RDONLY, 0, TRUE);

	if (fd == -1) {
		msg_err_map("can't open map for buffered reading %s: %s",
					fname, strerror(errno));
		return FALSE;
	}

	if (lseek(fd, off, SEEK_SET) == -1) {
		msg_err_map("can't seek in map to pos %d for buffered reading %s: %s",
					(int) off, fname, strerror(errno));
		close(fd);

		return FALSE;
	}

	buflen = MIN(len, buflen);
	bytes = g_malloc(buflen);
	avail = buflen;
	pos = bytes;

	while ((r = read(fd, pos, avail)) > 0) {
		char *end = bytes + (pos - bytes) + r;
		msg_debug_map("%s: read map chunk, %z bytes", fname,
					  r);
		pos = map->read_callback(bytes, end - bytes, cbdata, r == len);

		if (pos && pos > bytes && pos < end) {
			unsigned int remain = end - pos;

			memmove(bytes, pos, remain);
			pos = bytes + remain;
			/* Need to preserve the remain */
			avail = ((gssize) buflen) - remain;

			if (avail <= 0) {
				/* Try realloc, too large element */
				g_assert(buflen >= remain);
				bytes = g_realloc(bytes, buflen * 2);

				pos = bytes + remain; /* Adjust */
				avail += buflen;
				buflen *= 2;
			}
		}
		else {
			avail = buflen;
			pos = bytes;
		}

		len -= r;
	}

	if (r == -1) {
		msg_err_map("can't read from map %s: %s", fname, strerror(errno));
		close(fd);
		g_free(bytes);

		return FALSE;
	}

	close(fd);
	g_free(bytes);

	return TRUE;
}

static gboolean
rspamd_map_check_sig_pk_mem(const unsigned char *sig,
							gsize siglen,
							struct rspamd_map *map,
							const unsigned char *input,
							gsize inlen,
							struct rspamd_cryptobox_pubkey *pk)
{
	GString *b32_key;
	gboolean ret = TRUE;

	if (siglen != crypto_sign_bytes()) {
		msg_err_map("can't open signature for %s: invalid size: %z", map->name, siglen);

		ret = FALSE;
	}

	if (ret && !rspamd_cryptobox_verify(sig, siglen, input, inlen,
										rspamd_pubkey_get_pk(pk, NULL))) {
		msg_err_map("can't verify signature for %s: incorrect signature", map->name);

		ret = FALSE;
	}

	if (ret) {
		b32_key = rspamd_pubkey_print(pk,
									  RSPAMD_KEYPAIR_ENCODING_DEFAULT,
									  RSPAMD_KEYPAIR_PUBKEY);
		msg_info_map("verified signature for %s using trusted key %v",
					 map->name, b32_key);
		g_string_free(b32_key, TRUE);
	}

	return ret;
}

static gboolean
rspamd_map_check_file_sig(const char *fname,
						  struct rspamd_map *map,
						  struct rspamd_map_backend *bk,
						  const unsigned char *input,
						  gsize inlen)
{
	unsigned char *data;
	struct rspamd_cryptobox_pubkey *pk = NULL;
	GString *b32_key;
	gboolean ret = TRUE;
	gsize len = 0;
	char fpath[PATH_MAX];

	if (bk->trusted_pubkey == NULL) {
		/* Try to load and check pubkey */
		rspamd_snprintf(fpath, sizeof(fpath), "%s.pub", fname);
		data = rspamd_file_xmap(fpath, PROT_READ, &len, TRUE);

		if (data == NULL) {
			msg_err_map("can't open pubkey %s: %s", fpath, strerror(errno));
			return FALSE;
		}

		pk = rspamd_pubkey_from_base32(data, len, RSPAMD_KEYPAIR_SIGN);
		munmap(data, len);

		if (pk == NULL) {
			msg_err_map("can't load pubkey %s", fpath);
			return FALSE;
		}

		/* We just check pk against the trusted db of keys */
		b32_key = rspamd_pubkey_print(pk,
									  RSPAMD_KEYPAIR_ENCODING_DEFAULT,
									  RSPAMD_KEYPAIR_PUBKEY);
		g_assert(b32_key != NULL);

		if (g_hash_table_lookup(map->cfg->trusted_keys, b32_key->str) == NULL) {
			msg_err_map("pubkey loaded from %s is untrusted: %v", fpath,
						b32_key);
			g_string_free(b32_key, TRUE);
			rspamd_pubkey_unref(pk);

			return FALSE;
		}

		g_string_free(b32_key, TRUE);
	}
	else {
		pk = rspamd_pubkey_ref(bk->trusted_pubkey);
	}

	rspamd_snprintf(fpath, sizeof(fpath), "%s.sig", fname);
	data = rspamd_shmem_xmap(fpath, PROT_READ, &len);

	if (data == NULL) {
		msg_err_map("can't open signature %s: %s", fpath, strerror(errno));
		ret = FALSE;
	}

	if (ret) {
		ret = rspamd_map_check_sig_pk_mem(data, len, map, input, inlen, pk);
		munmap(data, len);
	}

	rspamd_pubkey_unref(pk);

	return ret;
}

/**
 * Callback for reading data from file
 */
static gboolean
read_map_file(struct rspamd_map *map, struct file_map_data *data,
			  struct rspamd_map_backend *bk, struct map_periodic_cbdata *periodic)
{
	char *bytes;
	gsize len;
	struct stat st;

	if (map->read_callback == NULL || map->fin_callback == NULL) {
		msg_err_map("%s: bad callback for reading map file",
					data->filename);
		return FALSE;
	}

	if (stat(data->filename, &st) == -1) {
		/* File does not exist, skipping */
		if (errno != ENOENT) {
			msg_err_map("%s: map file is unavailable for reading: %s",
						data->filename, strerror(errno));

			return FALSE;
		}
		else {
			msg_info_map("%s: map file is not found; "
						 "it will be read automatically if created",
						 data->filename);
			return TRUE;
		}
	}

	ev_stat_stat(map->event_loop, &data->st_ev);
	len = st.st_size;

	if (bk->is_signed) {
		bytes = rspamd_file_xmap(data->filename, PROT_READ, &len, TRUE);

		if (bytes == NULL) {
			msg_err_map("can't open map %s: %s", data->filename, strerror(errno));
			return FALSE;
		}

		if (!rspamd_map_check_file_sig(data->filename, map, bk, bytes, len)) {
			munmap(bytes, len);

			return FALSE;
		}

		munmap(bytes, len);
	}

	if (len > 0) {
		if (map->no_file_read) {
			/* We just call read callback with backend name */
			map->read_callback(data->filename, strlen(data->filename),
							   &periodic->cbdata, TRUE);
		}
		else {
			if (bk->is_compressed || bk->is_encrypted) {
				bytes = rspamd_file_xmap(data->filename, PROT_READ, &len, TRUE);

				if (bytes == NULL) {
					msg_err_map("can't open map %s: %s", data->filename, strerror(errno));
					return FALSE;
				}

				unsigned char *payload = (unsigned char *) bytes;
				gsize payload_len = len;

				unsigned char *dec = NULL;
				gsize declen = 0;

				if (bk->is_encrypted) {
					if (!rspamd_map_secretbox_decrypt_buf(bk, payload, payload_len, &dec, &declen)) {
						msg_err_map("%s: cannot decrypt data: secretbox auth failed",
									data->filename);
						munmap(bytes, len);
						return FALSE;
					}
					payload = dec;
					payload_len = declen;
				}

				/* If compressed flag is set OR payload looks like zstd, decompress */
				if (bk->is_compressed || rspamd_map_payload_is_zstd((const unsigned char *) payload, payload_len)) {
					ZSTD_DStream *zstream;
					ZSTD_inBuffer zin;
					ZSTD_outBuffer zout;
					unsigned char *out;
					gsize outlen, r;

					zstream = ZSTD_createDStream();
					ZSTD_initDStream(zstream);

					zin.pos = 0;
					zin.src = payload;
					zin.size = payload_len;

					if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
						outlen = ZSTD_DStreamOutSize();
					}

					out = g_malloc(outlen);

					zout.dst = out;
					zout.pos = 0;
					zout.size = outlen;

					while (zin.pos < zin.size) {
						r = ZSTD_decompressStream(zstream, &zout, &zin);

						if (ZSTD_isError(r)) {
							msg_err_map("%s: cannot decompress data: %s",
										data->filename,
										ZSTD_getErrorName(r));
							ZSTD_freeDStream(zstream);
							g_free(out);
							if (dec) {
								rspamd_explicit_memzero(dec, declen);
								g_free(dec);
							}
							munmap(bytes, len);
							return FALSE;
						}

						if (zout.pos == zout.size) {
							/* We need to extend output buffer */
							zout.size = zout.size * 2 + 1;
							out = g_realloc(zout.dst, zout.size);
							zout.dst = out;
						}
					}

					ZSTD_freeDStream(zstream);
					msg_info_map("%s: read map data, %z bytes compressed, "
								 "%z uncompressed)",
								 data->filename,
								 payload_len, zout.pos);
					map->read_callback(out, zout.pos, &periodic->cbdata, TRUE);
					g_free(out);

					if (dec) {
						rspamd_explicit_memzero(dec, declen);
						g_free(dec);
					}
					munmap(bytes, len);
				}
				else if (bk->is_encrypted) {
					/* Already decrypted, pass through */
					msg_info_map("%s: read map data, %z bytes (decrypted)",
								 data->filename, payload_len);
					map->read_callback(payload, payload_len, &periodic->cbdata, TRUE);
					rspamd_explicit_memzero(payload, payload_len);
					g_free(payload);
					munmap(bytes, len);
				}
				else {
					/* Should not happen here as we only mmap for compressed or encrypted */
					munmap(bytes, len);
					if (!read_map_file_chunks(map, &periodic->cbdata, data->filename,
											  len, 0)) {
						return FALSE;
					}
				}
			}
			else {
				/* Perform buffered read: fail-safe */
				if (!read_map_file_chunks(map, &periodic->cbdata, data->filename,
										  len, 0)) {
					return FALSE;
				}
			}
		}
	}
	else {
		/* Empty map */
		map->read_callback(NULL, 0, &periodic->cbdata, TRUE);
	}

	g_atomic_int_set(&map->shared->loaded, 1);

	return TRUE;
}

static gboolean
read_map_static(struct rspamd_map *map, struct static_map_data *data,
				struct rspamd_map_backend *bk, struct map_periodic_cbdata *periodic)
{
	unsigned char *bytes;
	gsize len;

	if (map->read_callback == NULL || map->fin_callback == NULL) {
		msg_err_map("%s: bad callback for reading map file", map->name);
		data->processed = TRUE;
		return FALSE;
	}

	bytes = data->data;
	len = data->len;

	if (len > 0) {
		if (bk->is_compressed) {
			ZSTD_DStream *zstream;
			ZSTD_inBuffer zin;
			ZSTD_outBuffer zout;
			unsigned char *out;
			gsize outlen, r;

			zstream = ZSTD_createDStream();
			ZSTD_initDStream(zstream);

			zin.pos = 0;
			zin.src = bytes;
			zin.size = len;

			if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
				outlen = ZSTD_DStreamOutSize();
			}

			out = g_malloc(outlen);

			zout.dst = out;
			zout.pos = 0;
			zout.size = outlen;

			while (zin.pos < zin.size) {
				r = ZSTD_decompressStream(zstream, &zout, &zin);

				if (ZSTD_isError(r)) {
					msg_err_map("%s: cannot decompress data: %s",
								map->name,
								ZSTD_getErrorName(r));
					ZSTD_freeDStream(zstream);
					g_free(out);

					return FALSE;
				}

				if (zout.pos == zout.size) {
					/* We need to extend output buffer */
					zout.size = zout.size * 2 + 1;
					out = g_realloc(zout.dst, zout.size);
					zout.dst = out;
				}
			}

			ZSTD_freeDStream(zstream);
			msg_info_map("%s: read map data, %z bytes compressed, "
						 "%z uncompressed)",
						 map->name,
						 len, zout.pos);
			map->read_callback(out, zout.pos, &periodic->cbdata, TRUE);
			g_free(out);
		}
		else {
			msg_info_map("%s: read map data, %z bytes",
						 map->name, len);
			map->read_callback(bytes, len, &periodic->cbdata, TRUE);
		}
	}
	else {
		map->read_callback(NULL, 0, &periodic->cbdata, TRUE);
	}

	data->processed = TRUE;
	g_atomic_int_set(&map->shared->loaded, 1);

	return TRUE;
}

static void
rspamd_map_periodic_dtor(struct map_periodic_cbdata *periodic)
{
	struct rspamd_map *map = periodic->map;
	msg_debug_map("periodic dtor %p; need_modify=%d", periodic, periodic->need_modify);

	if (periodic->need_modify || periodic->cbdata.errored) {
		/* Need to notify the real data structure */
		periodic->map->fin_callback(&periodic->cbdata, periodic->map->user_data);

		if (map->on_load_function) {
			map->on_load_function(map, map->on_load_ud);
		}
	}
	else {
		/* Not modified */
	}

	if (periodic->map->wrk->state == rspamd_worker_state_running) {
		rspamd_map_schedule_periodic(periodic->map,
									 RSPAMD_MAP_SCHEDULE_NORMAL);
	}
	else {
		msg_debug_map("stop scheduling periodics for %s; terminating state",
					  periodic->map->name);
	}

	g_free(periodic);
}

/* Called on timer execution */
static void
rspamd_map_periodic_callback(struct ev_loop *loop, ev_timer *w, int revents)
{
	struct map_periodic_cbdata *cbd = (struct map_periodic_cbdata *) w->data;

	MAP_RETAIN(cbd, "periodic");
	ev_timer_stop(loop, w);
	rspamd_map_process_periodic(cbd);
	MAP_RELEASE(cbd, "periodic");
}

static void
rspamd_map_schedule_periodic(struct rspamd_map *map, int how)
{
	const double error_mult = 20.0, lock_mult = 0.1;
	static const double min_timer_interval = 2.0;
	const char *reason = "unknown reason";
	double jittered_sec;
	double timeout;
	struct map_periodic_cbdata *cbd;

	if (map->scheduled_check || (map->wrk &&
								 map->wrk->state != rspamd_worker_state_running)) {
		/*
		 * Do not schedule check if some check is already scheduled or
		 * if worker is going to die
		 */
		return;
	}

	if (!(how & RSPAMD_MAP_SCHEDULE_INIT) && map->static_only) {
		/* No need to schedule anything for static maps */
		return;
	}

	map->seen = true;

	if (map->non_trivial && map->next_check != 0) {
		timeout = map->next_check - rspamd_get_calendar_ticks();
		map->next_check = 0;

		if (timeout > 0 && timeout < map->poll_timeout) {
			/* Early check case, jitter */
			double poll_timeout = map->poll_timeout;

			if (how & RSPAMD_MAP_SCHEDULE_ERROR) {
				poll_timeout = map->poll_timeout * error_mult;
				reason = "early active non-trivial check (after error)";
			}
			else if (how & RSPAMD_MAP_SCHEDULE_LOCKED) {
				poll_timeout = map->poll_timeout * lock_mult;
				reason = "early active non-trivial check (after being locked)";
			}
			else {
				reason = "early active non-trivial check";
			}

			jittered_sec = MIN(timeout, poll_timeout);
		}
		else if (timeout <= 0) {
			/* Data is already expired, need to check */
			if (how & RSPAMD_MAP_SCHEDULE_ERROR) {
				/* In case of error we still need to increase delay */
				jittered_sec = map->poll_timeout * error_mult;
				reason = "expired non-trivial data (after error)";
			}
			else {
				jittered_sec = 0.0;
				reason = "expired non-trivial data";
			}
		}
		else {
			/* No need to check now, wait till next_check */
			jittered_sec = timeout;
			reason = "valid non-trivial data";
		}
	}
	else {
		/* No valid information when to check a map, plan a timer based check */
		timeout = map->poll_timeout;

		if (how & RSPAMD_MAP_SCHEDULE_INIT) {
			if (map->non_trivial && map->active_http) {
				/* Spill maps load to get better chances to hit ssl cache */
				timeout = rspamd_time_jitter(0.0, 2.0);
			}
			else {
				timeout = 0.0;
			}

			reason = "init scheduled check";
		}
		else {
			if (how & RSPAMD_MAP_SCHEDULE_ERROR) {
				timeout = map->poll_timeout * error_mult;
				reason = "errored scheduled check";
			}
			else if (how & RSPAMD_MAP_SCHEDULE_LOCKED) {
				timeout = map->poll_timeout * lock_mult;
				reason = "locked scheduled check";
			}
			else {
				reason = "normal scheduled check";
			}
		}

		jittered_sec = rspamd_time_jitter(timeout, 0);
	}

	/* Now, we do some sanity checks for jittered seconds */
	if (!(how & RSPAMD_MAP_SCHEDULE_INIT)) {
		/* Never allow too low interval between timer checks, it is expensive */
		if (jittered_sec < min_timer_interval) {
			jittered_sec = rspamd_time_jitter(min_timer_interval, 0);
		}

		if (map->non_trivial) {
			/*
			 * Even if we are reported that we need to reload cache often, we
			 * still want to be sane in terms of events...
			 */
			if (jittered_sec < min_timer_interval * 2.0) {
				if (map->nelts > 0) {
					jittered_sec = min_timer_interval * 3.0;
				}
			}
		}
	}

	cbd = g_malloc0(sizeof(*cbd));
	cbd->cbdata.prev_data = *map->user_data;
	cbd->cbdata.cur_data = NULL;
	cbd->cbdata.map = map;
	cbd->map = map;
	map->scheduled_check = cbd;
	REF_INIT_RETAIN(cbd, rspamd_map_periodic_dtor);

	cbd->ev.data = cbd;
	ev_timer_init(&cbd->ev, rspamd_map_periodic_callback, jittered_sec, 0.0);
	ev_timer_start(map->event_loop, &cbd->ev);

	msg_debug_map("schedule new periodic event %p in %.3f seconds for %s; reason: %s",
				  cbd, jittered_sec, map->name, reason);
}

static int
rspamd_map_af_to_weight(const rspamd_inet_addr_t *addr)
{
	int ret;

	switch (rspamd_inet_address_get_af(addr)) {
	case AF_UNIX:
		ret = 2;
		break;
	case AF_INET:
		ret = 1;
		break;
	default:
		ret = 0;
		break;
	}

	return ret;
}

static int
rspamd_map_dns_address_sort_func(gconstpointer a, gconstpointer b)
{
	const rspamd_inet_addr_t *ip1 = *(const rspamd_inet_addr_t **) a,
							 *ip2 = *(const rspamd_inet_addr_t **) b;
	int w1, w2;

	w1 = rspamd_map_af_to_weight(ip1);
	w2 = rspamd_map_af_to_weight(ip2);

	/* Inverse order */
	return w2 - w1;
}

static void
rspamd_map_dns_callback(struct rdns_reply *reply, void *arg)
{
	struct http_callback_data *cbd = arg;
	struct rdns_reply_entry *cur_rep;
	struct rspamd_map *map;
	unsigned int flags = RSPAMD_HTTP_CLIENT_SIMPLE | RSPAMD_HTTP_CLIENT_SHARED;

	map = cbd->map;

	msg_debug_map("got dns reply with code %s on stage %d",
				  rdns_strerror(reply->code), cbd->stage);

	if (cbd->stage == http_map_terminated) {
		MAP_RELEASE(cbd, "http_callback_data");
		return;
	}

	if (reply->code == RDNS_RC_NOERROR) {
		DL_FOREACH(reply->entries, cur_rep)
		{
			rspamd_inet_addr_t *addr;
			addr = rspamd_inet_address_from_rnds(cur_rep);

			if (addr != NULL) {
				rspamd_inet_address_set_port(addr, cbd->data->port);
				g_ptr_array_add(cbd->addrs, (void *) addr);
			}
		}

		if (cbd->stage == http_map_resolve_host2) {
			/* We have still one request pending */
			cbd->stage = http_map_resolve_host1;
		}
		else if (cbd->stage == http_map_resolve_host1) {
			cbd->stage = http_map_http_conn;
		}
	}
	else if (cbd->stage < http_map_http_conn) {
		if (cbd->stage == http_map_resolve_host2) {
			/* We have still one request pending */
			cbd->stage = http_map_resolve_host1;
		}
		else if (cbd->addrs->len == 0) {
			/* We could not resolve host, so cowardly fail here */
			msg_err_map("cannot resolve %s: %s", cbd->data->host,
						rdns_strerror(reply->code));
			cbd->periodic->errored = 1;
			rspamd_map_process_periodic(cbd->periodic);
		}
		else {
			/* We have at least one address, so we can continue... */
			cbd->stage = http_map_http_conn;
		}
	}

	if (cbd->stage == http_map_http_conn && cbd->addrs->len > 0) {
		rspamd_ptr_array_shuffle(cbd->addrs);
		int idx = 0;
		/*
		 * For the existing addr we can just select any address as we have
		 * data available
		 */
		if (cbd->map->nelts > 0 && rspamd_random_double_fast() > 0.5) {
			/* Already shuffled, use whatever is the first */
			cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx);
		}
		else {
			/* Always prefer IPv4 as IPv6 is almost all the time broken */
			g_ptr_array_sort(cbd->addrs, rspamd_map_dns_address_sort_func);
			cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx);
		}

	retry:
		msg_debug_map("try open http connection to %s",
					  rspamd_inet_address_to_string_pretty(cbd->addr));
		if (cbd->bk->protocol == MAP_PROTO_HTTPS) {
			flags |= RSPAMD_HTTP_CLIENT_SSL;
		}
		cbd->conn = rspamd_http_connection_new_client(NULL,
													  NULL,
													  http_map_error,
													  http_map_finish,
													  flags,
													  cbd->addr);

		if (cbd->conn != NULL) {
			/* Apply optional staged timeouts and keepalive tuning */
			if (cbd->data->connect_timeout > 0 || cbd->data->ssl_timeout > 0 ||
				cbd->data->write_timeout > 0 || cbd->data->read_timeout > 0) {
				rspamd_http_connection_set_timeouts(cbd->conn,
													cbd->data->connect_timeout,
													cbd->data->ssl_timeout,
													cbd->data->write_timeout,
													cbd->data->read_timeout);
			}
			if (cbd->data->connection_ttl > 0 || cbd->data->idle_timeout > 0 || cbd->data->max_reuse > 0) {
				rspamd_http_connection_set_keepalive_tuning(cbd->conn,
															cbd->data->connection_ttl,
															cbd->data->idle_timeout,
															cbd->data->max_reuse);
			}
			write_http_request(cbd);
		}
		else {
			if (idx < cbd->addrs->len - 1) {
				/* We can retry */
				idx++;
				rspamd_inet_addr_t *prev_addr = cbd->addr;
				cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx);
				msg_info_map("cannot connect to %s to get data for %s: %s, retry with %s (%d of %d)",
							 rspamd_inet_address_to_string_pretty(prev_addr),
							 cbd->bk->uri,
							 strerror(errno),
							 rspamd_inet_address_to_string_pretty(cbd->addr),
							 idx + 1, cbd->addrs->len);
				goto retry;
			}
			else {
				/* Nothing else left */
				cbd->periodic->errored = TRUE;
				msg_err_map("error reading %s(%s): "
							"connection with http server terminated incorrectly: %s",
							cbd->bk->uri,
							cbd->addr ? rspamd_inet_address_to_string_pretty(cbd->addr) : "",
							strerror(errno));

				rspamd_map_process_periodic(cbd->periodic);
			}
		}
	}

	MAP_RELEASE(cbd, "http_callback_data");
}

static gboolean
rspamd_map_read_cached(struct rspamd_map *map, struct rspamd_map_backend *bk,
					   struct map_periodic_cbdata *periodic, const char *host)
{
	gsize mmap_len, len;
	gpointer in;
	struct http_map_data *data;

	data = bk->data.hd;

	if (map->no_file_read) {
		/*
		 * For no_file_read maps, the cache file was already written by the
		 * controller process. Pass its path to the read callback.
		 * The consumer mmaps the file at no_file_read_offset to skip the header.
		 */
		char cache_path[PATH_MAX];
		struct stat st;

		if (map->cfg->maps_cache_dir == NULL || map->cfg->maps_cache_dir[0] == '\0') {
			msg_err_map("%s: no maps cache dir configured for no_file_read map", bk->uri);
			return FALSE;
		}

		rspamd_map_cache_file_path(bk, map->cfg->maps_cache_dir,
								   cache_path, sizeof(cache_path));

		if (stat(cache_path, &st) == -1 ||
			st.st_size <= (off_t) RSPAMD_MAP_CACHE_HEADER_SIZE) {
			msg_info_map("%s: cache file %s not available yet", bk->uri, cache_path);
			return FALSE;
		}

		map->no_file_read_offset = RSPAMD_MAP_CACHE_HEADER_SIZE;
		msg_info_map("%s: read cached file %s, %uz bytes (offset %d)",
					 bk->uri, cache_path,
					 (gsize) (st.st_size - RSPAMD_MAP_CACHE_HEADER_SIZE),
					 RSPAMD_MAP_CACHE_HEADER_SIZE);
		map->read_callback(cache_path, strlen(cache_path),
						   &periodic->cbdata, TRUE);
		return TRUE;
	}

	in = rspamd_shmem_xmap(data->cache->shmem_name, PROT_READ, &mmap_len);

	if (in == NULL) {
		msg_err("cannot map cache from %s: %s", data->cache->shmem_name,
				strerror(errno));
		return FALSE;
	}

	if (mmap_len < data->cache->len) {
		msg_err("cannot map cache from %s: truncated length %z, %z expected",
				data->cache->shmem_name,
				mmap_len, data->cache->len);
		munmap(in, mmap_len);

		return FALSE;
	}

	/*
	 * Len is taken from the shmem file size that can be larger than the
	 * actual data length, as we use shared memory as a growing buffer for the
	 * HTTP input.
	 * Hence, we need to use len from the saved cache data, counting that it is
	 * at least not more than the cached file length (this is checked above).
	 */
	len = data->cache->len;

	if (bk->is_encrypted || bk->is_compressed) {
		unsigned char *payload = (unsigned char *) in;
		gsize payload_len = len;
		unsigned char *dec = NULL;
		gsize declen = 0;

		if (bk->is_encrypted) {
			if (!rspamd_map_secretbox_decrypt_buf(bk, payload, payload_len, &dec, &declen)) {
				munmap(in, mmap_len);
				return FALSE;
			}
			payload = dec;
			payload_len = declen;
		}

		/* If compressed flag is set OR payload looks like zstd, decompress */
		if (bk->is_compressed || rspamd_map_payload_is_zstd(payload, payload_len)) {
			ZSTD_DStream *zstream;
			ZSTD_inBuffer zin;
			ZSTD_outBuffer zout;
			unsigned char *out;
			gsize outlen, r;

			zstream = ZSTD_createDStream();
			ZSTD_initDStream(zstream);

			zin.pos = 0;
			zin.src = payload;
			zin.size = payload_len;

			if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
				outlen = ZSTD_DStreamOutSize();
			}

			out = g_malloc(outlen);

			zout.dst = out;
			zout.pos = 0;
			zout.size = outlen;

			while (zin.pos < zin.size) {
				r = ZSTD_decompressStream(zstream, &zout, &zin);

				if (ZSTD_isError(r)) {
					msg_err_map("%s: cannot decompress data: %s",
								bk->uri,
								ZSTD_getErrorName(r));
					ZSTD_freeDStream(zstream);
					g_free(out);
					if (dec) {
						rspamd_explicit_memzero(dec, declen);
						g_free(dec);
					}
					munmap(in, mmap_len);
					return FALSE;
				}

				if (zout.pos == zout.size) {
					/* We need to extend output buffer */
					zout.size = zout.size * 2 + 1;
					out = g_realloc(zout.dst, zout.size);
					zout.dst = out;
				}
			}

			ZSTD_freeDStream(zstream);
			msg_info_map("%s: read map data cached %z bytes compressed, "
						 "%z uncompressed",
						 bk->uri,
						 payload_len, zout.pos);
			map->read_callback(out, zout.pos, &periodic->cbdata, TRUE);
			g_free(out);
			if (dec) {
				rspamd_explicit_memzero(dec, declen);
				g_free(dec);
			}
		}
		else {
			msg_info_map("%s: read map data cached %z bytes%s", bk->uri, payload_len,
						 bk->is_encrypted ? " (decrypted)" : "");
			map->read_callback(payload, payload_len, &periodic->cbdata, TRUE);
			if (dec) {
				rspamd_explicit_memzero(dec, declen);
				g_free(dec);
			}
		}

		munmap(in, mmap_len);

		return TRUE;
	}
	else {
		/* Neither encrypted nor compressed: pass cached bytes as-is */
		msg_info_map("%s: read map data cached %z bytes", bk->uri, len);
		map->read_callback(in, len, &periodic->cbdata, TRUE);
		munmap(in, mmap_len);
		return TRUE;
	}
}

static gboolean
rspamd_map_has_http_cached_file(struct rspamd_map *map,
								struct rspamd_map_backend *bk)
{
	char path[PATH_MAX];
	struct rspamd_config *cfg = map->cfg;
	struct stat st;

	if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') {
		return FALSE;
	}

	rspamd_map_cache_file_path(bk, cfg->maps_cache_dir, path, sizeof(path));

	if (stat(path, &st) != -1 && st.st_size >
									 sizeof(struct rspamd_http_file_data)) {
		return TRUE;
	}

	return FALSE;
}

static gboolean
rspamd_map_save_http_cached_file(struct rspamd_map *map,
								 struct rspamd_map_backend *bk,
								 struct http_map_data *htdata,
								 const unsigned char *data,
								 gsize len)
{
	char path[PATH_MAX], temp_path[PATH_MAX];
	struct rspamd_config *cfg = map->cfg;
	int fd;
	struct rspamd_http_file_data header;

	if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') {
		return FALSE;
	}

	rspamd_map_cache_file_path(bk, cfg->maps_cache_dir, path, sizeof(path));
	rspamd_snprintf(temp_path, sizeof(temp_path), "%s.tmp.%d.%d", path,
					(int) getpid(), (int) rspamd_get_calendar_ticks());

	fd = rspamd_file_xopen(temp_path, O_WRONLY | O_TRUNC | O_CREAT,
						   00600, FALSE);

	if (fd == -1) {
		return FALSE;
	}

	if (!rspamd_file_lock(fd, FALSE)) {
		msg_err_map("cannot lock file %s: %s", temp_path, strerror(errno));
		close(fd);
		unlink(temp_path);

		return FALSE;
	}

	memset(&header, 0, sizeof(header));
	memcpy(header.magic, rspamd_http_file_magic, sizeof(rspamd_http_file_magic));
	header.mtime = htdata->last_modified;
	header.next_check = map->next_check;
	header.data_off = RSPAMD_MAP_CACHE_HEADER_SIZE;

	if (htdata->etag) {
		gsize max_etag = RSPAMD_MAP_CACHE_HEADER_SIZE - sizeof(header);

		if (RSPAMD_FSTRING_LEN(htdata->etag) > max_etag) {
			msg_warn_map("etag too long (%z > %z), truncating",
						 RSPAMD_FSTRING_LEN(htdata->etag), max_etag);
			header.etag_len = max_etag;
		}
		else {
			header.etag_len = RSPAMD_FSTRING_LEN(htdata->etag);
		}
	}
	else {
		header.etag_len = 0;
	}

	/*
	 * Write the full header page: struct + etag + zero padding.
	 * Data payload starts at page-aligned offset (4096) so consumers
	 * can mmap the file at that offset directly.
	 */
	unsigned char header_page[RSPAMD_MAP_CACHE_HEADER_SIZE];
	memset(header_page, 0, sizeof(header_page));
	memcpy(header_page, &header, sizeof(header));

	if (header.etag_len > 0) {
		memcpy(header_page + sizeof(header),
			   RSPAMD_FSTRING_DATA(htdata->etag), header.etag_len);
	}

	if (write(fd, header_page, sizeof(header_page)) != sizeof(header_page)) {
		msg_err_map("cannot write file %s (header stage): %s", temp_path, strerror(errno));
		rspamd_file_unlock(fd, FALSE);
		close(fd);
		unlink(temp_path);

		return FALSE;
	}

	/* Now write the payload */
	if (write(fd, data, len) != (gssize) len) {
		msg_err_map("cannot write file %s (data stage): %s", temp_path, strerror(errno));
		rspamd_file_unlock(fd, FALSE);
		close(fd);
		unlink(temp_path);

		return FALSE;
	}

	rspamd_file_unlock(fd, FALSE);
	close(fd);

	/* Atomically move temp file to final location */
	if (rename(temp_path, path) != 0) {
		msg_err_map("cannot rename %s to %s: %s", temp_path, path, strerror(errno));
		unlink(temp_path);
		return FALSE;
	}

	msg_info_map("saved data from %s in %s, %uz bytes",
				 bk->uri, path, len);

	return TRUE;
}

static gboolean
rspamd_map_update_http_cached_file(struct rspamd_map *map,
								   struct rspamd_map_backend *bk,
								   struct http_map_data *htdata)
{
	char path[PATH_MAX];
	struct rspamd_config *cfg = map->cfg;
	int fd;
	struct rspamd_http_file_data header;

	if (!rspamd_map_has_http_cached_file(map, bk)) {
		return FALSE;
	}

	rspamd_map_cache_file_path(bk, cfg->maps_cache_dir, path, sizeof(path));

	fd = rspamd_file_xopen(path, O_WRONLY,
						   00600, FALSE);

	if (fd == -1) {
		return FALSE;
	}

	if (!rspamd_file_lock(fd, FALSE)) {
		msg_err_map("cannot lock file %s: %s", path, strerror(errno));
		close(fd);

		return FALSE;
	}

	memset(&header, 0, sizeof(header));
	memcpy(header.magic, rspamd_http_file_magic, sizeof(rspamd_http_file_magic));
	header.mtime = htdata->last_modified;
	header.next_check = map->next_check;
	header.data_off = RSPAMD_MAP_CACHE_HEADER_SIZE;

	if (htdata->etag) {
		gsize max_etag = RSPAMD_MAP_CACHE_HEADER_SIZE - sizeof(header);

		if (RSPAMD_FSTRING_LEN(htdata->etag) > max_etag) {
			header.etag_len = max_etag;
		}
		else {
			header.etag_len = RSPAMD_FSTRING_LEN(htdata->etag);
		}
	}
	else {
		header.etag_len = 0;
	}

	/* Write the full page-aligned header (only the header page, not payload) */
	unsigned char header_page[RSPAMD_MAP_CACHE_HEADER_SIZE];
	memset(header_page, 0, sizeof(header_page));
	memcpy(header_page, &header, sizeof(header));

	if (header.etag_len > 0) {
		memcpy(header_page + sizeof(header),
			   RSPAMD_FSTRING_DATA(htdata->etag), header.etag_len);
	}

	if (write(fd, header_page, sizeof(header_page)) != sizeof(header_page)) {
		msg_err_map("cannot update file %s (header stage): %s", path, strerror(errno));
		rspamd_file_unlock(fd, FALSE);
		close(fd);

		return FALSE;
	}

	rspamd_file_unlock(fd, FALSE);
	close(fd);

	return TRUE;
}

static gboolean
rspamd_map_upgrade_http_cached_file(struct rspamd_map *map,
									struct rspamd_map_backend *bk,
									struct http_map_data *htdata,
									goffset legacy_data_off)
{
	char path[PATH_MAX];
	struct rspamd_config *cfg = map->cfg;
	gsize flen;
	unsigned char *fbytes;
	const unsigned char *payload;
	gsize payload_len;
	gboolean ret = FALSE;

	if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') {
		return FALSE;
	}

	rspamd_map_cache_file_path(bk, cfg->maps_cache_dir, path, sizeof(path));
	fbytes = rspamd_file_xmap(path, PROT_READ, &flen, TRUE);

	if (fbytes == NULL) {
		msg_err_map("cannot open legacy cache file %s: %s", path, strerror(errno));
		return FALSE;
	}

	if (legacy_data_off < (goffset) sizeof(struct rspamd_http_file_data) ||
		legacy_data_off >= (goffset) flen) {
		msg_warn_map("legacy cache file %s has invalid data offset: %z, file size: %z",
					 path, (gsize) legacy_data_off, flen);
		goto end;
	}

	payload = fbytes + legacy_data_off;
	payload_len = flen - legacy_data_off;

	if (!rspamd_map_save_http_cached_file(map, bk, htdata, payload, payload_len)) {
		goto end;
	}

	msg_info_map("upgraded legacy cache file %s to page-aligned format", path);
	ret = TRUE;

end:
	munmap(fbytes, flen);

	return ret;
}

static gboolean
rspamd_map_read_http_cached_file(struct rspamd_map *map,
								 struct rspamd_map_backend *bk,
								 struct http_map_data *htdata,
								 struct map_cb_data *cbdata)
{
	char path[PATH_MAX];
	struct rspamd_config *cfg = map->cfg;
	int fd;
	struct stat st;
	struct rspamd_http_file_data header;
	goffset legacy_data_off = -1;

	if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') {
		return FALSE;
	}

	rspamd_map_cache_file_path(bk, cfg->maps_cache_dir, path, sizeof(path));

	fd = rspamd_file_xopen(path, O_RDONLY, 00600, FALSE);

	if (fd == -1) {
		return FALSE;
	}

	if (!rspamd_file_lock(fd, FALSE)) {
		msg_err_map("cannot lock file %s: %s", path, strerror(errno));
		close(fd);

		return FALSE;
	}

	(void) fstat(fd, &st);

	if (read(fd, &header, sizeof(header)) != sizeof(header)) {
		msg_err_map("cannot read file %s (header stage): %s", path, strerror(errno));
		rspamd_file_unlock(fd, FALSE);
		close(fd);

		return FALSE;
	}

	if (memcmp(header.magic, rspamd_http_file_magic,
			   sizeof(rspamd_http_file_magic)) != 0) {
		if (memcmp(header.magic, rspamd_http_file_magic_old,
				   sizeof(rspamd_http_file_magic_old)) == 0) {
			msg_info_map("old version cache file %s; will be rewritten on next update", path);
			/* Old format: data_off is header + etag, not page-aligned */
			legacy_data_off = header.data_off;
		}
		else {
			msg_warn_map("invalid magic in file %s; ignore it", path);
			rspamd_file_unlock(fd, FALSE);
			close(fd);

			return FALSE;
		}
	}

	double now = rspamd_get_calendar_ticks();

	if (header.next_check > now) {
		/* We assume that we have this data inside the cached file */
		map->next_check = rspamd_http_map_process_next_check(map, bk, now, header.next_check,
															 map->poll_timeout,
															 header.etag_len > 0,
															 true);
	}
	else {
		map->next_check = now;
	}

	htdata->last_modified = header.mtime;

	if (header.etag_len > 0) {
		rspamd_fstring_t *etag = rspamd_fstring_sized_new(header.etag_len);

		if (read(fd, RSPAMD_FSTRING_DATA(etag), header.etag_len) != header.etag_len) {
			msg_err_map("cannot read file %s (etag stage): %s", path,
						strerror(errno));
			rspamd_file_unlock(fd, FALSE);
			rspamd_fstring_free(etag);
			close(fd);

			return FALSE;
		}

		etag->len = header.etag_len;

		if (htdata->etag) {
			/* FIXME: should be dealt somehow better */
			msg_warn_map("etag is already defined as %V; cached is %V; ignore cached",
						 htdata->etag, etag);
			rspamd_fstring_free(etag);
		}
		else {
			htdata->etag = etag;
		}
	}

	rspamd_file_unlock(fd, FALSE);
	close(fd);

	if (legacy_data_off != -1) {
		if (!rspamd_map_upgrade_http_cached_file(map, bk, htdata, legacy_data_off)) {
			unlink(path);
			return FALSE;
		}

		header.data_off = RSPAMD_MAP_CACHE_HEADER_SIZE;

		if (stat(path, &st) == -1) {
			msg_err_map("cannot stat upgraded cache file %s: %s", path, strerror(errno));
			return FALSE;
		}
	}

	if (map->no_file_read) {
		/*
		 * For no_file_read maps, pass the cache file path to the consumer.
		 * The consumer will mmap the file at no_file_read_offset to skip
		 * the page-aligned cache header.
		 */
		map->no_file_read_offset = header.data_off;
		msg_info_map("%s: passing cached file %s for no_file_read (offset %z)",
					 bk->uri, path, (gsize) header.data_off);
		map->read_callback(path, strlen(path), cbdata, TRUE);
	}
	else {
		if (!read_map_file_chunks(map, cbdata, path,
								  st.st_size - header.data_off, header.data_off)) {
			return FALSE;
		}
	}

	struct tm tm;
	char ncheck_buf[32], lm_buf[32];

	g_atomic_int_set(&map->shared->loaded, 1);
	g_atomic_int_set(&map->shared->cached, 1);

	rspamd_localtime(map->next_check, &tm);
	strftime(ncheck_buf, sizeof(ncheck_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm);
	rspamd_localtime(htdata->last_modified, &tm);
	strftime(lm_buf, sizeof(lm_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm);

	msg_info_map("read cached data for %s from %s, %uz bytes; next check at: %s;"
				 " last modified on: %s; etag: %V",
				 bk->uri,
				 path,
				 (size_t) (st.st_size - header.data_off),
				 ncheck_buf,
				 lm_buf,
				 htdata->etag);

	return TRUE;
}

/**
 * Async HTTP callback
 */
static void
rspamd_map_common_http_callback(struct rspamd_map *map,
								struct rspamd_map_backend *bk,
								struct map_periodic_cbdata *periodic,
								gboolean check)
{
	struct http_map_data *data;
	struct http_callback_data *cbd;
	unsigned int flags = RSPAMD_HTTP_CLIENT_SIMPLE | RSPAMD_HTTP_CLIENT_SHARED;

	data = bk->data.hd;

	if (g_atomic_int_get(&data->cache->available) == 1) {
		/* Read cached data */
		if (check) {
			if (data->last_modified < data->cache->last_modified) {
				msg_info_map("need to reread cached map triggered by %s "
							 "(%d our modify time, %d cached modify time)",
							 bk->uri,
							 (int) data->last_modified,
							 (int) data->cache->last_modified);
				periodic->need_modify = TRUE;
				periodic->cur_backend = 0;
				rspamd_map_process_periodic(periodic);
			}
			else {
				if (map->active_http) {
					/* Check even if there is a cached version */
					goto check;
				}
				else {
					/* Switch to the next backend */
					periodic->cur_backend++;
					rspamd_map_process_periodic(periodic);
				}
			}

			return;
		}
		else {
			if (map->active_http &&
				data->last_modified > data->cache->last_modified) {
				goto check;
			}
			else if (rspamd_map_read_cached(map, bk, periodic, data->host)) {
				/* Switch to the next backend */
				periodic->cur_backend++;
				data->last_modified = data->cache->last_modified;
				rspamd_map_process_periodic(periodic);

				return;
			}
		}
	}
	else if (!map->active_http) {
		/* Switch to the next backend */
		periodic->cur_backend++;
		rspamd_map_process_periodic(periodic);

		return;
	}

check:
	cbd = g_malloc0(sizeof(struct http_callback_data));

	cbd->event_loop = map->event_loop;
	cbd->addrs = g_ptr_array_sized_new(4);
	cbd->map = map;
	cbd->data = data;
	cbd->check = check;
	cbd->periodic = periodic;
	MAP_RETAIN(periodic, "periodic");
	cbd->bk = bk;
	MAP_RETAIN(bk, "rspamd_map_backend");
	cbd->stage = http_map_terminated;
	REF_INIT_RETAIN(cbd, free_http_cbdata);

	msg_debug_map("%s map data from %s", check ? "checking" : "reading",
				  data->host);

	/* Try address */
	rspamd_inet_addr_t *addr = NULL;

	if (rspamd_parse_inet_address(&addr, data->host,
								  strlen(data->host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
		rspamd_inet_address_set_port(addr, cbd->data->port);
		g_ptr_array_add(cbd->addrs, (void *) addr);

		if (bk->protocol == MAP_PROTO_HTTPS) {
			flags |= RSPAMD_HTTP_CLIENT_SSL;
		}

		cbd->conn = rspamd_http_connection_new_client(
			NULL,
			NULL,
			http_map_error,
			http_map_finish,
			flags,
			addr);

		if (cbd->conn != NULL) {
			/* Apply optional staged timeouts and keepalive tuning */
			if (cbd->data->connect_timeout > 0 || cbd->data->ssl_timeout > 0 ||
				cbd->data->write_timeout > 0 || cbd->data->read_timeout > 0) {
				rspamd_http_connection_set_timeouts(cbd->conn,
													cbd->data->connect_timeout,
													cbd->data->ssl_timeout,
													cbd->data->write_timeout,
													cbd->data->read_timeout);
			}
			if (cbd->data->connection_ttl > 0 || cbd->data->idle_timeout > 0 || cbd->data->max_reuse > 0) {
				rspamd_http_connection_set_keepalive_tuning(cbd->conn,
															cbd->data->connection_ttl,
															cbd->data->idle_timeout,
															cbd->data->max_reuse);
			}
			write_http_request(cbd);
			cbd->addr = addr;
			MAP_RELEASE(cbd, "http_callback_data");
		}
		else {
			msg_warn_map("cannot load map: cannot connect to %s: %s",
						 data->host, strerror(errno));
			MAP_RELEASE(cbd, "http_callback_data");
		}

		return;
	}
	else if (map->r->r) {
		/* Send both A and AAAA requests */
		unsigned int nreq = 0;

		if (rdns_make_request_full(map->r->r, rspamd_map_dns_callback, cbd,
								   map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
								   data->host, RDNS_REQUEST_A)) {
			MAP_RETAIN(cbd, "http_callback_data");
			nreq++;
		}
		if (rdns_make_request_full(map->r->r, rspamd_map_dns_callback, cbd,
								   map->cfg->dns_timeout, map->cfg->dns_retransmits, 1,
								   data->host, RDNS_REQUEST_AAAA)) {
			MAP_RETAIN(cbd, "http_callback_data");
			nreq++;
		}

		if (nreq == 2) {
			cbd->stage = http_map_resolve_host2;
		}
		else if (nreq == 1) {
			cbd->stage = http_map_resolve_host1;
		}

		map->tmp_dtor = free_http_cbdata_dtor;
		map->tmp_dtor_data = cbd;
	}
	else {
		msg_warn_map("cannot load map: DNS resolver is not initialized");
		cbd->periodic->errored = TRUE;
	}

	MAP_RELEASE(cbd, "http_callback_data");
}

static void
rspamd_map_http_check_callback(struct map_periodic_cbdata *cbd)
{
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;

	map = cbd->map;
	bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend);

	rspamd_map_common_http_callback(map, bk, cbd, TRUE);
}

static void
rspamd_map_http_read_callback(struct map_periodic_cbdata *cbd)
{
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;

	map = cbd->map;
	bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend);
	rspamd_map_common_http_callback(map, bk, cbd, FALSE);
}

static void
rspamd_map_file_check_callback(struct map_periodic_cbdata *periodic)
{
	struct rspamd_map *map;
	struct file_map_data *data;
	struct rspamd_map_backend *bk;

	map = periodic->map;
	bk = g_ptr_array_index(map->backends, periodic->cur_backend);
	data = bk->data.fd;

	if (data->need_modify) {
		periodic->need_modify = TRUE;
		periodic->cur_backend = 0;
		data->need_modify = FALSE;

		rspamd_map_process_periodic(periodic);

		return;
	}

	map = periodic->map;
	/* Switch to the next backend as the rest is handled by ev_stat */
	periodic->cur_backend++;
	rspamd_map_process_periodic(periodic);
}

static void
rspamd_map_static_check_callback(struct map_periodic_cbdata *periodic)
{
	struct rspamd_map *map;
	struct static_map_data *data;
	struct rspamd_map_backend *bk;

	map = periodic->map;
	bk = g_ptr_array_index(map->backends, periodic->cur_backend);
	data = bk->data.sd;

	if (!data->processed) {
		periodic->need_modify = TRUE;
		periodic->cur_backend = 0;

		rspamd_map_process_periodic(periodic);

		return;
	}

	/* Switch to the next backend */
	periodic->cur_backend++;
	rspamd_map_process_periodic(periodic);
}

static void
rspamd_map_file_read_callback(struct map_periodic_cbdata *periodic)
{
	struct rspamd_map *map;
	struct file_map_data *data;
	struct rspamd_map_backend *bk;

	map = periodic->map;

	bk = g_ptr_array_index(map->backends, periodic->cur_backend);
	data = bk->data.fd;

	msg_info_map("rereading map file %s", data->filename);

	if (!read_map_file(map, data, bk, periodic)) {
		periodic->errored = TRUE;
	}

	/* Switch to the next backend */
	periodic->cur_backend++;
	rspamd_map_process_periodic(periodic);
}

static void
rspamd_map_static_read_callback(struct map_periodic_cbdata *periodic)
{
	struct rspamd_map *map;
	struct static_map_data *data;
	struct rspamd_map_backend *bk;

	map = periodic->map;

	bk = g_ptr_array_index(map->backends, periodic->cur_backend);
	data = bk->data.sd;

	msg_info_map("rereading static map");

	if (!read_map_static(map, data, bk, periodic)) {
		periodic->errored = TRUE;
	}

	/* Switch to the next backend */
	periodic->cur_backend++;
	rspamd_map_process_periodic(periodic);
}

static void
rspamd_map_process_periodic(struct map_periodic_cbdata *cbd)
{
	struct rspamd_map_backend *bk;
	struct rspamd_map *map;

	map = cbd->map;
	map->scheduled_check = NULL;

	/* For each backend we need to check for modifications */
	if (cbd->cur_backend >= cbd->map->backends->len) {
		/* Last backend */
		msg_debug_map("finished map: %d of %d", cbd->cur_backend,
					  cbd->map->backends->len);
		MAP_RELEASE(cbd, "periodic");

		return;
	}

	bk = g_ptr_array_index(map->backends, cbd->cur_backend);

	if (cbd->errored) {
		/* We should not check other backends if some backend has failed*/
		rspamd_map_schedule_periodic(cbd->map, RSPAMD_MAP_SCHEDULE_ERROR);

		/* Also set error flag for the map consumer */
		cbd->cbdata.errored = true;

		msg_debug_map("unlocked map %s, refcount=%d", cbd->map->name,
					  cbd->ref.refcount);
		MAP_RELEASE(cbd, "periodic");

		return;
	}

	if (cbd->map->wrk && cbd->map->wrk->state == rspamd_worker_state_running) {

		if (cbd->need_modify) {
			/* Load data from the next backend */
			switch (bk->protocol) {
			case MAP_PROTO_HTTP:
			case MAP_PROTO_HTTPS:
				rspamd_map_http_read_callback(cbd);
				break;
			case MAP_PROTO_FILE:
				rspamd_map_file_read_callback(cbd);
				break;
			case MAP_PROTO_STATIC:
				rspamd_map_static_read_callback(cbd);
				break;
			}
		}
		else {
			/* Check the next backend */
			switch (bk->protocol) {
			case MAP_PROTO_HTTP:
			case MAP_PROTO_HTTPS:
				rspamd_map_http_check_callback(cbd);
				break;
			case MAP_PROTO_FILE:
				rspamd_map_file_check_callback(cbd);
				break;
			case MAP_PROTO_STATIC:
				rspamd_map_static_check_callback(cbd);
				break;
			}
		}
	}
}

static void
rspamd_map_on_stat(struct ev_loop *loop, ev_stat *w, int revents)
{
	struct rspamd_map *map = (struct rspamd_map *) w->data;

	if (w->attr.st_nlink > 0) {
		msg_info_map("old mtime is %t (size = %Hz), "
					 "new mtime is %t (size = %Hz) for map file %s",
					 w->prev.st_mtime, (gsize) w->prev.st_size,
					 w->attr.st_mtime, (gsize) w->attr.st_size,
					 w->path);

		/* Fire need modify flag */
		struct rspamd_map_backend *bk;
		unsigned int i;

		PTR_ARRAY_FOREACH(map->backends, i, bk)
		{
			if (bk->protocol == MAP_PROTO_FILE) {
				bk->data.fd->need_modify = TRUE;
			}
		}

		map->next_check = 0;

		if (map->scheduled_check) {
			ev_timer_stop(map->event_loop, &map->scheduled_check->ev);
			MAP_RELEASE(map->scheduled_check, "rspamd_map_on_stat");
			map->scheduled_check = NULL;
		}

		rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_INIT);
	}
}

/* Start watching event for all maps */
void rspamd_map_watch(struct rspamd_config *cfg,
					  struct ev_loop *event_loop,
					  struct rspamd_dns_resolver *resolver,
					  struct rspamd_worker *worker,
					  enum rspamd_map_watch_type how)
{
	GList *cur = cfg->maps;
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;
	unsigned int i;

	g_assert(how > RSPAMD_MAP_WATCH_MIN && how < RSPAMD_MAP_WATCH_MAX);

	/* First of all do synced read of data */
	while (cur) {
		map = cur->data;
		map->event_loop = event_loop;
		map->r = resolver;

		if (map->wrk == NULL && how != RSPAMD_MAP_WATCH_WORKER) {
			/* Generic scanner map */
			map->wrk = worker;

			if (how == RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER) {
				map->active_http = TRUE;
			}
			else {
				map->active_http = FALSE;
			}
		}
		else if (map->wrk != NULL && map->wrk == worker) {
			/* Map is bound to a specific worker */
			map->active_http = TRUE;
		}
		else {
			/* Skip map for this worker as irrelevant */
			cur = g_list_next(cur);
			continue;
		}

		if (!map->active_http) {
			/* Check cached version more frequently as it is cheap */

			if (map->poll_timeout >= cfg->map_timeout &&
				cfg->map_file_watch_multiplier < 1.0) {
				map->poll_timeout =
					map->poll_timeout * cfg->map_file_watch_multiplier;
			}
		}

		map->file_only = TRUE;
		map->static_only = TRUE;

		PTR_ARRAY_FOREACH(map->backends, i, bk)
		{
			bk->event_loop = event_loop;

			if (bk->protocol == MAP_PROTO_FILE) {
				struct file_map_data *data;

				data = bk->data.fd;

				if (!map->seen || map->user_data == NULL || *map->user_data == NULL) {
					/* Map has not been read, init it's reading if possible */
					struct stat st;

					if (stat(data->filename, &st) != -1) {
						data->need_modify = TRUE;
					}
				}

				ev_stat_init(&data->st_ev, rspamd_map_on_stat,
							 data->filename, map->poll_timeout * cfg->map_file_watch_multiplier);
				data->st_ev.data = map;
				ev_stat_start(event_loop, &data->st_ev);
				map->static_only = FALSE;
			}
			else if ((bk->protocol == MAP_PROTO_HTTP ||
					  bk->protocol == MAP_PROTO_HTTPS)) {
				if (map->active_http) {
					map->non_trivial = TRUE;
				}

				map->static_only = FALSE;
				map->file_only = FALSE;
			}
		}

		rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_INIT);

		cur = g_list_next(cur);
	}
}

void rspamd_map_preload(struct rspamd_config *cfg)
{
	GList *cur = cfg->maps;
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;
	unsigned int i;
	gboolean map_ok;

	/* First of all do synced read of data */
	while (cur) {
		map = cur->data;
		map_ok = TRUE;

		PTR_ARRAY_FOREACH(map->backends, i, bk)
		{
			if (!(bk->protocol == MAP_PROTO_FILE ||
				  bk->protocol == MAP_PROTO_STATIC)) {

				if (bk->protocol == MAP_PROTO_HTTP ||
					bk->protocol == MAP_PROTO_HTTPS) {
					if (!rspamd_map_has_http_cached_file(map, bk)) {

						if (!map->fallback_backend) {
							map_ok = FALSE;
						}
						break;
					}
					else {
						continue; /* We are yet fine */
					}
				}
				map_ok = FALSE;
				break;
			}
		}

		if (map_ok) {
			struct map_periodic_cbdata fake_cbd;
			gboolean succeed = TRUE;

			memset(&fake_cbd, 0, sizeof(fake_cbd));
			fake_cbd.cbdata.state = 0;
			fake_cbd.cbdata.prev_data = *map->user_data;
			fake_cbd.cbdata.cur_data = NULL;
			fake_cbd.cbdata.map = map;
			fake_cbd.map = map;

			PTR_ARRAY_FOREACH(map->backends, i, bk)
			{
				fake_cbd.cur_backend = i;

				if (bk->protocol == MAP_PROTO_FILE) {
					if (!read_map_file(map, bk->data.fd, bk, &fake_cbd)) {
						succeed = FALSE;
						break;
					}
				}
				else if (bk->protocol == MAP_PROTO_STATIC) {
					if (!read_map_static(map, bk->data.sd, bk, &fake_cbd)) {
						succeed = FALSE;
						break;
					}
				}
				else if (bk->protocol == MAP_PROTO_HTTP ||
						 bk->protocol == MAP_PROTO_HTTPS) {
					if (!rspamd_map_read_http_cached_file(map, bk, bk->data.hd,
														  &fake_cbd.cbdata)) {

						if (map->fallback_backend) {
							/* Try fallback */
							g_assert(map->fallback_backend->protocol ==
									 MAP_PROTO_FILE);
							if (!read_map_file(map,
											   map->fallback_backend->data.fd,
											   map->fallback_backend, &fake_cbd)) {
								succeed = FALSE;
								break;
							}
						}
						else {
							succeed = FALSE;
							break;
						}
					}
				}
				else {
					g_assert_not_reached();
				}
			}

			if (succeed) {
				map->fin_callback(&fake_cbd.cbdata, map->user_data);

				if (map->on_load_function) {
					map->on_load_function(map, map->on_load_ud);
				}

				map->seen = true;
			}
			else {
				msg_info_map("preload of %s failed", map->name);
			}
		}

		cur = g_list_next(cur);
	}
}

void rspamd_map_remove_all(struct rspamd_config *cfg)
{
	struct rspamd_map *map;
	GList *cur;
	struct rspamd_map_backend *bk;
	struct map_cb_data cbdata;
	unsigned int i;

	for (cur = cfg->maps; cur != NULL; cur = g_list_next(cur)) {
		map = cur->data;

		if (map->tmp_dtor) {
			map->tmp_dtor(map->tmp_dtor_data);
		}

		if (map->dtor) {
			cbdata.prev_data = NULL;
			cbdata.map = map;
			cbdata.cur_data = *map->user_data;

			map->dtor(&cbdata);
			*map->user_data = NULL;
		}

		if (map->on_load_ud_dtor) {
			map->on_load_ud_dtor(map->on_load_ud);
		}

		for (i = 0; i < map->backends->len; i++) {
			bk = g_ptr_array_index(map->backends, i);

			MAP_RELEASE(bk, "rspamd_map_backend");
		}

		if (map->fallback_backend) {
			MAP_RELEASE(map->fallback_backend, "rspamd_map_backend");
		}
	}

	g_list_free(cfg->maps);
	cfg->maps = NULL;
}

static const char *
rspamd_map_check_proto(struct rspamd_config *cfg,
					   const char *map_line, struct rspamd_map_backend *bk)
{
	const char *pos = map_line, *end, *end_key;

	g_assert(bk != NULL);
	g_assert(pos != NULL);

	end = pos + strlen(pos);

	/* Static check */
	if (g_ascii_strcasecmp(pos, "static") == 0) {
		bk->protocol = MAP_PROTO_STATIC;
		bk->uri = g_strdup(pos);

		return pos;
	}
	else if (g_ascii_strcasecmp(pos, "zst+static") == 0) {
		bk->protocol = MAP_PROTO_STATIC;
		bk->uri = g_strdup(pos + 4);
		bk->is_compressed = TRUE;

		return pos + 4;
	}

	for (;;) {
		if (g_ascii_strncasecmp(pos, "sign+", sizeof("sign+") - 1) == 0) {
			bk->is_signed = TRUE;
			pos += sizeof("sign+") - 1;
		}
		else if (g_ascii_strncasecmp(pos, "fallback+", sizeof("fallback+") - 1) == 0) {
			bk->is_fallback = TRUE;
			pos += sizeof("fallback+") - 1;
		}
		else if (g_ascii_strncasecmp(pos, "key=", sizeof("key=") - 1) == 0) {
			pos += sizeof("key=") - 1;
			end_key = memchr(pos, '+', end - pos);

			if (end_key != NULL) {
				bk->trusted_pubkey = rspamd_pubkey_from_base32(pos, end_key - pos, RSPAMD_KEYPAIR_SIGN);

				if (bk->trusted_pubkey == NULL) {
					msg_err_config("cannot read pubkey from map: %s",
								   map_line);
					return NULL;
				}
				pos = end_key + 1;
			}
			else if (end - pos > 64) {
				/* Try hex encoding */
				bk->trusted_pubkey = rspamd_pubkey_from_hex(pos, 64, RSPAMD_KEYPAIR_SIGN);

				if (bk->trusted_pubkey == NULL) {
					msg_err_config("cannot read pubkey from map: %s",
								   map_line);
					return NULL;
				}
				pos += 64;
			}
			else {
				msg_err_config("cannot read pubkey from map: %s",
							   map_line);
				return NULL;
			}

			if (*pos == '+' || *pos == ':') {
				pos++;
			}
		}
		else {
			/* No known flags */
			break;
		}
	}

	bk->protocol = MAP_PROTO_FILE;

	if (g_ascii_strncasecmp(pos, "http://", sizeof("http://") - 1) == 0) {
		bk->protocol = MAP_PROTO_HTTP;
		/* Include http:// */
		bk->uri = g_strdup(pos);
		pos += sizeof("http://") - 1;
	}
	else if (g_ascii_strncasecmp(pos, "https://", sizeof("https://") - 1) == 0) {
		bk->protocol = MAP_PROTO_HTTPS;
		/* Include https:// */
		bk->uri = g_strdup(pos);
		pos += sizeof("https://") - 1;
	}
	else if (g_ascii_strncasecmp(pos, "file://", sizeof("file://") - 1) == 0) {
		pos += sizeof("file://") - 1;
		/* Exclude file:// */
		bk->uri = g_strdup(pos);
	}
	else if (*pos == '/') {
		/* Trivial file case */
		bk->uri = g_strdup(pos);
	}
	else {
		msg_err_config("invalid map fetching protocol: %s", map_line);

		return NULL;
	}

	if (bk->protocol != MAP_PROTO_FILE && bk->is_signed) {
		msg_err_config("signed maps are no longer supported for HTTP(s): %s", map_line);
	}

	return pos;
}

gboolean
rspamd_map_is_map(const char *map_line)
{
	gboolean ret = FALSE;

	g_assert(map_line != NULL);

	if (map_line[0] == '/') {
		ret = TRUE;
	}
	else if (g_ascii_strncasecmp(map_line, "sign+", sizeof("sign+") - 1) == 0) {
		ret = TRUE;
	}
	else if (g_ascii_strncasecmp(map_line, "fallback+", sizeof("fallback+") - 1) == 0) {
		ret = TRUE;
	}
	else if (g_ascii_strncasecmp(map_line, "file://", sizeof("file://") - 1) == 0) {
		ret = TRUE;
	}
	else if (g_ascii_strncasecmp(map_line, "http://", sizeof("http://") - 1) == 0) {
		ret = TRUE;
	}
	else if (g_ascii_strncasecmp(map_line, "https://", sizeof("https://") - 1) == 0) {
		ret = TRUE;
	}

	return ret;
}

static void
rspamd_map_backend_dtor(struct rspamd_map_backend *bk)
{
	switch (bk->protocol) {
	case MAP_PROTO_FILE:
		if (bk->data.fd) {
			ev_stat_stop(bk->event_loop, &bk->data.fd->st_ev);
			g_free(bk->data.fd->filename);
			g_free(bk->data.fd);
		}
		break;
	case MAP_PROTO_STATIC:
		if (bk->data.sd) {
			if (bk->data.sd->data) {
				g_free(bk->data.sd->data);
			}

			g_free(bk->data.sd);
		}
		break;
	case MAP_PROTO_HTTP:
	case MAP_PROTO_HTTPS:
		if (bk->data.hd) {
			struct http_map_data *data = bk->data.hd;

			g_free(data->host);
			g_free(data->path);
			g_free(data->rest);

			if (data->userinfo) {
				g_free(data->userinfo);
			}

			if (data->etag) {
				rspamd_fstring_free(data->etag);
			}

			/*
			 * Clear cached file, but check if a worker is an active http worker
			 * as cur_cache_cbd is meaningful merely for active worker, who actually
			 * owns the cache
			 */
			if (bk->map && bk->map->active_http) {
				if (g_atomic_int_compare_and_exchange(&data->cache->available, 1, 0)) {
					if (data->cur_cache_cbd) {
						msg_info("clear shared memory cache for a map in %s as backend \"%s\" is closing",
								 data->cur_cache_cbd->shm->shm_name,
								 bk->uri);
						MAP_RELEASE(data->cur_cache_cbd->shm,
									"rspamd_http_map_cached_cbdata");
						ev_timer_stop(data->cur_cache_cbd->event_loop,
									  &data->cur_cache_cbd->timeout);
						g_free(data->cur_cache_cbd);
						data->cur_cache_cbd = NULL;
					}
				}
			}

			g_free(bk->data.hd);
		}
		break;
	}

	if (bk->trusted_pubkey) {
		rspamd_pubkey_unref(bk->trusted_pubkey);
	}

	g_free(bk->uri);
	g_free(bk);
}

static struct rspamd_map_backend *
rspamd_map_parse_backend(struct rspamd_config *cfg, const char *map_line)
{
	struct rspamd_map_backend *bk;
	struct file_map_data *fdata = NULL;
	struct http_map_data *hdata = NULL;
	struct static_map_data *sdata = NULL;
	struct http_parser_url up;
	const char *end;
	rspamd_ftok_t tok;

	bk = g_malloc0(sizeof(*bk));
	REF_INIT_RETAIN(bk, rspamd_map_backend_dtor);

	if (!rspamd_map_check_proto(cfg, map_line, bk)) {
		goto err;
	}

	if (bk->is_fallback && bk->protocol != MAP_PROTO_FILE) {
		msg_err_config("fallback backend must be file for %s", bk->uri);

		goto err;
	}

	end = map_line + strlen(map_line);
	if (end - map_line > 4) {
		/* Support combinations: .enc, .zst, .zstd, .zst.enc, .zstd.enc */
		const char *fname = map_line;
		gsize flen = end - map_line;

		/* Check .enc suffix */
		if (flen >= 4 && g_ascii_strcasecmp(fname + flen - 4, ".enc") == 0) {
			bk->is_encrypted = TRUE;
			flen -= 4;
		}

		if (flen >= 5 && g_ascii_strcasecmp(fname + flen - 5, ".zstd") == 0) {
			bk->is_compressed = TRUE;
		}
		else if (flen >= 4 && g_ascii_strcasecmp(fname + flen - 4, ".zst") == 0) {
			bk->is_compressed = TRUE;
		}
	}

	/* Now check for each proto separately */
	if (bk->protocol == MAP_PROTO_FILE) {
		fdata = g_malloc0(sizeof(struct file_map_data));

		if (access(bk->uri, R_OK) == -1) {
			if (errno != ENOENT) {
				msg_err_config("cannot open file '%s': %s", bk->uri, strerror(errno));
				goto err;
			}

			msg_info_config(
				"map '%s' is not found, but it can be loaded automatically later",
				bk->uri);
		}

		fdata->filename = g_strdup(bk->uri);
		bk->data.fd = fdata;
	}
	else if (bk->protocol == MAP_PROTO_HTTP || bk->protocol == MAP_PROTO_HTTPS) {
		hdata = g_malloc0(sizeof(struct http_map_data));

		memset(&up, 0, sizeof(up));
		if (http_parser_parse_url(bk->uri, strlen(bk->uri), FALSE,
								  &up) != 0) {
			msg_err_config("cannot parse HTTP url: %s", bk->uri);
			goto err;
		}
		else {
			if (!(up.field_set & 1u << UF_HOST)) {
				msg_err_config("cannot parse HTTP url: %s: no host", bk->uri);
				goto err;
			}

			tok.begin = bk->uri + up.field_data[UF_HOST].off;
			tok.len = up.field_data[UF_HOST].len;
			hdata->host = rspamd_ftokdup(&tok);

			if (up.field_set & (1u << UF_PORT)) {
				hdata->port = up.port;
			}
			else {
				if (bk->protocol == MAP_PROTO_HTTP) {
					hdata->port = 80;
				}
				else {
					hdata->port = 443;
				}
			}

			if (up.field_set & (1u << UF_PATH)) {
				tok.begin = bk->uri + up.field_data[UF_PATH].off;
				tok.len = up.field_data[UF_PATH].len;

				hdata->path = rspamd_ftokdup(&tok);

				/* We also need to check query + fragment */
				if (up.field_set & ((1u << UF_QUERY) | (1u << UF_FRAGMENT))) {
					tok.begin = bk->uri + up.field_data[UF_PATH].off +
								up.field_data[UF_PATH].len;
					tok.len = strlen(tok.begin);
					hdata->rest = rspamd_ftokdup(&tok);
				}
				else {
					hdata->rest = g_strdup("");
				}
			}
			else {
				/* No path in URL (e.g. https://host), default to "/" */
				hdata->path = g_strdup("/");
				hdata->rest = g_strdup("");
			}

			if (up.field_set & (1u << UF_USERINFO)) {
				/* Create authorisation header for basic auth */
				unsigned int len = sizeof("Basic ") +
								   up.field_data[UF_USERINFO].len * 8 / 5 + 4;
				hdata->userinfo = g_malloc(len);
				rspamd_snprintf(hdata->userinfo, len, "Basic %*Bs",
								(int) up.field_data[UF_USERINFO].len,
								bk->uri + up.field_data[UF_USERINFO].off);

				msg_debug("added userinfo for the map from the URL: %s", hdata->host);
			}
			else {
				/* Try to obtain authentication data from options in the configuration */
				const ucl_object_t *auth_obj, *opts_obj;

				opts_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "options");
				if (opts_obj != NULL) {
					auth_obj = ucl_object_lookup(opts_obj, "http_auth");
					if (auth_obj != NULL && ucl_object_type(auth_obj) == UCL_OBJECT) {
						const ucl_object_t *host_obj;

						/*
						 * Search first by the full URL and then by the host part
						 */
						host_obj = ucl_object_lookup(auth_obj, map_line);

						if (host_obj == NULL) {
							host_obj = ucl_object_lookup(auth_obj, hdata->host);
						}

						if (host_obj != NULL && ucl_object_type(host_obj) == UCL_OBJECT) {
							const ucl_object_t *user_obj, *password_obj;

							user_obj = ucl_object_lookup(host_obj, "user");
							password_obj = ucl_object_lookup(host_obj, "password");

							if (user_obj != NULL && password_obj != NULL &&
								ucl_object_type(user_obj) == UCL_STRING &&
								ucl_object_type(password_obj) == UCL_STRING) {

								char *tmpbuf;
								unsigned tlen;

								/* User + password + ':' */
								tlen = strlen(ucl_object_tostring(user_obj)) +
									   strlen(ucl_object_tostring(password_obj)) + 1;
								tmpbuf = g_malloc(tlen + 1);
								rspamd_snprintf(tmpbuf, tlen + 1, "%s:%s",
												ucl_object_tostring(user_obj),
												ucl_object_tostring(password_obj));
								/* Base64 encoding is not so greedy, but we add some space for simplicity */
								tlen *= 2;
								tlen += sizeof("Basic ") - 1;
								hdata->userinfo = g_malloc(tlen + 1);
								rspamd_snprintf(hdata->userinfo, tlen + 1, "Basic %Bs", tmpbuf);
								g_free(tmpbuf);
								msg_debug("added userinfo for the map from the configuration: %s", map_line);
							}
						}
					}
				}
			}
		}

		/* Parse optional HTTP timeouts and keepalive tuning from global options -> maps.* block */
		{
			const ucl_object_t *maps_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "maps");
			const ucl_object_t *opt = NULL;
			if (maps_obj && ucl_object_type(maps_obj) == UCL_OBJECT) {
				/* Per-URL overrides: allow stanza keyed by exact URL */
				const ucl_object_t *url_obj = ucl_object_lookup(maps_obj, bk->uri);
				const ucl_object_t *src = url_obj ? url_obj : maps_obj;
				opt = ucl_object_lookup_any(src,
											"connect_timeout", "connect-timeout", NULL);
				if (opt) hdata->connect_timeout = ucl_object_todouble(opt);
				opt = ucl_object_lookup_any(src,
											"ssl_timeout", "ssl-timeout", NULL);
				if (opt) hdata->ssl_timeout = ucl_object_todouble(opt);
				opt = ucl_object_lookup_any(src,
											"write_timeout", "write-timeout", NULL);
				if (opt) hdata->write_timeout = ucl_object_todouble(opt);
				opt = ucl_object_lookup_any(src,
											"read_timeout", "read-timeout", NULL);
				if (opt) hdata->read_timeout = ucl_object_todouble(opt);
				/* Keepalive tuning */
				opt = ucl_object_lookup_any(src,
											"connection_ttl", "connection-ttl", "keepalive_ttl", NULL);
				if (opt) hdata->connection_ttl = ucl_object_todouble(opt);
				opt = ucl_object_lookup_any(src,
											"idle_timeout", "idle-timeout", "keepalive_idle", NULL);
				if (opt) hdata->idle_timeout = ucl_object_todouble(opt);
				opt = ucl_object_lookup_any(src,
											"max_reuse", "max-reuse", "keepalive_max_reuse", NULL);
				if (opt) hdata->max_reuse = (unsigned int) ucl_object_toint(opt);
			}
		}

		hdata->cache = rspamd_mempool_alloc0_shared(cfg->cfg_pool,
													sizeof(*hdata->cache));

		bk->data.hd = hdata;
	}
	else if (bk->protocol == MAP_PROTO_STATIC) {
		sdata = g_malloc0(sizeof(*sdata));
		bk->data.sd = sdata;
	}

	/* Load optional secretbox key from maps { <bk->uri> { secretbox_key = "..." } }
	 * or maps { secretbox_key = ... } either at the root or under options { maps { ... } }
	 */
	if (bk->is_encrypted) {
		const ucl_object_t *maps_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "maps");
		if (maps_obj == NULL || ucl_object_type(maps_obj) != UCL_OBJECT) {
			const ucl_object_t *opts_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "options");
			if (opts_obj && ucl_object_type(opts_obj) == UCL_OBJECT) {
				maps_obj = ucl_object_lookup(opts_obj, "maps");
			}
		}
		const ucl_object_t *src = NULL;
		if (maps_obj && ucl_object_type(maps_obj) == UCL_OBJECT) {
			/* Per-backend */
			src = ucl_object_lookup(maps_obj, bk->uri);
			if (!src) src = maps_obj;
			const ucl_object_t *kobj = ucl_object_lookup_any(src, "secretbox_key", "secretbox-key", "enc_key", NULL);
			if (kobj && ucl_object_type(kobj) == UCL_STRING) {
				gsize klen = 0;
				const char *k = ucl_object_tolstring(kobj, &klen);
				if (rspamd_map_decode_secretbox_key(k, klen, bk->secretbox_key)) {
					bk->has_secretbox_key = TRUE;
					msg_info_config("loaded secretbox key for %s", bk->uri);
				}
				else {
					msg_info_config("cannot decode secretbox key for %s", bk->uri);
				}
			}
			else {
				msg_debug_config("no secretbox_key found in maps block for %s", bk->uri);
			}
		}
		else {
			msg_debug_config("maps block not found at root nor under options; no secretbox_key (uri=%s)", bk->uri);
		}
	}

	/* Fallback: if encrypted but key not loaded (e.g. maps block parsed later), try again */
	if (bk->is_encrypted && !bk->has_secretbox_key) {
		rspamd_map_try_load_secretbox_key(cfg, bk);
	}

	return bk;

err:
	MAP_RELEASE(bk, "rspamd_map_backend");

	if (hdata) {
		g_free(hdata);
	}

	if (fdata) {
		g_free(fdata);
	}

	if (sdata) {
		g_free(sdata);
	}

	return NULL;
}

static void
rspamd_map_calculate_hash(struct rspamd_map *map)
{
	struct rspamd_map_backend *bk;
	unsigned int i;
	rspamd_cryptobox_hash_state_t st;
	char *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES];

	rspamd_cryptobox_hash_init(&st, NULL, 0);

	if (map->name) {
		rspamd_cryptobox_hash_update(&st, map->name, strlen(map->name));
	}
	if (map->description) {
		rspamd_cryptobox_hash_update(&st, map->description, strlen(map->description));
	}

	for (i = 0; i < map->backends->len; i++) {
		bk = g_ptr_array_index(map->backends, i);
		rspamd_cryptobox_hash_update(&st, bk->uri, strlen(bk->uri));
	}

	rspamd_cryptobox_hash_final(&st, cksum);
	cksum_encoded = rspamd_encode_base32(cksum, sizeof(cksum), RSPAMD_BASE32_DEFAULT);
	rspamd_strlcpy(map->tag, cksum_encoded, sizeof(map->tag));

	for (i = 0; i < map->backends->len; i++) {
		bk = g_ptr_array_index(map->backends, i);

		/* Also update each backend */
		rspamd_cryptobox_fast_hash_state_t hst;
		rspamd_cryptobox_fast_hash_init(&hst, 0);
		rspamd_cryptobox_fast_hash_update(&hst, bk->uri, strlen(bk->uri));
		rspamd_cryptobox_fast_hash_update(&hst, map->tag, sizeof(map->tag));

		if (bk->protocol == MAP_PROTO_STATIC) {
			/* Static maps content is pre-defined */
			rspamd_cryptobox_fast_hash_update(&hst, bk->data.sd->data,
											  bk->data.sd->len);
		}

		/* We use only 52 bits to be compatible with other numbers representation */
		bk->id = rspamd_cryptobox_fast_hash_final(&hst) & ~(0xFFFULL << 52);
	}

	g_free(cksum_encoded);
}

static gboolean
rspamd_map_add_static_string(struct rspamd_config *cfg,
							 const ucl_object_t *elt,
							 GString *target)
{
	gsize sz;
	const char *dline;

	if (ucl_object_type(elt) != UCL_STRING) {
		msg_err_config("map has static backend but `data` is "
					   "not string like: %s",
					   ucl_object_type_to_string(elt->type));
		return FALSE;
	}

	/* Otherwise, we copy data to the backend */
	dline = ucl_object_tolstring(elt, &sz);

	if (sz == 0) {
		msg_err_config("map has static backend but empty no data");
		return FALSE;
	}

	g_string_append_len(target, dline, sz);
	g_string_append_c(target, '\n');

	return TRUE;
}

struct rspamd_map *
rspamd_map_add(struct rspamd_config *cfg,
			   const char *map_line,
			   const char *description,
			   map_cb_t read_callback,
			   map_fin_cb_t fin_callback,
			   map_dtor_t dtor,
			   void **user_data,
			   struct rspamd_worker *worker,
			   int flags)
{
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;

	bk = rspamd_map_parse_backend(cfg, map_line);
	if (bk == NULL) {
		return NULL;
	}

	if (bk->is_fallback) {
		msg_err_config("cannot add map with fallback only backend: %s", bk->uri);
		REF_RELEASE(bk);

		return NULL;
	}

	map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map));
	map->read_callback = read_callback;
	map->fin_callback = fin_callback;
	map->dtor = dtor;
	map->user_data = user_data;
	map->cfg = cfg;
	map->id = rspamd_random_uint64_fast();
	map->shared =
		rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(struct rspamd_map_shared_data));
	map->backends = g_ptr_array_sized_new(1);
	map->wrk = worker;
	rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard,
								  map->backends);
	g_ptr_array_add(map->backends, bk);
	map->name = rspamd_mempool_strdup(cfg->cfg_pool, map_line);
	map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ);

	if (bk->protocol == MAP_PROTO_FILE) {
		map->poll_timeout = (cfg->map_timeout * cfg->map_file_watch_multiplier);
	}
	else {
		map->poll_timeout = cfg->map_timeout;
	}

	if (description != NULL) {
		map->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
	}

	rspamd_map_calculate_hash(map);
	msg_info_map("added map %s", bk->uri);
	bk->map = map;

	cfg->maps = g_list_prepend(cfg->maps, map);

	return map;
}

struct rspamd_map *
rspamd_map_add_fake(struct rspamd_config *cfg,
					const char *description,
					const char *name)
{
	struct rspamd_map *map;

	map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map));
	map->cfg = cfg;
	map->id = rspamd_random_uint64_fast();
	map->name = rspamd_mempool_strdup(cfg->cfg_pool, name);
	map->user_data = (void **) &map; /* to prevent null pointer dereferencing */

	if (description != NULL) {
		map->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
	}

	return map;
}

static inline void
rspamd_map_add_backend(struct rspamd_map *map, struct rspamd_map_backend *bk)
{
	if (bk->is_fallback) {
		if (map->fallback_backend) {
			msg_warn_map("redefining fallback backend from %s to %s",
						 map->fallback_backend->uri, bk->uri);
		}

		map->fallback_backend = bk;
	}
	else {
		g_ptr_array_add(map->backends, bk);
	}

	bk->map = map;
}

struct rspamd_map *
rspamd_map_add_from_ucl(struct rspamd_config *cfg,
						const ucl_object_t *obj,
						const char *description,
						map_cb_t read_callback,
						map_fin_cb_t fin_callback,
						map_dtor_t dtor,
						void **user_data,
						struct rspamd_worker *worker,
						int flags)
{
	ucl_object_iter_t it = NULL;
	const ucl_object_t *cur, *elt;
	struct rspamd_map *map;
	struct rspamd_map_backend *bk;
	unsigned int i;

	g_assert(obj != NULL);

	if (ucl_object_type(obj) == UCL_STRING) {
		/* Just a plain string */
		return rspamd_map_add(cfg, ucl_object_tostring(obj), description,
							  read_callback, fin_callback, dtor, user_data, worker, flags);
	}

	map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map));
	map->read_callback = read_callback;
	map->fin_callback = fin_callback;
	map->dtor = dtor;
	map->user_data = user_data;
	map->cfg = cfg;
	map->id = rspamd_random_uint64_fast();
	map->shared =
		rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(struct rspamd_map_shared_data));
	map->backends = g_ptr_array_new();
	map->wrk = worker;
	map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ);
	rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard,
								  map->backends);
	map->poll_timeout = cfg->map_timeout;

	if (description) {
		map->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
	}

	if (ucl_object_type(obj) == UCL_ARRAY) {
		/* Add array of maps as multiple backends */
		while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) {
			if (ucl_object_type(cur) == UCL_STRING) {
				bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(cur));

				if (bk != NULL) {
					rspamd_map_add_backend(map, bk);

					if (!map->name) {
						map->name = rspamd_mempool_strdup(cfg->cfg_pool,
														  ucl_object_tostring(cur));
					}
				}
			}
			else {
				msg_err_config("bad map element type: %s",
							   ucl_object_type_to_string(ucl_object_type(cur)));
			}
		}

		if (map->backends->len == 0) {
			msg_err_config("map has no urls to be loaded: empty list");
			goto err;
		}
	}
	else if (ucl_object_type(obj) == UCL_OBJECT) {
		elt = ucl_object_lookup(obj, "name");
		if (elt && ucl_object_type(elt) == UCL_STRING) {
			map->name = rspamd_mempool_strdup(cfg->cfg_pool,
											  ucl_object_tostring(elt));
		}

		elt = ucl_object_lookup(obj, "description");
		if (elt && ucl_object_type(elt) == UCL_STRING) {
			map->description = rspamd_mempool_strdup(cfg->cfg_pool,
													 ucl_object_tostring(elt));
		}

		elt = ucl_object_lookup_any(obj, "timeout", "poll", "poll_time",
									"watch_interval", NULL);
		if (elt) {
			map->poll_timeout = ucl_object_todouble(elt);
		}

		elt = ucl_object_lookup_any(obj, "upstreams", "url", "urls", NULL);
		if (elt == NULL) {
			msg_err_config("map has no urls to be loaded: no elt");
			goto err;
		}

		if (ucl_object_type(elt) == UCL_ARRAY) {
			/* Add array of maps as multiple backends */
			it = ucl_object_iterate_new(elt);

			while ((cur = ucl_object_iterate_safe(it, true)) != NULL) {
				if (ucl_object_type(cur) == UCL_STRING) {
					bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(cur));

					if (bk != NULL) {
						rspamd_map_add_backend(map, bk);

						if (!map->name) {
							map->name = rspamd_mempool_strdup(cfg->cfg_pool,
															  ucl_object_tostring(cur));
						}
					}
				}
				else {
					msg_err_config("bad map element type: %s",
								   ucl_object_type_to_string(ucl_object_type(cur)));
					ucl_object_iterate_free(it);
					goto err;
				}
			}

			ucl_object_iterate_free(it);

			if (map->backends->len == 0) {
				msg_err_config("map has no urls to be loaded: empty object list");
				goto err;
			}
		}
		else if (ucl_object_type(elt) == UCL_STRING) {
			bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(elt));

			if (bk != NULL) {
				rspamd_map_add_backend(map, bk);

				if (!map->name) {
					map->name = rspamd_mempool_strdup(cfg->cfg_pool,
													  ucl_object_tostring(elt));
				}
			}
		}

		if (!map->backends || map->backends->len == 0) {
			msg_err_config("map has no urls to be loaded: no valid backends");
			goto err;
		}
	}
	else {
		msg_err_config("map has invalid type for value: %s",
					   ucl_object_type_to_string(ucl_object_type(obj)));
		goto err;
	}

	gboolean all_local = TRUE, all_loaded = TRUE;

	PTR_ARRAY_FOREACH(map->backends, i, bk)
	{
		if (bk->protocol == MAP_PROTO_STATIC) {
			GString *map_data;
			/* We need data field in ucl */
			elt = ucl_object_lookup(obj, "data");

			if (elt == NULL) {
				msg_err_config("map has static backend but no `data` field");
				goto err;
			}


			if (ucl_object_type(elt) == UCL_STRING) {
				map_data = g_string_sized_new(32);

				if (rspamd_map_add_static_string(cfg, elt, map_data)) {
					bk->data.sd->len = map_data->len;
					bk->data.sd->data = (unsigned char *) g_string_free(map_data, FALSE);
				}
				else {
					g_string_free(map_data, TRUE);
					msg_err_config("map has static backend with invalid `data` field");
					goto err;
				}
			}
			else if (ucl_object_type(elt) == UCL_ARRAY) {
				map_data = g_string_sized_new(32);
				it = ucl_object_iterate_new(elt);

				while ((cur = ucl_object_iterate_safe(it, true))) {
					if (!rspamd_map_add_static_string(cfg, cur, map_data)) {
						g_string_free(map_data, TRUE);
						msg_err_config("map has static backend with invalid "
									   "`data` field");
						ucl_object_iterate_free(it);
						goto err;
					}
				}

				ucl_object_iterate_free(it);
				bk->data.sd->len = map_data->len;
				bk->data.sd->data = (unsigned char *) g_string_free(map_data, FALSE);
			}
		}
		else if (bk->protocol != MAP_PROTO_FILE) {
			all_local = FALSE;
			all_loaded = FALSE; /* Will be loaded later */
		}
		else {
			all_loaded = FALSE; /* Will be loaded later (even for files) */
		}
	}

	if (all_local) {
		map->poll_timeout = (map->poll_timeout *
							 cfg->map_file_watch_multiplier);
	}

	if (all_loaded) {
		/* Static map */
		g_atomic_int_set(&map->shared->loaded, 1);
	}

	rspamd_map_calculate_hash(map);
	msg_debug_map("added map from ucl");

	cfg->maps = g_list_prepend(cfg->maps, map);

	return map;

err:

	if (map) {
		PTR_ARRAY_FOREACH(map->backends, i, bk)
		{
			MAP_RELEASE(bk, "rspamd_map_backend");
		}
	}

	return NULL;
}

rspamd_map_traverse_function
rspamd_map_get_traverse_function(struct rspamd_map *map)
{
	if (map) {
		return map->traverse_function;
	}

	return NULL;
}

void rspamd_map_traverse(struct rspamd_map *map, rspamd_map_traverse_cb cb,
						 gpointer cbdata, gboolean reset_hits)
{
	if (*map->user_data && map->traverse_function) {
		map->traverse_function(*map->user_data, cb, cbdata, reset_hits);
	}
}

void rspamd_map_set_on_load_function(struct rspamd_map *map, rspamd_map_on_load_function cb,
									 gpointer cbdata, GDestroyNotify dtor)
{
	if (map) {
		map->on_load_function = cb;
		map->on_load_ud = cbdata;
		map->on_load_ud_dtor = dtor;
	}
}

void rspamd_map_trigger_hyperscan_compilation(struct rspamd_map *map)
{
	/* Only trigger compilation in controller worker */
	if (!map->cfg || !map->cfg->cur_worker) {
		return;
	}

	struct rspamd_worker *worker = map->wrk;
	if (!rspamd_worker_is_primary_controller(worker)) {
		return;
	}

	/* Check if we have any scopes that need compilation */
	if (!map->cfg->re_cache) {
		return;
	}

	unsigned int scope_count = rspamd_re_cache_count_scopes(map->cfg->re_cache);
	if (scope_count == 0) {
		return;
	}

	/* Iterate through scopes and compile those that are loaded */
	struct rspamd_re_cache *scope;

	for (scope = rspamd_re_cache_scope_first(map->cfg->re_cache);
		 scope != NULL;
		 scope = rspamd_re_cache_scope_next(scope)) {
		const char *scope_name = rspamd_re_cache_scope_name(scope);
		const char *scope_for_check = (strcmp(scope_name, "default") == 0) ? NULL : scope_name;

		/* Only compile loaded scopes */
		if (rspamd_re_cache_is_loaded(map->cfg->re_cache, scope_for_check)) {
			msg_info_map("triggering hyperscan compilation for scope: %s after map update",
						 scope_name);

			/* Use default settings for compilation */
			rspamd_re_cache_compile_hyperscan_scoped_single(scope, scope_for_check,
															map->cfg->hs_cache_dir ? map->cfg->hs_cache_dir : RSPAMD_DBDIR "/",
															1.0,   /* max_time */
															FALSE, /* silent */
															worker->ctx ? ((struct rspamd_abstract_worker_ctx *) worker->ctx)->event_loop : NULL,
															worker,
															NULL,  /* callback */
															NULL); /* cbdata */
		}
	}
}

gsize rspamd_map_get_no_file_read_offset(struct rspamd_map *map)
{
	if (map) {
		return map->no_file_read_offset;
	}

	return 0;
}
