nfqws: ip->hostname cache

docs/changes.txt

@@ -488,5 +488,5 @@ nfqws,tpws: debug tls version, alpn, ech
 nfqws: --dpi-desync-fake-tls=! means default tls fake
 nfqws: --dup*
 nfqws: --orig*
-nfqws: autottl cache
+nfqws: ipcache of hop count and host names
 init.d: remove --ipset parameter prohibition

nfq/conntrack.h

@@ -86,6 +86,7 @@ typedef struct
 	t_l7proto l7proto;
 	bool l7proto_discovered;
 	char *hostname;
+	bool hostname_discovered;
 	bool hostname_ah_check;			// should perform autohostlist checks
 	
 	t_reassemble reasm_orig;

nfq/desync.c

@@ -353,7 +353,7 @@ static void wssize_cutoff(t_ctrack *ctrack)
 }
 static void forced_wssize_cutoff(t_ctrack *ctrack)
 {
- 	if (ctrack && ctrack->dp && ctrack->dp->wssize && !ctrack->b_wssize_cutoff)
+	if (ctrack && ctrack->dp && !ctrack->b_wssize_cutoff)
 	{
 		DLOG("forced wssize-cutoff\n");
 		wssize_cutoff(ctrack);
@@ -799,6 +799,49 @@ static void autottl_rediscover(t_ctrack *ctrack, const struct in_addr *a4, const
 	}
 }
 
+static bool ipcache_put_hostname(const struct in_addr *a4, const struct in6_addr *a6, const char *iface, const char *hostname)
+{
+	if (!params.cache_hostnames) return true;
+
+	ip_cache_item *ipc = ipcacheTouch(&params.ipcache,a4,a6,iface);
+	if (!ipc)
+	{
+		DLOG_ERR("ipcache_put_hostname: out of memory\n");
+		return false;
+	}
+	free(ipc->hostname);
+	if (!(ipc->hostname = strdup(hostname)))
+	{
+		DLOG_ERR("ipcache_put_hostname: out of memory\n");
+		return false;
+	}
+	DLOG("hostname cached: %s\n", hostname);
+	return true;
+}
+static bool ipcache_get_hostname(const struct in_addr *a4, const struct in6_addr *a6, const char *iface, char *hostname, size_t hostname_buf_len)
+{
+	if (!params.cache_hostnames)
+	{
+		*hostname = 0;
+		return true;
+	}
+	ip_cache_item *ipc = ipcacheTouch(&params.ipcache,a4,a6,iface);
+	if (!ipc)
+	{
+		DLOG_ERR("ipcache_get_hostname: out of memory\n");
+		return false;
+	}
+	if (ipc->hostname)
+	{
+		DLOG("got cached hostname: %s\n", ipc->hostname);
+		snprintf(hostname,hostname_buf_len,"%s",ipc->hostname);
+	}
+	else
+		*hostname = 0;
+	return true;
+}
+
+
 #ifdef BSD
 // BSD pass to divert socket ip_id=0 and does not auto set it if sent via divert socket
 static uint16_t IP4_IP_ID_FIX(const struct ip *ip)
@@ -1081,6 +1124,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 	uint32_t *timestamps;
 	bool bSack,DF;
 	uint16_t nmss;
+	char host[256];
 
 	uint32_t desync_fwmark = fwmark | params.desync_fwmark;
 	extract_endpoints(dis->ip, dis->ip6, dis->tcp, NULL, &src, &dst);
@@ -1098,6 +1142,12 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack_replay->dp_search_complete)
 		{
+			if (!ctrack_replay->hostname && !bReverse)
+			{
+				if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)
+					if (!(ctrack_replay->hostname = strdup(host)))
+						DLOG_ERR("strdup(host): out of memory\n");
+			}
 			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, ctrack_replay->hostname, ctrack_replay->l7proto, NULL, NULL, NULL);
 			ctrack_replay->dp_search_complete = true;
 		}
@@ -1121,7 +1171,18 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack || !ctrack->dp_search_complete)
 		{
-			dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, ctrack ? ctrack->hostname : NULL, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);
+			const char *hostname = NULL;
+			if (ctrack)
+			{
+				hostname = ctrack->hostname;
+				if (!hostname && !bReverse)
+				{
+					if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)
+						if (!(hostname = ctrack_replay->hostname = strdup(host)))
+							DLOG_ERR("strdup(host): out of memory\n");
+				}
+			}
+			dp = dp_find(&params.desync_profiles, IPPROTO_TCP, (struct sockaddr *)&dst, hostname, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);
 			if (ctrack)
 			{
 				ctrack->dp = dp;
@@ -1305,8 +1366,6 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 	{
 		struct blob_collection_head *fake;
 
-		char host[256];
-		bool bHaveHost=false;
 		uint8_t *p, *phost=NULL;
 		const uint8_t *rdata_payload = dis->data_payload;
 		size_t rlen_payload = dis->len_payload;
@@ -1315,6 +1374,7 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 		int multisplit_count;
 		int i;
 		uint16_t ip_id;
+		bool bHaveHost=false;
 		t_l7proto l7proto = UNKNOWN;
 
 		if (replay)
@@ -1447,12 +1507,14 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 			bDiscoveredL7 = !ctrack_replay && l7proto!=UNKNOWN;
 		if (bDiscoveredL7) DLOG("discovered l7 protocol\n");
 
-		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname);
+		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname_discovered);
 		if (bDiscoveredHostname)
 		{
 			DLOG("discovered hostname\n");
 			if (ctrack_replay)
 			{
+				ctrack_replay->hostname_discovered=true;
+				free(ctrack_replay->hostname);
 				ctrack_replay->hostname=strdup(host);
 				if (!ctrack_replay->hostname)
 				{
@@ -1460,6 +1522,12 @@ static uint8_t dpi_desync_tcp_packet_play(bool replay, size_t reasm_offset, uint
 					reasm_orig_cancel(ctrack);
 					goto send_orig;
 				}
+				if (!ipcache_put_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host))
+				{
+					reasm_orig_cancel(ctrack);
+					goto send_orig;
+				}
+
 			}
 		}
 
@@ -2262,6 +2330,7 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 	size_t pkt1_len, pkt2_len;
 	uint8_t ttl_orig,ttl_fake;
 	bool DF;
+	char host[256];
 	t_l7proto l7proto = UNKNOWN;
 
 	extract_endpoints(dis->ip, dis->ip6, NULL, dis->udp, &src, &dst);
@@ -2279,6 +2348,12 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack_replay->dp_search_complete)
 		{
+			if (!ctrack_replay->hostname && !bReverse)
+			{
+				if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)
+					if (!(ctrack_replay->hostname = strdup(host)))
+						DLOG_ERR("strdup(host): out of memory\n");
+			}
 			dp = ctrack_replay->dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, ctrack_replay->hostname, ctrack_replay->l7proto, NULL, NULL, NULL);
 			ctrack_replay->dp_search_complete = true;
 		}
@@ -2305,7 +2380,18 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			DLOG("using cached desync profile %d\n",dp->n);
 		else if (!ctrack || !ctrack->dp_search_complete)
 		{
-			dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, ctrack ? ctrack->hostname : NULL, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);
+			const char *hostname = NULL;
+			if (ctrack)
+			{
+				hostname = ctrack->hostname;
+				if (!hostname && !bReverse)
+				{
+					if (ipcache_get_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host, sizeof(host)) && *host)
+						if (!(hostname = ctrack_replay->hostname = strdup(host)))
+							DLOG_ERR("strdup(host): out of memory\n");
+				}
+			}
+			dp = dp_find(&params.desync_profiles, IPPROTO_UDP, (struct sockaddr *)&dst, hostname, ctrack ? ctrack->l7proto : UNKNOWN, NULL, NULL, NULL);
 			if (ctrack)
 			{
 				ctrack->dp = dp;
@@ -2353,7 +2439,6 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 	if (dis->len_payload)
 	{
 		struct blob_collection_head *fake;
-		char host[256];
 		bool bHaveHost=false;
 		uint16_t ip_id;
 
@@ -2553,18 +2638,22 @@ static uint8_t dpi_desync_udp_packet_play(bool replay, size_t reasm_offset, uint
 			bDiscoveredL7 = !ctrack_replay && l7proto!=UNKNOWN;
 		if (bDiscoveredL7) DLOG("discovered l7 protocol\n");
 
-		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname);
+		bool bDiscoveredHostname = bHaveHost && !(ctrack_replay && ctrack_replay->hostname_discovered);
 		if (bDiscoveredHostname)
 		{
 			DLOG("discovered hostname\n");
 			if (ctrack_replay)
 			{
+				ctrack_replay->hostname_discovered=true;
+				free(ctrack_replay->hostname);
 				ctrack_replay->hostname=strdup(host);
 				if (!ctrack_replay->hostname)
 				{
 					DLOG_ERR("hostname dup : out of memory");
 					goto send_orig;
 				}
+				if (!ipcache_put_hostname(dis->ip ? &dis->ip->ip_dst : NULL,dis->ip6 ? &dis->ip6->ip6_dst : NULL , ifout, host))
+					goto send_orig;
 			}
 		}
 
@@ -2907,7 +2996,7 @@ static uint8_t dpi_desync_packet_play(bool replay, size_t reasm_offset, uint32_t
 }
 uint8_t dpi_desync_packet(uint32_t fwmark, const char *ifin, const char *ifout, uint8_t *data_pkt, size_t *len_pkt)
 {
-	ipcachePurgeRateLimited(&params.ipcache, params.autottl_cache_lifetime);
+	ipcachePurgeRateLimited(&params.ipcache, params.ipcache_lifetime);
 	return dpi_desync_packet_play(false, 0, fwmark, ifin, ifout, data_pkt, len_pkt);
 }
 

nfq/nfqws.c

@@ -99,9 +99,9 @@ static void onusr2(int sig)
 		printf("\nDESYNC PROFILE %d\n",dpl->dp.n);
 		HostFailPoolDump(dpl->dp.hostlist_auto_fail_counters);
 	}
-	if (params.autottl_present)
+	if (params.autottl_present || params.cache_hostnames)
 	{
-		printf("\nAUTOTTL IP CACHE\n");
+		printf("\nIPCACHE\n");
 		ipcachePrint(&params.ipcache);
 	}
 	printf("\n");
@@ -1417,7 +1417,8 @@ static void exithelp(void)
 		" --bind-fix6\t\t\t\t\t; apply outgoing interface selection fix for generated ipv6 packets\n"
 #endif
 		" --ctrack-timeouts=S:E:F[:U]\t\t\t; internal conntrack timeouts for TCP SYN, ESTABLISHED, FIN stages, UDP timeout. default %u:%u:%u:%u\n"
-		" --autottl-cache-lifetime=<int>\t\t\t; time in seconds to keep cached hop count (default %u)\n"
+		" --ipcache-lifetime=<int>\t\t\t; time in seconds to keep cached hop count and domain name (default %u)\n"
+		" --ipcache-hostnames=[0|1]\t\t\t; 1 or no argument enables ip->hostname caching\n"
 #ifdef __CYGWIN__
 		"\nWINDIVERT FILTER:\n"
 		" --wf-iface=<int>[.<int>]\t\t\t; numeric network interface and subinterface indexes\n"
@@ -1521,7 +1522,7 @@ static void exithelp(void)
 		" --dpi-desync-start=[n|d|s]N\t\t\t; apply dpi desync only to packet numbers (n, default), data packet numbers (d), relative sequence (s) greater or equal than N\n"
 		" --dpi-desync-cutoff=[n|d|s]N\t\t\t; apply dpi desync only to packet numbers (n, default), data packet numbers (d), relative sequence (s) less than N\n",
 		CTRACK_T_SYN, CTRACK_T_EST, CTRACK_T_FIN, CTRACK_T_UDP,
-		AUTOTTL_CACHE_LIFETIME,
+		IPCACHE_LIFETIME,
 		HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT, HOSTLIST_AUTO_FAIL_TIME_DEFAULT, HOSTLIST_AUTO_RETRANS_THRESHOLD_DEFAULT,
 		AUTOTTL_DEFAULT_ORIG_DELTA,AUTOTTL_DEFAULT_ORIG_MIN,AUTOTTL_DEFAULT_ORIG_MAX,
 		AUTOTTL_DEFAULT_DUP_DELTA,AUTOTTL_DEFAULT_DUP_MIN,AUTOTTL_DEFAULT_DUP_MAX,
@@ -1619,7 +1620,8 @@ enum opt_indices {
 	IDX_WSSIZE,
 	IDX_WSSIZE_CUTOFF,
 	IDX_CTRACK_TIMEOUTS,
-	IDX_AUTOTTL_CACHE_LIFETIME,
+	IDX_IPCACHE_LIFETIME,
+	IDX_IPCACHE_HOSTNAMES,
 	IDX_HOSTCASE,
 	IDX_HOSTSPELL,
 	IDX_HOSTNOSPACE,
@@ -1737,7 +1739,8 @@ static const struct option long_options[] = {
 	[IDX_WSSIZE] = {"wssize", required_argument, 0, 0},
 	[IDX_WSSIZE_CUTOFF] = {"wssize-cutoff", required_argument, 0, 0},
 	[IDX_CTRACK_TIMEOUTS] = {"ctrack-timeouts", required_argument, 0, 0},
-	[IDX_AUTOTTL_CACHE_LIFETIME] = {"autottl-cache-lifetime", required_argument, 0, 0},
+	[IDX_IPCACHE_LIFETIME] = {"ipcache-lifetime", required_argument, 0, 0},
+	[IDX_IPCACHE_HOSTNAMES] = {"ipcache-hostnames", optional_argument, 0, 0},
 	[IDX_HOSTCASE] = {"hostcase", no_argument, 0, 0},
 	[IDX_HOSTSPELL] = {"hostspell", required_argument, 0, 0},
 	[IDX_HOSTNOSPACE] = {"hostnospace", no_argument, 0, 0},
@@ -1891,7 +1894,7 @@ int main(int argc, char **argv)
 	params.ctrack_t_est = CTRACK_T_EST;
 	params.ctrack_t_fin = CTRACK_T_FIN;
 	params.ctrack_t_udp = CTRACK_T_UDP;
-	params.autottl_cache_lifetime = AUTOTTL_CACHE_LIFETIME;
+	params.ipcache_lifetime = IPCACHE_LIFETIME;
 
 	LIST_INIT(&params.hostlists);
 	LIST_INIT(&params.ipsets);
@@ -2045,13 +2048,16 @@ int main(int argc, char **argv)
 				exit_clean(1);
 			}
 			break;
-		case IDX_AUTOTTL_CACHE_LIFETIME:
+		case IDX_IPCACHE_LIFETIME:
-			if (sscanf(optarg, "%u", &params.autottl_cache_lifetime)!=1)
+			if (sscanf(optarg, "%u", &params.ipcache_lifetime)!=1)
 			{
-				DLOG_ERR("invalid autottl-cache-lifetime value\n");
+				DLOG_ERR("invalid ipcache-lifetime value\n");
 				exit_clean(1);
 			}
 			break;
+		case IDX_IPCACHE_HOSTNAMES:
+			params.cache_hostnames = !optarg || !!atoi(optarg);
+			break;
 		case IDX_HOSTCASE:
 			dp->hostcase = true;
 			break;
@@ -2958,7 +2964,7 @@ int main(int argc, char **argv)
 	}
 
 	DLOG("initializing conntrack with timeouts tcp=%u:%u:%u udp=%u\n", params.ctrack_t_syn, params.ctrack_t_est, params.ctrack_t_fin, params.ctrack_t_udp);
-	if (params.autottl_present) DLOG("autottl cache lifetime %us\n", params.autottl_cache_lifetime);
+	if (params.autottl_present || params.cache_hostnames) DLOG("ipcache lifetime %us\n", params.ipcache_lifetime);
 	ConntrackPoolInit(&params.conntrack, 10, params.ctrack_t_syn, params.ctrack_t_est, params.ctrack_t_fin, params.ctrack_t_udp);
 
 #ifdef __linux__

nfq/params.h

@@ -36,7 +36,8 @@
 #define	HOSTLIST_AUTO_FAIL_TIME_DEFAULT 	60
 #define	HOSTLIST_AUTO_RETRANS_THRESHOLD_DEFAULT	3
 
-#define AUTOTTL_CACHE_LIFETIME		86400
+#define IPCACHE_LIFETIME		7200
+
 #define AUTOTTL_DEFAULT_DESYNC_DELTA	-1
 #define AUTOTTL_DEFAULT_DESYNC_MIN	3
 #define AUTOTTL_DEFAULT_DESYNC_MAX	20
@@ -197,8 +198,8 @@ struct params_s
 	unsigned int ctrack_t_syn, ctrack_t_est, ctrack_t_fin, ctrack_t_udp;
 	t_conntrack conntrack;
 
-	unsigned int autottl_cache_lifetime;
+	unsigned int ipcache_lifetime;
-	bool autottl_present;
+	bool autottl_present,cache_hostnames;
 	ip_cache ipcache;
 };
 

nfq/pools.c

@@ -589,8 +589,13 @@ static void ipcache_item_touch(ip_cache_item *item)
 static void ipcache_item_init(ip_cache_item *item)
 {
 	ipcache_item_touch(item);
+	item->hostname = NULL;
 	item->hops = 0;
 }
+static void ipcache_item_destroy(ip_cache_item *item)
+{
+	free(item->hostname);
+}
 
 static void ipcache4Destroy(ip_cache4 **ipcache)
 {
@@ -598,6 +603,7 @@ static void ipcache4Destroy(ip_cache4 **ipcache)
 	HASH_ITER(hh, *ipcache, elem, tmp)
 	{
 		HASH_DEL(*ipcache, elem);
+		ipcache_item_destroy(&elem->data);
 		free(elem);
 	}
 }
@@ -645,7 +651,7 @@ static void ipcache4Print(ip_cache4 *ipcache)
 	{
 		*s_ip=0;
 		inet_ntop(AF_INET, &ipc->key.addr, s_ip, sizeof(s_ip));
-		printf("%s iface=%s : hops %u now=last+%llu\n", s_ip, ipc->key.iface, ipc->data.hops, (unsigned long long)(now-ipc->data.last));
+		printf("%s iface=%s : hops %u hostname=%s now=last+%llu\n", s_ip, ipc->key.iface, ipc->data.hops, ipc->data.hostname ? ipc->data.hostname : "", (unsigned long long)(now-ipc->data.last));
 	}
 }
 
@@ -655,6 +661,7 @@ static void ipcache6Destroy(ip_cache6 **ipcache)
 	HASH_ITER(hh, *ipcache, elem, tmp)
 	{
 		HASH_DEL(*ipcache, elem);
+		ipcache_item_destroy(&elem->data);
 		free(elem);
 	}
 }
@@ -702,7 +709,7 @@ static void ipcache6Print(ip_cache6 *ipcache)
 	{
 		*s_ip=0;
 		inet_ntop(AF_INET6, &ipc->key.addr, s_ip, sizeof(s_ip));
-		printf("%s iface=%s : hops %u now=last+%llu\n", s_ip, ipc->key.iface, ipc->data.hops, (unsigned long long)(now-ipc->data.last));
+		printf("%s iface=%s : hops %u hostname=%s now=last+%llu\n", s_ip, ipc->key.iface, ipc->data.hops, ipc->data.hostname ? ipc->data.hostname : "", (unsigned long long)(now-ipc->data.last));
 	}
 }
 

nfq/pools.h

@@ -177,6 +177,7 @@ typedef struct ip6if
 typedef struct ip_cache_item
 {
 	time_t last;
+	char *hostname;
 	uint8_t hops;
 } ip_cache_item;
 typedef struct ip_cache4