tpws,nfqws: autohostlist reread on modify

docs/readme.eng.md

@@ -796,6 +796,10 @@ may start to break the website. This situation can only be controlled manually.
 Remove undesired domain from the autohostlist file, restart nfqws/tpws or send them SIGHUP.
 Use exclude hostlist to prevent further auto additions.
 
+It's possible to use one auto hostlist with multiple processes. All processes check for file modification time.
+If a process modified autohostlist, all others will reread it automatically.
+All processes must run with the same uid.
+
 If zapret scripts are used then autohostlist is `ipset/zapret-hosts-auto.txt`
 and exlude list is `ipset/zapret-hosts-user-exclude.txt`. autohostlist mode
 includes hostlist mode. You can use `ipset/zapret-hosts-user.txt`.

docs/readme.txt

@@ -1041,6 +1041,10 @@ nfqws и tpws могут сечь варианты 1-3, 4 они не распо
 В лог заносятся только основные события, ведущие к занесению хоста в лист.
 По логу можно понять как избежать ложных срабатываний и подходит ли вообще вам этот режим.
 
+Возможно использование одного auto листа с множеством процессов. Все процессы проверяют
+время изменения файла. Если другой процесс туда что-то записал, то остальные перечитают все include листы.
+Все процессы должны работать под одним uid, чтобы избежать проблем с доступом к файлу.
+
 Скрипты zapret ведут autohostlist в ipset/zapret-hosts-auto.txt.
 install_easy.sh при апгрейде zapret сохраняет этот файл.
 Режим autohostlist включает в себя режим hostlist.

nfq/desync.c

@@ -235,7 +235,7 @@ static void auto_hostlist_failed(const char *hostname)
 		
 		DLOG("auto hostlist : rechecking %s to avoid duplicates\n", hostname);
 		bool bExcluded=false;
-		if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded)
+		if (!HostlistCheck(hostname, &bExcluded) && !bExcluded)
 		{
 			DLOG("auto hostlist : adding %s\n", hostname);
 			HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname);
@@ -249,6 +249,7 @@ static void auto_hostlist_failed(const char *hostname)
 				perror("write to auto hostlist:");
 				return;
 			}
+			params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename);
 		}
 		else
 		{
@@ -615,7 +616,7 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 		{
 			bool bExcluded;
 			DLOG("hostname: %s\n",host)
-			if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded))
+			if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(host, &bExcluded))
 			{
 				DLOG("not applying tampering to this request\n")
 				if (ctrack)
@@ -1074,7 +1075,7 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout,
 		{
 			DLOG("hostname: %s\n",host)
 			bool bExcluded;
-			if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded))
+			if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(host, &bExcluded))
 			{
 				DLOG("not applying tampering to this request\n")
 				if (!bExcluded && *params.hostlist_auto_filename && ctrack)

nfq/helpers.c

@@ -6,6 +6,7 @@
 #include <unistd.h>
 #include <ctype.h>
 #include <time.h>
+#include <sys/stat.h>
 
 void hexdump_limited_dlog(const uint8_t *data, size_t size, size_t limit)
 {
@@ -256,3 +257,9 @@ int fprint_localtime(FILE *F)
 	localtime_r(&now,&t);
 	return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec);
 }
+
+time_t file_mod_time(const char *filename)
+{
+	struct stat st;
+	return stat(filename,&st)==-1 ? 0 : st.st_mtime;
+}

nfq/helpers.h

@@ -45,3 +45,5 @@ bool parse_hex_str(const char *s, uint8_t *pbuf, size_t *size);
 void fill_pattern(uint8_t *buf,size_t bufsize,const void *pattern,size_t patsize);
 
 int fprint_localtime(FILE *F);
+
+time_t file_mod_time(const char *filename);

nfq/hostlist.c

@@ -2,7 +2,7 @@
 #include "hostlist.h"
 #include "gzip.h"
 #include "params.h"
-
+#include "helpers.h"
 
 // inplace tolower() and add to pool
 static bool addpool(strpool **hostlist, char **s, const char *end)
@@ -132,7 +132,7 @@ bool SearchHostList(strpool *hostlist, const char *host)
 }
 
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
+static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
 {
 	if (excluded) *excluded = false;
 	if (hostlist_exclude)
@@ -151,3 +151,36 @@ bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *hos
 	}
 	return true;
 }
+
+// return : true = apply fooling, false = do not apply
+bool HostlistCheck(const char *host, bool *excluded)
+{
+	if (*params.hostlist_auto_filename)
+	{
+		time_t t = file_mod_time(params.hostlist_auto_filename);
+		if (t!=params.hostlist_auto_mod_time)
+		{
+			printf("Autohostlist was modified by another process. Reloading include hostslist.\n");
+			if (!LoadIncludeHostLists())
+			{
+				// what will we do without hostlist ?? sure, gonna die
+				exit(1);
+			}
+			params.hostlist_auto_mod_time = t;
+		}
+	}
+	return HostlistCheck_(params.hostlist, params.hostlist_exclude, host, excluded);
+}
+
+bool LoadIncludeHostLists()
+{
+	if (!LoadHostLists(&params.hostlist, &params.hostlist_files))
+		return false;
+	if (*params.hostlist_auto_filename)
+		params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename);
+	return true;
+}
+bool LoadExcludeHostLists()
+{
+	return LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files);
+}

nfq/hostlist.h

@@ -5,7 +5,9 @@
 
 bool AppendHostList(strpool **hostlist, char *filename);
 bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list);
+bool LoadIncludeHostLists();
+bool LoadExcludeHostLists();
 bool NonEmptyHostlist(strpool **hostlist);
 bool SearchHostList(strpool *hostlist, const char *host);
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded);
+bool HostlistCheck(const char *host, bool *excluded);

nfq/nfqws.c

@@ -56,8 +56,7 @@ static void dohup(void)
 {
 	if (bHup)
 	{
-		if (!LoadHostLists(&params.hostlist, &params.hostlist_files) ||
-			!LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files))
+		if (!LoadIncludeHostLists() || !LoadExcludeHostLists())
 		{
 			// what will we do without hostlist ?? sure, gonna die
 			exit(1);
@@ -1232,13 +1231,13 @@ int main(int argc, char **argv)
 	}
 #endif
 
-	if (!LoadHostLists(&params.hostlist, &params.hostlist_files))
+	if (!LoadIncludeHostLists())
 	{
 		fprintf(stderr, "Include hostlist load failed\n");
 		exit_clean(1);
 	}
 	if (*params.hostlist_auto_filename) NonEmptyHostlist(&params.hostlist);
-	if (!LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files))
+	if (!LoadExcludeHostLists())
 	{
 		fprintf(stderr, "Exclude hostlist load failed\n");
 		exit_clean(1);

nfq/params.h

@@ -10,6 +10,7 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdio.h>
+#include <time.h>
 
 #define TLS_PARTIALS_ENABLE	true
 
@@ -68,6 +69,7 @@ struct params_s
 	struct str_list_head hostlist_files, hostlist_exclude_files;
 	char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX];
 	int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold;
+	time_t hostlist_auto_mod_time;
 	hostfail_pool *hostlist_auto_fail_counters;
 
 	unsigned int ctrack_t_syn, ctrack_t_est, ctrack_t_fin, ctrack_t_udp;

tpws/helpers.c

@@ -9,6 +9,7 @@
 #include <arpa/inet.h>
 #include <ifaddrs.h>
 #include <time.h>
+#include <sys/stat.h>
 
 char *strncasestr(const char *s,const char *find, size_t slen)
 {
@@ -222,3 +223,9 @@ int fprint_localtime(FILE *F)
 	localtime_r(&now,&t);
 	return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec);
 }
+
+time_t file_mod_time(const char *filename)
+{
+	struct stat st;
+	return stat(filename,&st)==-1 ? 0 : st.st_mtime;
+}

tpws/helpers.h

@@ -6,6 +6,7 @@
 #include <sys/socket.h>
 #include <netdb.h>
 #include <stdio.h>
+#include <time.h>
 
 char *strncasestr(const char *s,const char *find, size_t slen);
 
@@ -44,3 +45,5 @@ static inline void phton16(uint8_t *p, uint16_t v) {
 }
 
 int fprint_localtime(FILE *F);
+
+time_t file_mod_time(const char *filename);

tpws/hostlist.c

@@ -2,7 +2,7 @@
 #include "hostlist.h"
 #include "gzip.h"
 #include "params.h"
-
+#include "helpers.h"
 
 // inplace tolower() and add to pool
 static bool addpool(strpool **hostlist, char **s, const char *end)
@@ -132,7 +132,7 @@ bool SearchHostList(strpool *hostlist, const char *host)
 }
 
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
+static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
 {
 	if (excluded) *excluded = false;
 	if (hostlist_exclude)
@@ -151,3 +151,36 @@ bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *hos
 	}
 	return true;
 }
+
+// return : true = apply fooling, false = do not apply
+bool HostlistCheck(const char *host, bool *excluded)
+{
+	if (*params.hostlist_auto_filename)
+	{
+		time_t t = file_mod_time(params.hostlist_auto_filename);
+		if (t!=params.hostlist_auto_mod_time)
+		{
+			printf("Autohostlist was modified by another process. Reloading include hostslist.\n");
+			if (!LoadIncludeHostLists())
+			{
+				// what will we do without hostlist ?? sure, gonna die
+				exit(1);
+			}
+			params.hostlist_auto_mod_time = t;
+		}
+	}
+	return HostlistCheck_(params.hostlist, params.hostlist_exclude, host, excluded);
+}
+
+bool LoadIncludeHostLists()
+{
+	if (!LoadHostLists(&params.hostlist, &params.hostlist_files))
+		return false;
+	if (*params.hostlist_auto_filename)
+		params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename);
+	return true;
+}
+bool LoadExcludeHostLists()
+{
+	return LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files);
+}

tpws/hostlist.h

@@ -5,7 +5,9 @@
 
 bool AppendHostList(strpool **hostlist, char *filename);
 bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list);
+bool LoadIncludeHostLists();
+bool LoadExcludeHostLists();
 bool NonEmptyHostlist(strpool **hostlist);
 bool SearchHostList(strpool *hostlist, const char *host);
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded);
+bool HostlistCheck(const char *host, bool *excluded);

tpws/params.h

@@ -5,9 +5,10 @@
 #include <stdint.h>
 #include <sys/param.h>
 #include <sys/queue.h>
+#include <time.h>
 #include "pools.h"
 
-#define HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT	2
+#define HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT	3
 #define	HOSTLIST_AUTO_FAIL_TIME_DEFAULT 	60
 
 enum splithttpreq { split_none = 0, split_method, split_host };
@@ -59,6 +60,7 @@ struct params_s
 	struct str_list_head hostlist_files, hostlist_exclude_files;
 	char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX];
 	int hostlist_auto_fail_threshold, hostlist_auto_fail_time;
+	time_t hostlist_auto_mod_time;
 	hostfail_pool *hostlist_auto_fail_counters;
 
 	bool tamper_start_n,tamper_cutoff_n;

tpws/tamper.c

@@ -64,7 +64,7 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si
 			bHaveHost = true;
 			VPRINT("Requested Host is : %s", Host)
 			for(pc = Host; *pc; pc++) *pc=tolower(*pc);
-			bBypass = !HostlistCheck(params.hostlist, params.hostlist_exclude, Host, &bHostExcluded);
+			bBypass = !HostlistCheck(Host, &bHostExcluded);
 		}
 		if (!bBypass)
 		{
@@ -228,7 +228,7 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si
 		{
 			VPRINT("hostname: %s",Host)
 			bHaveHost = true;
-			bBypass = !HostlistCheck(params.hostlist, params.hostlist_exclude, Host, &bHostExcluded);
+			bBypass = !HostlistCheck(Host, &bHostExcluded);
 		}
 		if (bBypass)
 		{
@@ -307,7 +307,7 @@ static void auto_hostlist_failed(const char *hostname)
 		
 		VPRINT("auto hostlist : rechecking %s to avoid duplicates", hostname);
 		bool bExcluded=false;
-		if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded)
+		if (!HostlistCheck(hostname, &bExcluded) && !bExcluded)
 		{
 			VPRINT("auto hostlist : adding %s", hostname);
 			HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname);
@@ -321,6 +321,7 @@ static void auto_hostlist_failed(const char *hostname)
 				perror("write to auto hostlist:");
 				return;
 			}
+			params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename);
 		}
 		else
 		{

tpws/tpws.c

@@ -54,8 +54,7 @@ void dohup(void)
 {
 	if (bHup)
 	{
-		if (!LoadHostLists(&params.hostlist, &params.hostlist_files) ||
-			!LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files))
+		if (!LoadIncludeHostLists() || !LoadExcludeHostLists())
 		{
 			// what will we do without hostlist ?? sure, gonna die
 			exit(1);
@@ -717,13 +716,14 @@ void parse_params(int argc, char *argv[])
 		exit_clean(1);
 	}
 
-	if (!LoadHostLists(&params.hostlist, &params.hostlist_files))
+	if (*params.hostlist_auto_filename) params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename);
+	if (!LoadIncludeHostLists())
 	{
 		fprintf(stderr, "Include hostlist load failed\n");
 		exit_clean(1);
 	}
 	if (*params.hostlist_auto_filename) NonEmptyHostlist(&params.hostlist);
-	if (!LoadHostLists(&params.hostlist_exclude, &params.hostlist_exclude_files))
+	if (!LoadExcludeHostLists())
 	{
 		fprintf(stderr, "Exclude hostlist load failed\n");
 		exit_clean(1);