From 7355b20544ca86d15556530e142d66c7500fd29a Mon Sep 17 00:00:00 2001
From: bol-van <k@vodka.home.kg>
Date: Tue, 19 Oct 2021 11:22:25 +0300
Subject: [PATCH] Revert "ipset: check dns before processing lists"

This reverts commit 033def82ffec51155bb44daf7cef408647e5b8cd.
---
 ipset/def.sh                    | 58 ++++++++------------------
 ipset/get_antifilter_ip.sh      |  9 ++--
 ipset/get_antifilter_ipsmart.sh |  9 ++--
 ipset/get_antifilter_ipsum.sh   |  9 ++--
 ipset/get_reestr_combined.sh    | 43 ++++++++++---------
 ipset/get_reestr_hostlist.sh    |  4 +-
 ipset/get_reestr_ip.sh          | 45 ++++++++++----------
 ipset/get_reestr_resolve.sh     | 73 ++++++++++++++++-----------------
 8 files changed, 111 insertions(+), 139 deletions(-)

diff --git a/ipset/def.sh b/ipset/def.sh
index fbacf87..9d1dc06 100644
--- a/ipset/def.sh
+++ b/ipset/def.sh
@@ -1,8 +1,3 @@
-[ -n "$IPSET_DIR" ] || {
- IPSET_DIR="$(dirname "$0")"
- IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
-}
-
 . "$IPSET_DIR/../config"
 
 [ -z "$TMPDIR" ] && TMPDIR=/tmp
@@ -52,15 +47,12 @@ exists()
 # MacoS in cron does not include /usr/local/bin to PATH
 if [ -x /usr/local/bin/ggrep ] ; then
  GREP=/usr/local/bin/ggrep
-elif [ -x /usr/local/bin/grep ] ; then
- GREP=/usr/local/bin/grep
 elif exists ggrep; then
  GREP=$(which ggrep)
 else
  GREP=$(which grep)
 fi
 
-
 grep_supports_b()
 {
  # \b does not work with BSD grep
@@ -126,32 +118,19 @@ zzsize()
 }
 
 digger()
-{
- # $1 - family (4|6)
- # $2 - s=enable mdig stats
- if [ -x "$MDIG" ]; then
-  local cmd
-  [ "$2" == "s" ] && cmd=--stats=1000
-  "$MDIG" --family=$1 --threads=$MDIG_THREADS $1 
- else
-  local A=A
-  [ "$1" = "6" ] && A=AAAA
-  dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
- fi
-}
-filedigger()
 {
  # $1 - hostlist
  # $2 - family (4|6)
  >&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
- zzcat "$1" | digger $2 s
-}
-dnstest()
-{
- local ip=$(echo w3.org | digger 46)
- [ -n "$ip" ]
-}
 
+ if [ -x "$MDIG" ]; then
+  zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000
+ else
+  local A=A
+  [ "$2" = "6" ] && A=AAAA
+  zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
+ fi
+}
 
 cut_local()
 {
@@ -173,29 +152,24 @@ oom_adjust_high()
 getexclude()
 {
  oom_adjust_high
- dnstest || {
-    echo "! DNS is not working. list processing aborted."
-    return 1
- }
+
  [ -f "$ZUSERLIST_EXCLUDE" ] && {
-  [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
-  [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
+  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
+  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
  }
- return 0
 }
 
 getuser()
 {
- getexclude || return
+ getexclude
  [ -f "$ZUSERLIST" ] && {
-  [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
-  [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
+  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
+  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
  }
  [ -f "$ZUSERLIST_IPBAN" ] && {
-  [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
-  [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
+  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
+  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
  }
- return 0
 }
 
 hup_zapret_daemons()
diff --git a/ipset/get_antifilter_ip.sh b/ipset/get_antifilter_ip.sh
index 91eda04..1628890 100755
--- a/ipset/get_antifilter_ip.sh
+++ b/ipset/get_antifilter_ip.sh
@@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-getuser && {
- . "$IPSET_DIR/antifilter.helper"
- get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
-}
+getuser
+
+. "$IPSET_DIR/antifilter.helper"
+
+get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
 
 "$IPSET_DIR/create_ipset.sh"
diff --git a/ipset/get_antifilter_ipsmart.sh b/ipset/get_antifilter_ipsmart.sh
index 9f0d671..b71298f 100755
--- a/ipset/get_antifilter_ipsmart.sh
+++ b/ipset/get_antifilter_ipsmart.sh
@@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-getuser && {
- . "$IPSET_DIR/antifilter.helper"
- get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
-}
+getuser
+
+. "$IPSET_DIR/antifilter.helper"
+
+get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
 
 "$IPSET_DIR/create_ipset.sh"
diff --git a/ipset/get_antifilter_ipsum.sh b/ipset/get_antifilter_ipsum.sh
index 71759de..3fc2673 100755
--- a/ipset/get_antifilter_ipsum.sh
+++ b/ipset/get_antifilter_ipsum.sh
@@ -5,9 +5,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-getuser && {
- . "$IPSET_DIR/antifilter.helper"
- get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
-}
+getuser
+
+. "$IPSET_DIR/antifilter.helper"
+
+get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
 
 "$IPSET_DIR/create_ipset.sh"
diff --git a/ipset/get_reestr_combined.sh b/ipset/get_reestr_combined.sh
index 1a5569e..675d37d 100755
--- a/ipset/get_reestr_combined.sh
+++ b/ipset/get_reestr_combined.sh
@@ -9,6 +9,7 @@ ZREESTR="$TMPDIR/reestr.txt"
 #ZURL_REESTR=https://reestr.rublacklist.net/api/current
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
+getuser
 
 dig_reestr()
 {
@@ -35,31 +36,29 @@ dig_reestr()
  rm -f "$TMP"
 }
 
-getuser && {
 
- curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
- {
-  echo reestr list download failed
-  exit 2
- }
- dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
- if test $dlsize -lt 1048576; then
-  echo reestr ip list is too small. can be bad.
-  exit 2
- fi
- #sed -i 's/\\n/\r\n/g' $ZREESTR
+curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
+{
+ echo reestr list download failed
+ exit 2
+}
+dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+if test $dlsize -lt 1048576; then
+ echo reestr ip list is too small. can be bad.
+ exit 2
+fi
+#sed -i 's/\\n/\r\n/g' $ZREESTR
 
- get_ip_regex
+get_ip_regex
 
- [ "$DISABLE_IPV4" != "1" ] && {
-  dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
- }
-
- [ "$DISABLE_IPV6" != "1" ] && {
-  dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
- }
-
- rm -f "$ZREESTR"
+[ "$DISABLE_IPV4" != "1" ] && {
+ dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
 }
 
+[ "$DISABLE_IPV6" != "1" ] && {
+ dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
+}
+
+rm -f "$ZREESTR"
+
 "$IPSET_DIR/create_ipset.sh"
diff --git a/ipset/get_reestr_hostlist.sh b/ipset/get_reestr_hostlist.sh
index 3900434..d204d27 100755
--- a/ipset/get_reestr_hostlist.sh
+++ b/ipset/get_reestr_hostlist.sh
@@ -6,10 +6,8 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 . "$IPSET_DIR/def.sh"
 
 # useful in case ipban set is used in custom scripts
-FAIL=
-getuser || FAIL=1
+getuser
 "$IPSET_DIR/create_ipset.sh"
-[ -n "$FAIL" ] && exit
 
 ZREESTR="$TMPDIR/zapret.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
diff --git a/ipset/get_reestr_ip.sh b/ipset/get_reestr_ip.sh
index 7ab8f27..0eeec80 100755
--- a/ipset/get_reestr_ip.sh
+++ b/ipset/get_reestr_ip.sh
@@ -9,6 +9,7 @@ ZREESTR="$TMPDIR/reestr.txt"
 #ZURL_REESTR=https://reestr.rublacklist.net/api/current
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
+getuser
 
 dig_reestr()
 {
@@ -22,31 +23,29 @@ dig_reestr()
 }
 
 
-getuser && {
- # assume all https banned by ip
- curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
- {
-  echo reestr list download failed
-  exit 2
- }
- dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
- if test $dlsize -lt 1048576; then
-  echo reestr ip list is too small. can be bad.
-  exit 2
- fi
- #sed -i 's/\\n/\r\n/g' $ZREESTR
+# assume all https banned by ip
+curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
+{
+ echo reestr list download failed
+ exit 2
+}
+dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+if test $dlsize -lt 1048576; then
+ echo reestr ip list is too small. can be bad.
+ exit 2
+fi
+#sed -i 's/\\n/\r\n/g' $ZREESTR
 
- get_ip_regex
+get_ip_regex
 
- [ "$DISABLE_IPV4" != "1" ] && {
-  dig_reestr "$REG_IPV4" "$ZIPLIST" 4
- }
-
- [ "$DISABLE_IPV6" != "1" ] && {
-  dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
- }
-
- rm -f "$ZREESTR"
+[ "$DISABLE_IPV4" != "1" ] && {
+ dig_reestr "$REG_IPV4" "$ZIPLIST" 4
 }
 
+[ "$DISABLE_IPV6" != "1" ] && {
+ dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
+}
+
+rm -f "$ZREESTR"
+
 "$IPSET_DIR/create_ipset.sh"
diff --git a/ipset/get_reestr_resolve.sh b/ipset/get_reestr_resolve.sh
index ae986ed..c6913ad 100755
--- a/ipset/get_reestr_resolve.sh
+++ b/ipset/get_reestr_resolve.sh
@@ -11,45 +11,44 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
 ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
-getuser && {
- # both disabled
- [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
+getuser
 
- curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
- {
-  echo reestr list download failed   
-  exit 2
- }
+# both disabled
+[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
 
- dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
- if test $dlsize -lt 204800; then
-  echo list file is too small. can be bad.
-  exit 2
- fi
-
- echo preparing dig list ..
- LANG=C cut -f2 -d ';' "$ZREESTR"  | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
- rm -f "$ZREESTR"
-
- echo digging started. this can take long ...
-
- [ "$DISABLE_IPV4" != "1" ] && {
-  filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
-   rm -f "$ZDIG"
-   exit 1
-  }
-  ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
-  rm -f "$ZIPLISTTMP"
- }
- [ "$DISABLE_IPV6" != "1" ] && {
-  filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
-   rm -f "$ZDIG"
-   exit 1
-  }
-  ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
-  rm -f "$ZIPLISTTMP"
- }
- rm -f "$ZDIG"
+curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
+{
+ echo reestr list download failed   
+ exit 2
 }
 
+dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+if test $dlsize -lt 204800; then
+ echo list file is too small. can be bad.
+ exit 2
+fi
+
+echo preparing dig list ..
+LANG=C cut -f2 -d ';' "$ZREESTR"  | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
+rm -f "$ZREESTR"
+
+echo digging started. this can take long ...
+
+[ "$DISABLE_IPV4" != "1" ] && {
+ digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
+  rm -f "$ZDIG"
+  exit 1
+ }
+ ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
+ rm -f "$ZIPLISTTMP"
+}
+[ "$DISABLE_IPV6" != "1" ] && {
+ digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
+  rm -f "$ZDIG"
+  exit 1
+ }
+ ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
+ rm -f "$ZIPLISTTMP"
+}
+rm -f "$ZDIG"
 "$IPSET_DIR/create_ipset.sh"