ipset: check dns before processing lists

ipset/def.sh

@@ -1,3 +1,8 @@
+[ -n "$IPSET_DIR" ] || {
+ IPSET_DIR="$(dirname "$0")"
+ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
+}
+
 . "$IPSET_DIR/../config"
 
 [ -z "$TMPDIR" ] && TMPDIR=/tmp
@@ -47,12 +52,15 @@ exists()
 # MacoS in cron does not include /usr/local/bin to PATH
 if [ -x /usr/local/bin/ggrep ] ; then
  GREP=/usr/local/bin/ggrep
+elif [ -x /usr/local/bin/grep ] ; then
+ GREP=/usr/local/bin/grep
 elif exists ggrep; then
  GREP=$(which ggrep)
 else
  GREP=$(which grep)
 fi
 
+
 grep_supports_b()
 {
  # \b does not work with BSD grep
@@ -118,19 +126,32 @@ zzsize()
 }
 
 digger()
+{
+ # $1 - family (4|6)
+ # $2 - s=enable mdig stats
+ if [ -x "$MDIG" ]; then
+  local cmd
+  [ "$2" == "s" ] && cmd=--stats=1000
+  "$MDIG" --family=$1 --threads=$MDIG_THREADS $1 
+ else
+  local A=A
+  [ "$1" = "6" ] && A=AAAA
+  dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
+ fi
+}
+filedigger()
 {
  # $1 - hostlist
  # $2 - family (4|6)
  >&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
-
+ zzcat "$1" | digger $2 s
- if [ -x "$MDIG" ]; then
-  zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000
- else
-  local A=A
-  [ "$2" = "6" ] && A=AAAA
-  zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
- fi
 }
+dnstest()
+{
+ local ip=$(echo w3.org | digger 46)
+ [ -n "$ip" ]
+}
+
 
 cut_local()
 {
@@ -152,24 +173,29 @@ oom_adjust_high()
 getexclude()
 {
  oom_adjust_high
-
+ dnstest || {
- [ -f "$ZUSERLIST_EXCLUDE" ] && {
+    echo "! DNS is not working. list processing aborted."
-  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
+    return 1
-  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
  }
+ [ -f "$ZUSERLIST_EXCLUDE" ] && {
+  [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
+  [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
+ }
+ return 0
 }
 
 getuser()
 {
- getexclude
+ getexclude || return
  [ -f "$ZUSERLIST" ] && {
-  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
+  [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
-  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
+  [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
  }
  [ -f "$ZUSERLIST_IPBAN" ] && {
-  [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
+  [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
-  [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
+  [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
  }
+ return 0
 }
 
 hup_zapret_daemons()

ipset/get_antifilter_ip.sh

@@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-getuser
+getuser && {
-
+ . "$IPSET_DIR/antifilter.helper"
-. "$IPSET_DIR/antifilter.helper"
+ get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
-
+}
-get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
 
 "$IPSET_DIR/create_ipset.sh"

ipset/get_antifilter_ipsmart.sh

@@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-getuser
+getuser && {
-
+ . "$IPSET_DIR/antifilter.helper"
-. "$IPSET_DIR/antifilter.helper"
+ get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
-
+}
-get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
 
 "$IPSET_DIR/create_ipset.sh"

ipset/get_antifilter_ipsum.sh

@@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-getuser
+getuser && {
-
+ . "$IPSET_DIR/antifilter.helper"
-. "$IPSET_DIR/antifilter.helper"
+ get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
-
+}
-get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
 
 "$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_combined.sh

@@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
 #ZURL_REESTR=https://reestr.rublacklist.net/api/current
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
-getuser
 
 dig_reestr()
 {
@@ -36,29 +35,31 @@ dig_reestr()
  rm -f "$TMP"
 }
 
+getuser && {
 
-curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
+ curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
-{
+ {
- echo reestr list download failed
+  echo reestr list download failed
- exit 2
+  exit 2
+ }
+ dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+ if test $dlsize -lt 1048576; then
+  echo reestr ip list is too small. can be bad.
+  exit 2
+ fi
+ #sed -i 's/\\n/\r\n/g' $ZREESTR
+
+ get_ip_regex
+
+ [ "$DISABLE_IPV4" != "1" ] && {
+  dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
+ }
+
+ [ "$DISABLE_IPV6" != "1" ] && {
+  dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
+ }
+
+ rm -f "$ZREESTR"
 }
-dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
-if test $dlsize -lt 1048576; then
- echo reestr ip list is too small. can be bad.
- exit 2
-fi
-#sed -i 's/\\n/\r\n/g' $ZREESTR
-
-get_ip_regex
-
-[ "$DISABLE_IPV4" != "1" ] && {
- dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
-}
-
-[ "$DISABLE_IPV6" != "1" ] && {
- dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
-}
-
-rm -f "$ZREESTR"
 
 "$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_hostlist.sh

@@ -6,8 +6,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 . "$IPSET_DIR/def.sh"
 
 # useful in case ipban set is used in custom scripts
-getuser
+FAIL=
+getuser || FAIL=1
 "$IPSET_DIR/create_ipset.sh"
+[ -n "$FAIL" ] && exit
 
 ZREESTR="$TMPDIR/zapret.txt"
 #ZURL=https://reestr.rublacklist.net/api/current

ipset/get_reestr_ip.sh

@@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
 #ZURL_REESTR=https://reestr.rublacklist.net/api/current
 ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
-getuser
 
 dig_reestr()
 {
@@ -23,29 +22,31 @@ dig_reestr()
 }
 
 
-# assume all https banned by ip
+getuser && {
-curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
+ # assume all https banned by ip
-{
+ curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
- echo reestr list download failed
+ {
- exit 2
+  echo reestr list download failed
+  exit 2
+ }
+ dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
+ if test $dlsize -lt 1048576; then
+  echo reestr ip list is too small. can be bad.
+  exit 2
+ fi
+ #sed -i 's/\\n/\r\n/g' $ZREESTR
+
+ get_ip_regex
+
+ [ "$DISABLE_IPV4" != "1" ] && {
+  dig_reestr "$REG_IPV4" "$ZIPLIST" 4
+ }
+
+ [ "$DISABLE_IPV6" != "1" ] && {
+  dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
+ }
+
+ rm -f "$ZREESTR"
 }
-dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
-if test $dlsize -lt 1048576; then
- echo reestr ip list is too small. can be bad.
- exit 2
-fi
-#sed -i 's/\\n/\r\n/g' $ZREESTR
-
-get_ip_regex
-
-[ "$DISABLE_IPV4" != "1" ] && {
- dig_reestr "$REG_IPV4" "$ZIPLIST" 4
-}
-
-[ "$DISABLE_IPV6" != "1" ] && {
- dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
-}
-
-rm -f "$ZREESTR"
 
 "$IPSET_DIR/create_ipset.sh"

ipset/get_reestr_resolve.sh

@@ -11,44 +11,45 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
 ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
-getuser
+getuser && {
+ # both disabled
+ [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
 
-# both disabled
+ curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
-[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
+ {
-
+  echo reestr list download failed   
-curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
+  exit 2
-{
- echo reestr list download failed   
- exit 2
-}
-
-dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
-if test $dlsize -lt 204800; then
- echo list file is too small. can be bad.
- exit 2
-fi
-
-echo preparing dig list ..
-LANG=C cut -f2 -d ';' "$ZREESTR"  | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
-rm -f "$ZREESTR"
-
-echo digging started. this can take long ...
-
-[ "$DISABLE_IPV4" != "1" ] && {
- digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
-  rm -f "$ZDIG"
-  exit 1
  }
- ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
+
- rm -f "$ZIPLISTTMP"
+ dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
-}
+ if test $dlsize -lt 204800; then
-[ "$DISABLE_IPV6" != "1" ] && {
+  echo list file is too small. can be bad.
- digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
+  exit 2
-  rm -f "$ZDIG"
+ fi
-  exit 1
+
+ echo preparing dig list ..
+ LANG=C cut -f2 -d ';' "$ZREESTR"  | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
+ rm -f "$ZREESTR"
+
+ echo digging started. this can take long ...
+
+ [ "$DISABLE_IPV4" != "1" ] && {
+  filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
+   rm -f "$ZDIG"
+   exit 1
+  }
+  ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
+  rm -f "$ZIPLISTTMP"
  }
- ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
+ [ "$DISABLE_IPV6" != "1" ] && {
- rm -f "$ZIPLISTTMP"
+  filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
+   rm -f "$ZDIG"
+   exit 1
+  }
+  ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
+  rm -f "$ZIPLISTTMP"
+ }
+ rm -f "$ZDIG"
 }
-rm -f "$ZDIG"
+
 "$IPSET_DIR/create_ipset.sh"