diff --git a/ipset/def.sh b/ipset/def.sh index 9d1dc06..fbacf87 100644 --- a/ipset/def.sh +++ b/ipset/def.sh @@ -1,3 +1,8 @@ +[ -n "$IPSET_DIR" ] || { + IPSET_DIR="$(dirname "$0")" + IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" +} + . "$IPSET_DIR/../config" [ -z "$TMPDIR" ] && TMPDIR=/tmp @@ -47,12 +52,15 @@ exists() # MacoS in cron does not include /usr/local/bin to PATH if [ -x /usr/local/bin/ggrep ] ; then GREP=/usr/local/bin/ggrep +elif [ -x /usr/local/bin/grep ] ; then + GREP=/usr/local/bin/grep elif exists ggrep; then GREP=$(which ggrep) else GREP=$(which grep) fi + grep_supports_b() { # \b does not work with BSD grep @@ -118,19 +126,32 @@ zzsize() } digger() +{ + # $1 - family (4|6) + # $2 - s=enable mdig stats + if [ -x "$MDIG" ]; then + local cmd + [ "$2" == "s" ] && cmd=--stats=1000 + "$MDIG" --family=$1 --threads=$MDIG_THREADS $1 + else + local A=A + [ "$1" = "6" ] && A=AAAA + dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$' + fi +} +filedigger() { # $1 - hostlist # $2 - family (4|6) >&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1" - - if [ -x "$MDIG" ]; then - zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000 - else - local A=A - [ "$2" = "6" ] && A=AAAA - zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$' - fi + zzcat "$1" | digger $2 s } +dnstest() +{ + local ip=$(echo w3.org | digger 46) + [ -n "$ip" ] +} + cut_local() { @@ -152,24 +173,29 @@ oom_adjust_high() getexclude() { oom_adjust_high - - [ -f "$ZUSERLIST_EXCLUDE" ] && { - [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE" - [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6" + dnstest || { + echo "! DNS is not working. list processing aborted." + return 1 } + [ -f "$ZUSERLIST_EXCLUDE" ] && { + [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE" + [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6" + } + return 0 } getuser() { - getexclude + getexclude || return [ -f "$ZUSERLIST" ] && { - [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER" - [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6" + [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER" + [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6" } [ -f "$ZUSERLIST_IPBAN" ] && { - [ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN" - [ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6" + [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN" + [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6" } + return 0 } hup_zapret_daemons() diff --git a/ipset/get_antifilter_ip.sh b/ipset/get_antifilter_ip.sh index 1628890..91eda04 100755 --- a/ipset/get_antifilter_ip.sh +++ b/ipset/get_antifilter_ip.sh @@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" -getuser - -. "$IPSET_DIR/antifilter.helper" - -get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST" +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST" +} "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ipsmart.sh b/ipset/get_antifilter_ipsmart.sh index b71298f..9f0d671 100755 --- a/ipset/get_antifilter_ipsmart.sh +++ b/ipset/get_antifilter_ipsmart.sh @@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" -getuser - -. "$IPSET_DIR/antifilter.helper" - -get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST" +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST" +} "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ipsum.sh b/ipset/get_antifilter_ipsum.sh index 3fc2673..71759de 100755 --- a/ipset/get_antifilter_ipsum.sh +++ b/ipset/get_antifilter_ipsum.sh @@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" -getuser - -. "$IPSET_DIR/antifilter.helper" - -get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST" +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST" +} "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_combined.sh b/ipset/get_reestr_combined.sh index 675d37d..1a5569e 100755 --- a/ipset/get_reestr_combined.sh +++ b/ipset/get_reestr_combined.sh @@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt" #ZURL_REESTR=https://reestr.rublacklist.net/api/current ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv -getuser dig_reestr() { @@ -36,29 +35,31 @@ dig_reestr() rm -f "$TMP" } +getuser && { -curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || -{ - echo reestr list download failed - exit 2 + curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || + { + echo reestr list download failed + exit 2 + } + dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt 1048576; then + echo reestr ip list is too small. can be bad. + exit 2 + fi + #sed -i 's/\\n/\r\n/g' $ZREESTR + + get_ip_regex + + [ "$DISABLE_IPV4" != "1" ] && { + dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4 + } + + [ "$DISABLE_IPV6" != "1" ] && { + dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6 + } + + rm -f "$ZREESTR" } -dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') -if test $dlsize -lt 1048576; then - echo reestr ip list is too small. can be bad. - exit 2 -fi -#sed -i 's/\\n/\r\n/g' $ZREESTR - -get_ip_regex - -[ "$DISABLE_IPV4" != "1" ] && { - dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4 -} - -[ "$DISABLE_IPV6" != "1" ] && { - dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6 -} - -rm -f "$ZREESTR" "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_hostlist.sh b/ipset/get_reestr_hostlist.sh index d204d27..3900434 100755 --- a/ipset/get_reestr_hostlist.sh +++ b/ipset/get_reestr_hostlist.sh @@ -6,8 +6,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" # useful in case ipban set is used in custom scripts -getuser +FAIL= +getuser || FAIL=1 "$IPSET_DIR/create_ipset.sh" +[ -n "$FAIL" ] && exit ZREESTR="$TMPDIR/zapret.txt" #ZURL=https://reestr.rublacklist.net/api/current diff --git a/ipset/get_reestr_ip.sh b/ipset/get_reestr_ip.sh index 0eeec80..7ab8f27 100755 --- a/ipset/get_reestr_ip.sh +++ b/ipset/get_reestr_ip.sh @@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt" #ZURL_REESTR=https://reestr.rublacklist.net/api/current ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv -getuser dig_reestr() { @@ -23,29 +22,31 @@ dig_reestr() } -# assume all https banned by ip -curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || -{ - echo reestr list download failed - exit 2 +getuser && { + # assume all https banned by ip + curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || + { + echo reestr list download failed + exit 2 + } + dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt 1048576; then + echo reestr ip list is too small. can be bad. + exit 2 + fi + #sed -i 's/\\n/\r\n/g' $ZREESTR + + get_ip_regex + + [ "$DISABLE_IPV4" != "1" ] && { + dig_reestr "$REG_IPV4" "$ZIPLIST" 4 + } + + [ "$DISABLE_IPV6" != "1" ] && { + dig_reestr "$REG_IPV6" "$ZIPLIST6" 6 + } + + rm -f "$ZREESTR" } -dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') -if test $dlsize -lt 1048576; then - echo reestr ip list is too small. can be bad. - exit 2 -fi -#sed -i 's/\\n/\r\n/g' $ZREESTR - -get_ip_regex - -[ "$DISABLE_IPV4" != "1" ] && { - dig_reestr "$REG_IPV4" "$ZIPLIST" 4 -} - -[ "$DISABLE_IPV6" != "1" ] && { - dig_reestr "$REG_IPV6" "$ZIPLIST6" 6 -} - -rm -f "$ZREESTR" "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_resolve.sh b/ipset/get_reestr_resolve.sh index c6913ad..ae986ed 100755 --- a/ipset/get_reestr_resolve.sh +++ b/ipset/get_reestr_resolve.sh @@ -11,44 +11,45 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt" #ZURL=https://reestr.rublacklist.net/api/current ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv -getuser +getuser && { + # both disabled + [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0 -# both disabled -[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0 - -curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" || -{ - echo reestr list download failed - exit 2 -} - -dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') -if test $dlsize -lt 204800; then - echo list file is too small. can be bad. - exit 2 -fi - -echo preparing dig list .. -LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG" -rm -f "$ZREESTR" - -echo digging started. this can take long ... - -[ "$DISABLE_IPV4" != "1" ] && { - digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || { - rm -f "$ZDIG" - exit 1 + curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" || + { + echo reestr list download failed + exit 2 } - ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST" - rm -f "$ZIPLISTTMP" -} -[ "$DISABLE_IPV6" != "1" ] && { - digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || { - rm -f "$ZDIG" - exit 1 + + dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt 204800; then + echo list file is too small. can be bad. + exit 2 + fi + + echo preparing dig list .. + LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG" + rm -f "$ZREESTR" + + echo digging started. this can take long ... + + [ "$DISABLE_IPV4" != "1" ] && { + filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || { + rm -f "$ZDIG" + exit 1 + } + ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST" + rm -f "$ZIPLISTTMP" } - ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6" - rm -f "$ZIPLISTTMP" + [ "$DISABLE_IPV6" != "1" ] && { + filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || { + rm -f "$ZDIG" + exit 1 + } + ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6" + rm -f "$ZIPLISTTMP" + } + rm -f "$ZDIG" } -rm -f "$ZDIG" + "$IPSET_DIR/create_ipset.sh"