ForkMaster/zapret
1
0
Fork 0
mirror of https://github.com/bol-van/zapret.git synced 2024-11-14 10:48:32 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

ipset: check dns before processing lists

Browse Source
This commit is contained in:
bol-van 2021-10-19 11:21:18 +03:00
parent cc85c1b489
commit e9b125e7ef
8 changed files with 144 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
ipset/def.sh
View File

@ -1,3 +1,8 @@
[ -n "$IPSET_DIR" ] || {
IPSET_DIR="$(dirname "$0")"
IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
}
. "$IPSET_DIR/../config" . "$IPSET_DIR/../config"
[ -z "$TMPDIR" ] && TMPDIR=/tmp [ -z "$TMPDIR" ] && TMPDIR=/tmp
@ -47,12 +52,15 @@ exists()
# MacoS in cron does not include /usr/local/bin to PATH # MacoS in cron does not include /usr/local/bin to PATH
if [ -x /usr/local/bin/ggrep ] ; then if [ -x /usr/local/bin/ggrep ] ; then
GREP=/usr/local/bin/ggrep GREP=/usr/local/bin/ggrep
elif [ -x /usr/local/bin/grep ] ; then
GREP=/usr/local/bin/grep
elif exists ggrep; then elif exists ggrep; then
GREP=$(which ggrep) GREP=$(which ggrep)
else else
GREP=$(which grep) GREP=$(which grep)
fi fi
grep_supports_b() grep_supports_b()
{ {
# \b does not work with BSD grep # \b does not work with BSD grep
@ -118,19 +126,32 @@ zzsize()
} }
digger() digger()
{
# $1 - family (4|6)
# $2 - s=enable mdig stats
if [ -x "$MDIG" ]; then
local cmd
[ "$2" == "s" ] && cmd=--stats=1000
"$MDIG" --family=$1 --threads=$MDIG_THREADS $1
else
local A=A
[ "$1" = "6" ] && A=AAAA
dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
fi
}
filedigger()
{ {
# $1 - hostlist # $1 - hostlist
# $2 - family (4|6) # $2 - family (4|6)
>&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1" >&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
zzcat "$1" | digger $2 s
if [ -x "$MDIG" ]; then
zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000
else
local A=A
[ "$2" = "6" ] && A=AAAA
zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
fi
} }
dnstest()
{
local ip=$(echo w3.org | digger 46)
[ -n "$ip" ]
}
cut_local() cut_local()
{ {
@ -152,24 +173,29 @@ oom_adjust_high()
getexclude() getexclude()
{ {
oom_adjust_high oom_adjust_high
dnstest || {
[ -f "$ZUSERLIST_EXCLUDE" ] && { echo "! DNS is not working. list processing aborted."
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE" return 1
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
} }
[ -f "$ZUSERLIST_EXCLUDE" ] && {
[ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
[ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
}
return 0
} }
getuser() getuser()
{ {
getexclude getexclude || return
[ -f "$ZUSERLIST" ] && { [ -f "$ZUSERLIST" ] && {
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER" [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6" [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
} }
[ -f "$ZUSERLIST_IPBAN" ] && { [ -f "$ZUSERLIST_IPBAN" ] && {
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN" [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6" [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
} }
return 0
} }
hup_zapret_daemons() hup_zapret_daemons()

9
ipset/get_antifilter_ip.sh
View File

@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
getuser getuser && {
. "$IPSET_DIR/antifilter.helper"
. "$IPSET_DIR/antifilter.helper" get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
}
get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

9
ipset/get_antifilter_ipsmart.sh
View File

@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
getuser getuser && {
. "$IPSET_DIR/antifilter.helper"
. "$IPSET_DIR/antifilter.helper" get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
}
get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

9
ipset/get_antifilter_ipsum.sh
View File

@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
getuser getuser && {
. "$IPSET_DIR/antifilter.helper"
. "$IPSET_DIR/antifilter.helper" get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
}
get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

29
ipset/get_reestr_combined.sh
View File

@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
#ZURL_REESTR=https://reestr.rublacklist.net/api/current #ZURL_REESTR=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser
dig_reestr() dig_reestr()
{ {
@ -36,29 +35,31 @@ dig_reestr()
rm -f "$TMP" rm -f "$TMP"
} }
getuser && {
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
{ {
echo reestr list download failed echo reestr list download failed
exit 2 exit 2
} }
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 1048576; then if test $dlsize -lt 1048576; then
echo reestr ip list is too small. can be bad. echo reestr ip list is too small. can be bad.
exit 2 exit 2
fi fi
#sed -i 's/\\n/\r\n/g' $ZREESTR #sed -i 's/\\n/\r\n/g' $ZREESTR
get_ip_regex get_ip_regex
[ "$DISABLE_IPV4" != "1" ] && { [ "$DISABLE_IPV4" != "1" ] && {
dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4 dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
} }
[ "$DISABLE_IPV6" != "1" ] && { [ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6 dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
} }
rm -f "$ZREESTR" rm -f "$ZREESTR"
}
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

4
ipset/get_reestr_hostlist.sh
View File

@ -6,8 +6,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
# useful in case ipban set is used in custom scripts # useful in case ipban set is used in custom scripts
getuser FAIL=
getuser || FAIL=1
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"
[ -n "$FAIL" ] && exit
ZREESTR="$TMPDIR/zapret.txt" ZREESTR="$TMPDIR/zapret.txt"
#ZURL=https://reestr.rublacklist.net/api/current #ZURL=https://reestr.rublacklist.net/api/current

31
ipset/get_reestr_ip.sh
View File

@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
#ZURL_REESTR=https://reestr.rublacklist.net/api/current #ZURL_REESTR=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser
dig_reestr() dig_reestr()
{ {
@ -23,29 +22,31 @@ dig_reestr()
} }
# assume all https banned by ip getuser && {
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || # assume all https banned by ip
{ curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
{
echo reestr list download failed echo reestr list download failed
exit 2 exit 2
} }
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 1048576; then if test $dlsize -lt 1048576; then
echo reestr ip list is too small. can be bad. echo reestr ip list is too small. can be bad.
exit 2 exit 2
fi fi
#sed -i 's/\\n/\r\n/g' $ZREESTR #sed -i 's/\\n/\r\n/g' $ZREESTR
get_ip_regex get_ip_regex
[ "$DISABLE_IPV4" != "1" ] && { [ "$DISABLE_IPV4" != "1" ] && {
dig_reestr "$REG_IPV4" "$ZIPLIST" 4 dig_reestr "$REG_IPV4" "$ZIPLIST" 4
} }
[ "$DISABLE_IPV6" != "1" ] && { [ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" 6 dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
} }
rm -f "$ZREESTR" rm -f "$ZREESTR"
}
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

41
ipset/get_reestr_resolve.sh
View File

@ -11,44 +11,45 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
#ZURL=https://reestr.rublacklist.net/api/current #ZURL=https://reestr.rublacklist.net/api/current
ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser getuser && {
# both disabled
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
# both disabled curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0 {
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
{
echo reestr list download failed echo reestr list download failed
exit 2 exit 2
} }
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then if test $dlsize -lt 204800; then
echo list file is too small. can be bad. echo list file is too small. can be bad.
exit 2 exit 2
fi fi
echo preparing dig list .. echo preparing dig list ..
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG" LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
rm -f "$ZREESTR" rm -f "$ZREESTR"
echo digging started. this can take long ... echo digging started. this can take long ...
[ "$DISABLE_IPV4" != "1" ] && { [ "$DISABLE_IPV4" != "1" ] && {
digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || { filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
rm -f "$ZDIG" rm -f "$ZDIG"
exit 1 exit 1
} }
ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST" ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
rm -f "$ZIPLISTTMP" rm -f "$ZIPLISTTMP"
} }
[ "$DISABLE_IPV6" != "1" ] && { [ "$DISABLE_IPV6" != "1" ] && {
digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || { filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
rm -f "$ZDIG" rm -f "$ZDIG"
exit 1 exit 1
} }
ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6" ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
rm -f "$ZIPLISTTMP" rm -f "$ZIPLISTTMP"
}
rm -f "$ZDIG"
} }
rm -f "$ZDIG"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"