diff --git a/ipset/get_reestr_hostlist.sh b/ipset/get_reestr_hostlist.sh index f0883e1..aa5ad18 100755 --- a/ipset/get_reestr_hostlist.sh +++ b/ipset/get_reestr_hostlist.sh @@ -5,37 +5,60 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" -# useful in case ipban set is used in custom scripts -FAIL= -getipban || FAIL=1 -"$IPSET_DIR/create_ipset.sh" -[ -n "$FAIL" ] && exit - ZREESTR="$TMPDIR/zapret.txt" -#ZURL=https://reestr.rublacklist.net/api/current +IPB="$TMPDIR/ipb.txt" ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv -curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" || +dl_checked() { - echo reestr list download failed - exit 2 + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + # $5 - maxtime + curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" || + { + echo list download failed : $1 + return 2 + } + dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + return 2 + fi + return 0 } reestr_list() { LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }' } +reestr_extract_ip() +{ + LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}' +} -dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') -if test $dlsize -lt 204800; then - echo list file is too small. can be bad. +ipban_fin() +{ + getipban + "$IPSET_DIR/create_ipset.sh" +} + +dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || { + ipban_fin exit 2 -fi +} reestr_list | sort -u | zz "$ZHOSTLIST" +reestr_extract_ip <"$ZREESTR" >"$IPB" rm -f "$ZREESTR" +$AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN" +$AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6" +rm -f "$IPB" hup_zapret_daemons +ipban_fin + exit 0 diff --git a/ipset/get_reestr_preresolved.sh b/ipset/get_reestr_preresolved.sh index 984035a..db6f288 100755 --- a/ipset/get_reestr_preresolved.sh +++ b/ipset/get_reestr_preresolved.sh @@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" - -TMPLIST="$TMPDIR/list_nethub.txt" +TMPLIST="$TMPDIR/list.txt" BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main" URL4="$BASEURL/reestr_resolved4.txt" URL6="$BASEURL/reestr_resolved6.txt" +IPB4="$BASEURL/reestr_ipban4.txt" +IPB6="$BASEURL/reestr_ipban6.txt" dl() { # $1 - url # $2 - file - curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" || + # $3 - minsize + # $4 - maxsize + curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" || { echo list download failed : $1 exit 2 } dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') - if test $dlsize -lt 32768; then + if test $dlsize -lt $3; then echo list is too small : $dlsize bytes. can be bad. exit 2 fi @@ -31,8 +34,14 @@ dl() } getuser && { - [ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST" - [ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6" + [ "$DISABLE_IPV4" != "1" ] && { + dl "$URL4" "$ZIPLIST" 32768 4194304 + dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576 + } + [ "$DISABLE_IPV6" != "1" ] && { + dl "$URL6" "$ZIPLIST6" 8192 4194304 + dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576 + } } "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_preresolved_smart.sh b/ipset/get_reestr_preresolved_smart.sh index e8a70ae..f4a1395 100755 --- a/ipset/get_reestr_preresolved_smart.sh +++ b/ipset/get_reestr_preresolved_smart.sh @@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" . "$IPSET_DIR/def.sh" - -TMPLIST="$TMPDIR/list_nethub.txt" +TMPLIST="$TMPDIR/list.txt" BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main" URL4="$BASEURL/reestr_smart4.txt" URL6="$BASEURL/reestr_smart6.txt" +IPB4="$BASEURL/reestr_ipban4.txt" +IPB6="$BASEURL/reestr_ipban6.txt" dl() { # $1 - url # $2 - file - curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" || + # $3 - minsize + # $4 - maxsize + curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" || { echo list download failed : $1 exit 2 } dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') - if test $dlsize -lt 32768; then + if test $dlsize -lt $3; then echo list is too small : $dlsize bytes. can be bad. exit 2 fi @@ -31,8 +34,14 @@ dl() } getuser && { - [ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST" - [ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6" + [ "$DISABLE_IPV4" != "1" ] && { + dl "$URL4" "$ZIPLIST" 32768 4194304 + dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576 + } + [ "$DISABLE_IPV6" != "1" ] && { + dl "$URL6" "$ZIPLIST6" 8192 4194304 + dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576 + } } "$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_resolvable_domains.sh b/ipset/get_reestr_resolvable_domains.sh index 010b598..fa4837f 100755 --- a/ipset/get_reestr_resolvable_domains.sh +++ b/ipset/get_reestr_resolvable_domains.sh @@ -9,18 +9,22 @@ TMPLIST="$TMPDIR/list_nethub.txt" BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main" URL="$BASEURL/reestr_hostname_resolvable.txt" +IPB4="$BASEURL/reestr_ipban4.txt" +IPB6="$BASEURL/reestr_ipban6.txt" dl() { # $1 - url # $2 - file - curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" || + # $3 - minsize + # $4 - maxsize + curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" || { echo list download failed : $1 exit 2 } dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') - if test $dlsize -lt 65536; then + if test $dlsize -lt $3; then echo list is too small : $dlsize bytes. can be bad. exit 2 fi @@ -28,8 +32,14 @@ dl() rm -f "$TMPLIST" } -dl "$URL" "$ZHOSTLIST" +dl "$URL" "$ZHOSTLIST" 65536 67108864 hup_zapret_daemons +[ "$DISABLE_IPV4" != "1" ] && dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576 +[ "$DISABLE_IPV6" != "1" ] && dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576 + +getipban +"$IPSET_DIR/create_ipset.sh" + exit 0 diff --git a/ipset/get_reestr_resolve.sh b/ipset/get_reestr_resolve.sh index 59bad01..74b7201 100755 --- a/ipset/get_reestr_resolve.sh +++ b/ipset/get_reestr_resolve.sh @@ -7,28 +7,56 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" ZREESTR="$TMPDIR/zapret.txt" ZDIG="$TMPDIR/zapret-dig.txt" +IPB="$TMPDIR/ipb.txt" ZIPLISTTMP="$TMPDIR/zapret-ip.txt" #ZURL=https://reestr.rublacklist.net/api/current ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv +dl_checked() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + # $5 - maxtime + curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" || + { + echo list download failed : $1 + return 2 + } + dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + return 2 + fi + return 0 +} + +reestr_list() +{ + LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' +} +reestr_extract_ip() +{ + LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}' +} + getuser && { # both disabled [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0 - curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" || - { - echo reestr list download failed - exit 2 - } - - dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') - if test $dlsize -lt 204800; then - echo list file is too small. can be bad. - exit 2 - fi + dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || exit 2 + + echo preparing ipban list .. + + reestr_extract_ip <"$ZREESTR" >"$IPB" + $AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN" + $AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6" + rm -f "$IPB" echo preparing dig list .. - LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG" + reestr_list | sort -u >"$ZDIG" + rm -f "$ZREESTR" echo digging started. this can take long ...