ForkMaster/zapret
1
0
Fork 0
mirror of https://github.com/bol-van/zapret.git synced 2024-11-11 17:29:16 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

reestr lists: support ipban

Browse Source
This commit is contained in:
bol-van 2023-11-26 14:31:37 +03:00
parent 55c2783aea
commit e716971739
5 changed files with 120 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
ipset/get_reestr_hostlist.sh
View File

@ -5,37 +5,60 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
# useful in case ipban set is used in custom scripts
FAIL=
getipban || FAIL=1
"$IPSET_DIR/create_ipset.sh"
[ -n "$FAIL" ] && exit
ZREESTR="$TMPDIR/zapret.txt"
#ZURL=https://reestr.rublacklist.net/api/current
IPB="$TMPDIR/ipb.txt"
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
dl_checked()
{
echo reestr list download failed
exit 2
# $1 - url
# $2 - file
# $3 - minsize
# $4 - maxsize
# $5 - maxtime
curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
{
echo list download failed : $1
return 2
}
dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
return 2
fi
return 0
}
reestr_list()
{
LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
}
reestr_extract_ip()
{
LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
ipban_fin()
{
getipban
"$IPSET_DIR/create_ipset.sh"
}
dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || {
ipban_fin
exit 2
fi
}
reestr_list | sort -u | zz "$ZHOSTLIST"
reestr_extract_ip <"$ZREESTR" >"$IPB"
rm -f "$ZREESTR"
$AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
$AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
rm -f "$IPB"
hup_zapret_daemons
ipban_fin
exit 0

21
ipset/get_reestr_preresolved.sh
View File

@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
TMPLIST="$TMPDIR/list_nethub.txt"
TMPLIST="$TMPDIR/list.txt"
BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
URL4="$BASEURL/reestr_resolved4.txt"
URL6="$BASEURL/reestr_resolved6.txt"
IPB4="$BASEURL/reestr_ipban4.txt"
IPB6="$BASEURL/reestr_ipban6.txt"
dl()
{
# $1 - url
# $2 - file
curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
# $3 - minsize
# $4 - maxsize
curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
{
echo list download failed : $1
exit 2
}
dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 32768; then
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
exit 2
fi
@ -31,8 +34,14 @@ dl()
}
getuser && {
[ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
[ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
[ "$DISABLE_IPV4" != "1" ] && {
dl "$URL4" "$ZIPLIST" 32768 4194304
dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
}
[ "$DISABLE_IPV6" != "1" ] && {
dl "$URL6" "$ZIPLIST6" 8192 4194304
dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
}
}
"$IPSET_DIR/create_ipset.sh"

21
ipset/get_reestr_preresolved_smart.sh
View File

@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
TMPLIST="$TMPDIR/list_nethub.txt"
TMPLIST="$TMPDIR/list.txt"
BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
URL4="$BASEURL/reestr_smart4.txt"
URL6="$BASEURL/reestr_smart6.txt"
IPB4="$BASEURL/reestr_ipban4.txt"
IPB6="$BASEURL/reestr_ipban6.txt"
dl()
{
# $1 - url
# $2 - file
curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
# $3 - minsize
# $4 - maxsize
curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
{
echo list download failed : $1
exit 2
}
dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 32768; then
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
exit 2
fi
@ -31,8 +34,14 @@ dl()
}
getuser && {
[ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
[ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
[ "$DISABLE_IPV4" != "1" ] && {
dl "$URL4" "$ZIPLIST" 32768 4194304
dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
}
[ "$DISABLE_IPV6" != "1" ] && {
dl "$URL6" "$ZIPLIST6" 8192 4194304
dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
}
}
"$IPSET_DIR/create_ipset.sh"

16
ipset/get_reestr_resolvable_domains.sh
View File

@ -9,18 +9,22 @@ TMPLIST="$TMPDIR/list_nethub.txt"
BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
URL="$BASEURL/reestr_hostname_resolvable.txt"
IPB4="$BASEURL/reestr_ipban4.txt"
IPB6="$BASEURL/reestr_ipban6.txt"
dl()
{
# $1 - url
# $2 - file
curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
# $3 - minsize
# $4 - maxsize
curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
{
echo list download failed : $1
exit 2
}
dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 65536; then
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
exit 2
fi
@ -28,8 +32,14 @@ dl()
rm -f "$TMPLIST"
}
dl "$URL" "$ZHOSTLIST"
dl "$URL" "$ZHOSTLIST" 65536 67108864
hup_zapret_daemons
[ "$DISABLE_IPV4" != "1" ] && dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
[ "$DISABLE_IPV6" != "1" ] && dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
getipban
"$IPSET_DIR/create_ipset.sh"
exit 0

50
ipset/get_reestr_resolve.sh
View File

@ -7,28 +7,56 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
ZREESTR="$TMPDIR/zapret.txt"
ZDIG="$TMPDIR/zapret-dig.txt"
IPB="$TMPDIR/ipb.txt"
ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
#ZURL=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
dl_checked()
{
# $1 - url
# $2 - file
# $3 - minsize
# $4 - maxsize
# $5 - maxtime
curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
{
echo list download failed : $1
return 2
}
dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
return 2
fi
return 0
}
reestr_list()
{
LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p'
}
reestr_extract_ip()
{
LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
}
getuser && {
# both disabled
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
{
echo reestr list download failed
exit 2
}
dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || exit 2
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
exit 2
fi
echo preparing ipban list ..
reestr_extract_ip <"$ZREESTR" >"$IPB"
$AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
$AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
rm -f "$IPB"
echo preparing dig list ..
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
reestr_list | sort -u >"$ZDIG"
rm -f "$ZREESTR"
echo digging started. this can take long ...