From 50616896c8d39536dede718aadbd26e2a8c1012b Mon Sep 17 00:00:00 2001
From: bol-van <none@none.none>
Date: Sun, 22 Dec 2024 12:50:36 +0300
Subject: [PATCH] ipset: scripts maintenance

---
 ipset/get_reestr_hostlist.sh  | 8 ++++----
 ipset/get_reestr_resolve.sh   | 8 ++++----
 ipset/get_refilter_domains.sh | 2 +-
 ipset/get_refilter_ipsum.sh   | 2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ipset/get_reestr_hostlist.sh b/ipset/get_reestr_hostlist.sh
index 2c7450d..afa5d9b 100755
--- a/ipset/get_reestr_hostlist.sh
+++ b/ipset/get_reestr_hostlist.sh
@@ -5,9 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-ZREESTR="$TMPDIR/zapret.txt"
+ZREESTR="$TMPDIR/zapret.txt.gz"
 IPB="$TMPDIR/ipb.txt"
-ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
 
 dl_checked()
 {
@@ -31,11 +31,11 @@ dl_checked()
 
 reestr_list()
 {
- LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
+ LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
 reestr_extract_ip()
 {
- LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+ LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
 }
 
 ipban_fin()
diff --git a/ipset/get_reestr_resolve.sh b/ipset/get_reestr_resolve.sh
index 924a073..41adb72 100755
--- a/ipset/get_reestr_resolve.sh
+++ b/ipset/get_reestr_resolve.sh
@@ -5,12 +5,12 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-ZREESTR="$TMPDIR/zapret.txt"
+ZREESTR="$TMPDIR/zapret.txt.gz"
 ZDIG="$TMPDIR/zapret-dig.txt"
 IPB="$TMPDIR/ipb.txt"
 ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
-ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
 
 dl_checked()
 {
@@ -34,11 +34,11 @@ dl_checked()
 
 reestr_list()
 {
- LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p'
+ LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
 reestr_extract_ip()
 {
- LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+ LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
 }
 
 getuser && {
diff --git a/ipset/get_refilter_domains.sh b/ipset/get_refilter_domains.sh
index 88cb7fd..807c025 100755
--- a/ipset/get_refilter_domains.sh
+++ b/ipset/get_refilter_domains.sh
@@ -25,7 +25,7 @@ dl()
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
-  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  zzcat "$TMPLIST" | zz "$2"
   rm -f "$TMPLIST"
 }
 
diff --git a/ipset/get_refilter_ipsum.sh b/ipset/get_refilter_ipsum.sh
index 811d9ba..f5dd192 100755
--- a/ipset/get_refilter_ipsum.sh
+++ b/ipset/get_refilter_ipsum.sh
@@ -26,7 +26,7 @@ dl()
    exit 2
   fi
   # remove DOS EOL \r
-  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  zzcat "$TMPLIST" | zz "$2"
   rm -f "$TMPLIST"
 }