mirror of
https://github.com/borekon/Blocklists.git
synced 2025-06-08 14:34:25 +02:00
Update blocklist-update.sh
Update from LEXO blog with minor changes
This commit is contained in:
Alfonso
GitHub
parent
73ffeee0d1
commit
652f1de07a
|
|
@ -1,128 +1,108 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
##
|
# The download path to the file which contains all the IP addresses
|
||||||
## Configuration
|
TO_DOWNLOAD="https://lists.blocklist.de/lists/all.txt"
|
||||||
##
|
|
||||||
|
|
||||||
# Files which should be downloaded
|
|
||||||
TO_DOWNLOAD[0]="http://lists.blocklist.de/lists/ftp.txt"
|
|
||||||
#TO_DOWNLOAD[1]="http://lists.blocklist.de/lists/bots.txt"
|
|
||||||
#TO_DOWNLOAD[2]="http://lists.blocklist.de/lists/ssh.txt"
|
|
||||||
TO_DOWNLOAD[1]="http://lists.blocklist.de/lists/bruteforcelogin.txt"
|
|
||||||
TO_DOWNLOAD[2]="http://lists.blocklist.de/lists/apache.txt"
|
|
||||||
|
|
||||||
# Other settings; Edit if necesarry
|
# Other settings; Edit if necesarry
|
||||||
CHAINNAME="blocklist-de"
|
CHAINNAME="blocklist-de"
|
||||||
ACTION="REJECT" # Can be DROP
|
ACTION="DROP" # Can be DROP or REJECT
|
||||||
PRINT_REPORT=1
|
|
||||||
IPTABLES_PATH="/sbin/iptables"
|
IPTABLES_PATH="/sbin/iptables"
|
||||||
|
IPSET_PATH="/sbin/ipset"
|
||||||
|
SORT_PATH="/usr/bin/sort"
|
||||||
|
MAIL_PATH="/usr/bin/mail"
|
||||||
|
GREP_PATH="/bin/grep"
|
||||||
|
|
||||||
########## Do not edit anything below this line ##########
|
if [ -z $IPTABLES_PATH ]; then echo "Cannot find [ iptables ]. Is it installed? Exiting"; exit 1; fi;
|
||||||
|
if [ -z $IPSET_PATH ]; then echo "Cannot find [ ipset ]. Is it installed? Exiting"; exit 1; fi;
|
||||||
|
if [ -z $SORT_PATH ]; then echo "Cannot find [ sort ]. Is it installed? Exiting"; exit 1; fi;
|
||||||
|
if [ -z $MAIL_PATH ]; then echo "Cannot find [ mail ]. Is it installed? Exiting"; exit 1; fi;
|
||||||
|
if [ -z $GREP_PATH ]; then echo "Cannot find [ grep ]. Is it installed? Exiting"; exit 1; fi;
|
||||||
|
|
||||||
#
|
# E-Mail variables
|
||||||
## Needed variables
|
MAILLOG="/var/log/blocklist-update.log"
|
||||||
#
|
MAIL_SENDER="backupinfo" #this defines a system-user without a shell or password. It's used as the e-mail sender name. You can create one like this: useradd -M -N -s /usr/sbin/nologin myuser && passwd -d myuser
|
||||||
started=`date`
|
MAIL_SUBJECT="ERROR - IP blocklist script failed to download the IP set"
|
||||||
version="1.0.0"
|
MAIL_RECIPIENTS="mail-recipient@yourdomain.tld" #send mail to multiple receipients by overgiving a space-seperated address list
|
||||||
amountDownloaded=0
|
|
||||||
amountAfterSortAndUnique=0
|
|
||||||
amountInserted=0
|
|
||||||
amountDeleted=-1
|
|
||||||
|
|
||||||
fileUnfiltered="/tmp/blocklist-ips-unfiltered.txt"
|
BLOCKLIST_FILE="/tmp/ip-blocklist.txt"
|
||||||
fileFiltered="/tmp/blocklist-ips-filtered.txt"
|
BLOCKLIST_TMP_FILE="/tmp/ip-blocklist.txt.tmp"
|
||||||
|
|
||||||
#
|
# Create a new MAILLOG from scratch. Do it the very simplest way possible
|
||||||
## Download every file and concat to one file
|
rm -f $MAILLOG
|
||||||
#
|
touch $MAILLOG
|
||||||
for currentFile in "${TO_DOWNLOAD[@]}"
|
|
||||||
do
|
|
||||||
wget -qO - $currentFile >> $fileUnfiltered
|
|
||||||
done
|
|
||||||
|
|
||||||
#
|
echo "" >>$MAILLOG
|
||||||
## Sort and filter
|
echo "Downloading the most recent IP list from $TO_DOWNLOAD ..." >>$MAILLOG
|
||||||
#
|
wgetOK=$(wget -qO - $TO_DOWNLOAD >> $BLOCKLIST_FILE) >>$MAILLOG 2>&1
|
||||||
cat $fileUnfiltered | sort | uniq > $fileFiltered
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "Most recent IP blocklist could not be downloaded from $TO_DOWNLOAD" >>$MAILLOG
|
||||||
|
echo "Please check manually. The script calling this function: $0" >>$MAILLOG
|
||||||
|
echo "You can download and import the IP list manually like this:" >>$MAILLOG
|
||||||
|
echo "wget -qO - $TO_DOWNLOAD >> /tmp/blocklist-de.txt"
|
||||||
|
echo "for i in $( cat /tmp/blocklist-de.txt ); do ipset add $CHAINNAME $i; done" >>$MAILLOG
|
||||||
|
|
||||||
amountDownloaded=`cat $fileUnfiltered | wc -l`
|
### Sending warning e-mail and cancelling the update process
|
||||||
amountAfterSortAndUnique=`cat $fileFiltered | wc -l`
|
sudo -u $MAIL_SENDER /usr/bin/mail -s "$MAIL_SUBJECT" $MAIL_RECIPIENTS < $MAILLOG
|
||||||
|
|
||||||
#
|
### Exit with error in this case
|
||||||
## Create chain if it does not exist
|
exit 1
|
||||||
#
|
fi
|
||||||
$IPTABLES_PATH --new-chain $CHAINNAME >/dev/null 2>&1
|
|
||||||
|
|
||||||
# Insert rule (if necesarry) into INPUT chain so the chain above will also be used
|
echo "" >>$MAILLOG
|
||||||
|
echo "Parsing the downloaded file and filter out only IPv4 addresses ..." >>$MAILLOG
|
||||||
|
grep -E -o "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" $BLOCKLIST_FILE > $BLOCKLIST_TMP_FILE
|
||||||
|
|
||||||
|
echo "" >>$MAILLOG
|
||||||
|
echo "Removing duplicate IPs from the list ..." >>$MAILLOG
|
||||||
|
sort -u $BLOCKLIST_TMP_FILE -o $BLOCKLIST_FILE >>$MAILLOG 2>&1
|
||||||
|
rm $BLOCKLIST_TMP_FILE
|
||||||
|
|
||||||
|
echo "" >>$MAILLOG
|
||||||
|
echo "Setting up the ipset configuration by creating the '$CHAINNAME' IP set ..." >>$MAILLOG
|
||||||
|
if [ `$IPSET_PATH list | grep "Name: $CHAINNAME" | wc -l` -eq 0 ]
|
||||||
|
then
|
||||||
|
# Create the new ipset set
|
||||||
|
$IPSET_PATH create $CHAINNAME hash:ip maxelem 16777216 >>$MAILLOG 2>&1
|
||||||
|
else
|
||||||
|
echo "ipset configuration already exists - Flushing and recreating the iptables/ipset configuration ..." >>$MAILLOG
|
||||||
|
# Reason: The kernel sometimes did not properly flush the ipset list which caused errors. Thus we remove the whole list and recreate it from scatch
|
||||||
|
$IPTABLES_PATH --flush $CHAINNAME >>$MAILLOG 2>&1
|
||||||
|
$IPSET_PATH flush $CHAINNAME >>$MAILLOG 2>&1
|
||||||
|
$IPSET_PATH destroy $CHAINNAME >>$MAILLOG 2>&1
|
||||||
|
$IPSET_PATH create $CHAINNAME hash:ip maxelem 16777216 >>$MAILLOG 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "" >>$MAILLOG
|
||||||
|
echo "Setting up the $CHAINNAME chain on iptables, if required..." >>$MAILLOG
|
||||||
|
if [ `$IPTABLES_PATH -L -n | grep "Chain $CHAINNAME" | wc -l` -eq 0 ]
|
||||||
|
then
|
||||||
|
# Create the iptables chain
|
||||||
|
$IPTABLES_PATH --new-chain $CHAINNAME >>$MAILLOG 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "" >>$MAILLOG
|
||||||
|
echo "Inserting the new chain $CHAINNAME into iptables INPUT, if required" >>$MAILLOG
|
||||||
|
# Insert rule (if necesarry) into the INPUT chain so the chain above will also be used
|
||||||
if [ `$IPTABLES_PATH -L INPUT | grep $CHAINNAME | wc -l` -eq 0 ]
|
if [ `$IPTABLES_PATH -L INPUT | grep $CHAINNAME | wc -l` -eq 0 ]
|
||||||
then
|
then
|
||||||
|
# Insert rule because it is not present
|
||||||
# Insert rule because it is not present
|
$IPTABLES_PATH -I INPUT -j $CHAINNAME >>$MAILLOG 2>&1
|
||||||
$IPTABLES_PATH -I INPUT -j $CHAINNAME
|
|
||||||
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#
|
# Create rule (if necesarry) into the $CHAINNAME
|
||||||
## Insert all IPs from the downloaded list if there is no rule stored
|
echo "" >>$MAILLOG
|
||||||
#
|
echo "Creating the firewall rule, if required..." >>$MAILLOG
|
||||||
while read currentIP
|
if [ `$IPTABLES_PATH -L $CHAINNAME | grep REJECT | wc -l` -eq 0 ]
|
||||||
do
|
|
||||||
|
|
||||||
# Check via command
|
|
||||||
$IPTABLES_PATH -C $CHAINNAME -s $currentIP -j $ACTION >/dev/null 2>&1
|
|
||||||
|
|
||||||
# Now we have to check the exit code of iptables via $?
|
|
||||||
#
|
|
||||||
# 0 = rule exists and don't has to be stored again
|
|
||||||
# 1 = rule does not exist and has to be stored
|
|
||||||
|
|
||||||
if [ $? -eq 1 ]
|
|
||||||
then
|
|
||||||
|
|
||||||
# Append the IP
|
|
||||||
$IPTABLES_PATH -A $CHAINNAME -s $currentIP -j $ACTION >/dev/null 2>&1
|
|
||||||
|
|
||||||
# Increment the counter
|
|
||||||
amountInserted=$((amountInserted + 1))
|
|
||||||
|
|
||||||
fi
|
|
||||||
|
|
||||||
done < $fileFiltered
|
|
||||||
|
|
||||||
## Now we delete the IPs which are stored in iptables but not anymore in the list
|
|
||||||
while read currentIP
|
|
||||||
do
|
|
||||||
# Check if the ip is in the downloaded list
|
|
||||||
if [ `cat $fileFiltered | grep $currentIP | wc -l` -eq 0 ]
|
|
||||||
then
|
|
||||||
# Delete the rule by its rulenumber
|
|
||||||
# Because changing the action would result in errors
|
|
||||||
$IPTABLES_PATH -D $CHAINNAME -s $currentIP -j $ACTION >/dev/null 2>&1
|
|
||||||
|
|
||||||
# Increment the counter
|
|
||||||
amountDeleted=$((amountDeleted + 1))
|
|
||||||
|
|
||||||
fi
|
|
||||||
|
|
||||||
done <<< "`$IPTABLES_PATH -n -L blocklist-de | awk '{print $4}'`"
|
|
||||||
|
|
||||||
## Print report
|
|
||||||
if [ $PRINT_REPORT -eq 1 ]
|
|
||||||
then
|
then
|
||||||
echo "--- Blockliste.de :: Update-Report"
|
# Create the one and only firewall rule
|
||||||
echo ""
|
$IPTABLES_PATH -I $CHAINNAME -m set --match-set $CHAINNAME src -j $ACTION >>$MAILLOG 2>&1
|
||||||
echo "Script Version: $version"
|
|
||||||
echo "Started: $started"
|
|
||||||
echo "Finished: `date`"
|
|
||||||
echo ""
|
|
||||||
echo "--> Downloaded IPs: $amountDownloaded"
|
|
||||||
echo "--> Unique IPs: $amountAfterSortAndUnique"
|
|
||||||
echo "--> Inserted: $amountInserted"
|
|
||||||
echo "--> Deleted: $amountDeleted"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#
|
## Read all IPs from the downloaded IP list and fill up the ipset filter set
|
||||||
## Cleanup
|
echo "" >>$MAILLOG
|
||||||
#
|
echo "Importing the IP list into the IP set..." >>$MAILLOG
|
||||||
rm -f /tmp/blocklist-ips-unfiltered.txt
|
for i in $( cat $BLOCKLIST_FILE ); do $IPSET_PATH add $CHAINNAME $i >>$MAILLOG 2>&1; done
|
||||||
rm -f /tmp/blocklist-ips-filtered.txt
|
|
||||||
|
echo "" >>$MAILLOG
|
||||||
|
echo "Done." >>$MAILLOG
|
||||||
|
sudo -u $MAIL_SENDER $MAIL_PATH -s "SUCCESS - IP blocklist script has updated the IP set with the newest IP list" $MAIL_RECIPIENTS < $MAILLOG
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user