diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..84b8cb2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +MIT License + +Portions Copyright (c) 2018 Intellex +Portions Copyright (c) 2020 Oliver Mueller +Portions Copyright (c) 2020 Andrej Walilko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 05c7f72..abb47f1 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ # blcheck -A powerful script for testing a domain or an IP against mailing black lists. +A powerful script for testing a domain or an IP against mailing block lists and allow lists. Script will use dig if it is found. If dig is not found script will use host. Features -------------------- -* More then __100 black lists__ already included! +* More then __280 block lists__ already included! * Automatic distinction between __domain or IP__ * Performs __PTR validation__ (only if domain is supplied, does not work for IP) * 3 verbose (-v) levels and a quiet (-q) mode -* The result of script is the number of servers which blacklisted the domain, so it can be used for any kind of __automated scripts or cronjobs__ +* The exit code of script is the number of servers which blocklisted the domain, so it can be used for any kind of __automated scripts or cronjobs__ * Informative and pleasant output @@ -32,50 +32,29 @@ If the IP is supplied, the PTR check cannot be executed and will be skipped. 
 -d dnshost  Use host as DNS server to make lookups
--l file     Load blacklists from file, separated by space or new line
--c          Warn if the top level domain of the blacklist has expired
+-l file     Load blocklists from file, separated by space or new line
+-c          Warn if the top level domain of the blocklist has expired
 -v          Verbose mode, can be used multiple times (up to -vvv)
 -q          Quiet modem with absolutely no output (useful for scripts)
--p          Plain text output (no coloring, no interactive status)
+-p          Plain text output (no coloring, no interactive status, useful
+            for tee'd output to a text file)
+-j num	    The number of parallel processes to use (default is 200%, or
+	    2x system core count. See `man parallel` for details.)
 -h          The help you are just reading
-Result of the script is the number of blacklisted entries. So if the supplied -IP is not blacklisted on any of the servers the result is 0. +Exit code of the script is the number of blocklisted entries. So if the supplied +IP is not blocklisted on any of the servers the exit code is 0. TODO -------------------- 1. Handle domains with multiple DNS entries. -Licence --------------------- -MIT License - -Copyright (c) 2018 Intellex - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - Credits -------------------- Script has been written by the [Intellex](http://intellex.rs/en) team. -Contributors: +Additional contributors: [Darko Poljak](https://github.com/darko-poljak) - - + [Oliver Mueller](https://github.com/ogmueller) + [Andrej Walilko](https://github.com/ch604) diff --git a/blcheck b/blcheck old mode 100755 new mode 100644 index 02bdb59..f3210cd --- a/blcheck +++ b/blcheck @@ -1,165 +1,352 @@ #!/bin/bash -# -# MIT License -# -# Copyright (c) Intellex 2015 -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# +# shellcheck disable=SC2317 + # ============================================================================= # # title : blcheck -# description : Test any domain against more then 100 black lists. +# description : Test any domain against more than 100 blocklists. # author : Intellex -# contributors : Darko Poljak -# date : 2016-03-18 -# version : 0.6.0 +# contributors : Darko Poljak, Oliver Mueller, Andrej Walilko +# date : May 28 2025 +# version : 0.8.0 # usage : blcheck [options] +# reference : http://multirbl.valli.org/list/ +# code style : https://google.github.io/styleguide/shell.xml +# url : github.com/ch604/blcheck # # ============================================================================= # Config { + # where to log blocklist hits permanently if there are any + # this file will be appended, not clobbered. + # format is ${TARGET}\n$blocklist1\n$blocklist2, etc + output=~/blcheck_blocklists.txt + # how many dig/host processes to run in parallel + # this can be a number of threads (6) + # or a percentage of cpu cores (75%) + j="200%" # How many tries and for how long to wait for DNS queries CONF_DNS_TRIES=2 CONF_DNS_DURATION=3 - # Blacklists to check - CONF_BLACKLISTS=" - 0spam-killlist.fusionzero.com + # blocklists to check + # Example formats: + # - General format + # {ip_or_domain}={filter}#{type} + # - IP based blocklist (default DNSBL, any valid return code will match) + # block.list.com + # - IP based blocklist + # block.list.com#DNSBL + # - IP based allowlist + # allow.list.com#DNSWL + # - URI based blocklist + # uri.list.com#URIBL + # - Regexp defined return code, only a match will qualify for list. Here it is 127.0.0.2 and 127.0.0.3 + # block.list.com=127\.0\.0\.[23] + # - Regexp defined return code for a URI based blocklist + # block.list.com=127\.0\.[0-9]+\.0#URIBL + CONF_BLOCKLISTS=" + whitelist.rbl.ispa.at#DNSWL + t3direct.dnsbl.net.au + ucepn.dnsbl.net.au + wpbl.dnsbl.net.au + bl.reynolds.net.au + bl.mav.com.br + openproxy.bls.digibase.ca + proxyabuse.bls.digibase.ca + spambot.bls.digibase.ca + probes.dnsbl.net.auproxy.bl.gweep.ca + rbl2.triumf.ca + wbl.triumf.ca#DNSWL + ipbl.zeustracker.abuse.ch + uribl.zeustracker.abuse.ch + spamrbl.imp.ch + wormrbl.imp.ch + rbl.lugh.ch + dnsrbl.swinog.ch + uribl.swinog.ch + blacklist.woody.ch + ipv6.blacklist.woody.ch + uri.blacklist.woody.ch + block.ascams.com + superblock.ascams.com + rbl.blockedservers.com + netscan.rbl.blockedservers.com + spam.rbl.blockedservers.com + list.blogspambl.com + dnsbl1.dnsbl.borderware.com + dnsbl2.dnsbl.borderware.com + dnsbl3.dnsbl.borderware.com + dul.dnsbl.borderware.com + csi.cloudmark.com + dnsbl.cobion.com + bogons.cymru.com + v4.fullbogons.cymru.com + v6.fullbogons.cymru.com + rbl.dns-servicios.com + 0outspam.fusionzero.com 0spam.fusionzero.com - access.redhawk.org - all.rbl.jp - all.spam-rbl.fr + 0spam-n.fusionzero.com + 0spamtrust.fusionzero.com + 0spamurl.fusionzero.com + accredit.habeas.com + hil.habeas.com + hul.habeas.com + sa-accredit.habeas.com + sohul.habeas.com + blacklist.hostkarma.com + iadb.isipp.com + iadb2.isipp.com + iddb.isipp.com + wadb.isipp.com + black.junkemailfilter.com + hostkarma.junkemailfilter.com + nobl.junkemailfilter.com + krn.korumail.com + q.mail-abuse.com + r.mail-abuse.com + cidr.bl.mcafee.com + dnsbl.forefront.microsoft.com + bl.mipspace.com + bl.nordspam.com + dbl.nordspam.com#URIBL + safe.dnsbl.prs.proofpoint.com + rbl.realtimeblacklist.com + mailsl.dnsbl.rjek.com + urlsl.dnsbl.rjek.com + dynip.rothen.com + blackholes.scconsult.com + reputation-domain.rbl.scrolloutf1.com + reputation-ip.rbl.scrolloutf1.com + reputation-ns.rbl.scrolloutf1.com + bl.score.senderscore.com + feb.spamlab.com + rbl.spamlab.com all.spamrats.com - aspews.ext.sorbs.net - b.barracudacentral.org + auth.spamrats.com + dyna.spamrats.com + noptr.spamrats.com + spam.spamrats.com + rbl.suresupport.com + psbl.surriel.com + whitelist.surriel.com#DNSWL + bl.tiopan.com + dbl.tiopan.com#URIBL + ubl.unsubscore.com + spam.dnsbl.anonmails.de + bl.blocklist.de + dnsbl.darklist.de + dnsbl.inps.de + dnswl.inps.de#DNSWL + admin.bl.kundenserver.de + relays.bl.kundenserver.de + schizo-bl.kundenserver.de + spamblock.kundenserver.de + worms-bl.kundenserver.de + dnsbl.madavi.de + torserver.tor.dnsbl.sectoor.de + dunk.dnsbl.tuxad.de + hartkore.dnsbl.tuxad.de + spamsources.fabel.dk + st.technovision.dk + dnsbl.abyan.es + eswlrev.dnsbl.rediris.es#DNSWL + mtawlrev.dnsbl.rediris.es#DNSWL + dnsbl.isx.fr + all.spam-rbl.fr + pofon.foobar.hu + ispmx.pofon.foobar.hu + uribl.pofon.foobar.hu + singular.ttk.pte.hu + blacklist.netcore.co.in + dnsbl.rv-soft.info + db.wpbl.info + rbl.jp + spamlist.or.kr + bl.fmb.la + communicado.fmb.la + nsbl.fmb.la + sa.fmb.la + short.fmb.la + dnsbl.anticaptcha.net + dnsbl6.anticaptcha.net + blacklist.mail.ops.asp.att.net + blacklist.sequoia.ops.asp.att.net + blacklist.mailrelay.att.net + rbl.blakjak.net + dul.blackhole.cantv.net + hog.blackhole.cantv.net + rhsbl.blackhole.cantv.net + rot.blackhole.cantv.net + spam.blackhole.cantv.net + rbl.choon.net + ipv6.rbl.choon.net + rwl.choon.net + ipv6.rwl.choon.net + dnsbl.cyberlogic.net + fnrbl.fast.net + truncate.gbudb.net + rbl.interserver.net + rbl.iprange.net + dnsbl.kempt.net + spamguard.leadmon.net + niprbl.mailcleaner.net + uribl.mailcleaner.net + dnsbl.mailshell.net + bl.mailspike.net + rep.mailspike.net + wl.mailspike.net#DNSWL + z.mailspike.net + combined.rbl.msrbl.net + images.rbl.msrbl.net + phishing.rbl.msrbl.net + spam.rbl.msrbl.net + virus.rbl.msrbl.net + web.rbl.msrbl.net + relays.nether.net + trusted.nether.net + unsure.nether.net + dul.pacifier.net + all.s5h.net + bl.scientificspam.net + rhsbl.scientificspam.net + korea.services.net + bl.spamcop.net backscatter.spameatingmonkey.net - badnets.spameatingmonkey.net - bb.barracudacentral.org - bl.drmx.org - bl.konstant.no - bl.nszones.com - bl.spamcannibal.org bl.spameatingmonkey.net - bl.spamstinks.com - black.junkemailfilter.com - blackholes.five-ten-sg.com - blacklist.sci.kun.nl - blacklist.woody.ch - bogons.cymru.com - bsb.empty.us + fresh.spameatingmonkey.net + fresh10.spameatingmonkey.net + fresh15.spameatingmonkey.net + fresh30.spameatingmonkey.net + freshzero.spameatingmonkey.net + bl.ipv6.spameatingmonkey.net + netbl.spameatingmonkey.net + uribl.spameatingmonkey.net + urired.spameatingmonkey.net bsb.spamlookup.net - cart00ney.surriel.com - cbl.abuseat.org - cbl.anti-spam.org.cn - cblless.anti-spam.org.cn - cblplus.anti-spam.org.cn - cdl.anti-spam.org.cn - cidr.bl.mcafee.com - combined.rbl.msrbl.net - db.wpbl.info - dev.null.dk - dialups.visi.com + dnsbl.spfbl.net + score.spfbl.net + dnswl.spfbl.net#DNSWL + bl.suomispam.net + dbl.suomispam.net#URIBL + gl.suomispam.net + dob.sibl.support-intelligence.net + srn.surgate.net + srnblack.surgate.net + rbl.tdk.net dnsbl-0.uceprotect.net dnsbl-1.uceprotect.net dnsbl-2.uceprotect.net dnsbl-3.uceprotect.net - dnsbl.anticaptcha.net - dnsbl.aspnet.hu - dnsbl.inps.de - dnsbl.justspam.org - dnsbl.kempt.net - dnsbl.madavi.de - dnsbl.rizon.net - dnsbl.rv-soft.info - dnsbl.rymsho.ru - dnsbl.sorbs.net dnsbl.zapbl.net - dnsrbl.swinog.ch - dul.pacifier.net - dyn.nszones.com - dyna.spamrats.com - fnrbl.fast.net - fresh.spameatingmonkey.net - hostkarma.junkemailfilter.com - images.rbl.msrbl.net + rhsbl.zapbl.net + rbl.zenon.net + dnsbl.beetjevreemd.nl + bitonly.dnsbl.bit.nl + virbl.bit.nl + blacklist.sci.kun.nl + whitelist.sci.kun.nl#DNSWL + blackholes.tepucom.nl + bl.konstant.no + bl.0spam.org + nbl.0spam.org + url.0spam.org + orvedb.aupads.org + rsbl.aupads.org ips.backscatterer.org - ix.dnsbl.manitu.net - korea.services.net - l2.bbfh.ext.sorbs.net - l3.bbfh.ext.sorbs.net - l4.bbfh.ext.sorbs.net + b.barracudacentral.org + bb.barracudacentral.org list.bbfh.org - list.blogspambl.com - mail-abuse.blacklist.jippg.org - netbl.spameatingmonkey.net - netscan.rbl.blockedservers.com - no-more-funn.moensted.dk - noptr.spamrats.com - orvedb.aupads.org - pbl.spamhaus.org - phishing.rbl.msrbl.net - pofon.foobar.hu - psbl.surriel.com - rbl.abuse.ro - rbl.blockedservers.com - rbl.dns-servicios.com + plus.bondedsender.org + query.bondedsender.org + dnsblchile.org + dnsrbl.org + dwl.dnswl.org#DNSWL + list.dnswl.org#DNSWL + bl.drmx.org + dnsbl.dronebl.org rbl.efnet.org + tor.efnet.org rbl.efnetrbl.org - rbl.iprange.net + dnsbl.rangers.eu.org + dnsbl.httpbl.org + mail-abuse.blacklist.jippg.org + dnsbl.justspam.org + bl.mailspike.org + bl.nosolicitado.org + bl.worst.nosolicitado.org + dnsbl.openresolvers.org + spam.pedantic.org + access.redhawk.org + abuse.rfc-clueless.org + bogusmx.rfc-clueless.org + dsn.rfc-clueless.org + elitist.rfc-clueless.org + fulldom.rfc-clueless.org + postmaster.rfc-clueless.org + whois.rfc-clueless.org + public.sarbl.org rbl.schulte.org - rbl.talkactive.net - rbl2.triumf.ca - rsbl.aupads.org - sbl-xbl.spamhaus.org - sbl.nszones.com + sbl.spamdown.org + dbl.spamhaus.org#URIBL + vouch.dwl.spamhaus.org#DNSWL + pbl.spamhaus.org sbl.spamhaus.org - short.rbl.jp - spam.dnsbl.anonmails.de - spam.pedantic.org - spam.rbl.blockedservers.com - spam.rbl.msrbl.net - spam.spamrats.com - spamrbl.imp.ch - spamsources.fabel.dk - st.technovision.dk - tor.dan.me.uk - tor.dnsbl.sectoor.de - tor.efnet.org - torexit.dan.me.uk - truncate.gbudb.net - ubl.unsubscore.com - uribl.spameatingmonkey.net - urired.spameatingmonkey.net - virbl.dnsbl.bit.nl - virus.rbl.jp - virus.rbl.msrbl.net + sbl-xbl.spamhaus.org + swl.spamhaus.org#DNSWL + xbl.spamhaus.org + zen.spamhaus.org + abuse.surbl.org + cr.surbl.org + multi.surbl.org + mw.surbl.org + dnsbl.tornevall.org + opm.tornevall.org + free.v4bl.org + ip.v4bl.org + ips.whitelisted.org#DNSWL + dnsbl.calivent.com.pe + forbidden.icm.edu.pl + dyn.rbl.polspam.pl + lblip4.rbl.polspam.pl + lblip6.rbl.polspam.pl + rblip4.rbl.polspam.pl + rblip6.rbl.polspam.pl + rhsbl.rbl.polspam.pl + rbl.abuse.ro + uribl.abuse.ro vote.drbl.caravan.ru - vote.drbl.gremlin.ru - web.rbl.msrbl.net work.drbl.caravan.ru + vote.drbl.gremlin.ru work.drbl.gremlin.ru - wormrbl.imp.ch - xbl.spamhaus.org - zen.spamhaus.org" + dul.dnsbl.sorbs.netdul.ru + bl.shlink.orgdul.ru + dnsbl.rymsho.ru + rhsbl.rymsho.ru + netblockbl.spamgrouper.to + dnsbl.mcu.edu.tw + dnsbl.net.ua + rbl.fasthosts.co.uk + torexit.dan.me.uk + bsb.empty.us + 88.blocklist.zap" + + + # reputation/scoring pages + # score.senderscore.com + + # shellcheck disable=SC2034 + CONF_BLOCKLISTS_TEST=" + rbl.rbldns.ru + rbl.rbldns.ru=127\.0\.0\.1 + rbl.rbldns.ru#DNSWL + black.list.com=127.0.[0-9]+.0#URIBL + dbl.spamhaus.org#URIBL + t3direct.dnsbl.net.au=127\.0\.0\.1#DNSWL + t3direct.dnsbl.net.au#BLA + t3direct.dnsbl.net.au%127\.0\.0\.1#DNSWL" #~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ } @@ -171,6 +358,7 @@ REGEX_IP='\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)\.\([0-9]\{1,3\}\)' REGEX_DOMAIN='\([a-zA-Z0-9]\+\(-[a-zA-Z0-9]\+\)*\.\)\+[a-zA-Z]\{2,\}' REGEX_TDL='\([a-zA-Z0-9]\+\(-[a-zA-Z0-9]\+\)*\.\)[a-zA-Z]\{2,\}$' + REGEX_LIST='^[ \\t]*([^=#]+)(=(([^#])+))?(#(DNSBL|DNSWL|URIBL))?[ \\t]*$' # Colors if [[ $- == *i* ]]; then @@ -187,17 +375,8 @@ # Define spinner SPINNER="-\|/" - #SPINNER=".oO@*" - #SPINNER="▉▊▋▌▍▎▏▎▍▌▋▊▉" - #SPINNER="←↖↑↗→↘↓↙" - #SPINNER="▁▂▃▄▅▆▇█▇▆▅▄▃▁" - #SPINNER="▖▘▝▗" - #SPINNER="┤┘┴└├┌┬┐" - #SPINNER="◢◣◤◥" - #SPINNER="◰◳◲◱" - #SPINNER="◴◷◶◵" - #SPINNER="◐◓◑◒" + VERBOSE=0 #~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ } @@ -205,9 +384,8 @@ # Macros { # Verbose printing - VERBOSE=0 info() { - if [ $VERBOSE -ge "$1" ]; then + if [[ ${VERBOSE} -ge "$1" ]]; then echo "$2" fi } @@ -215,303 +393,442 @@ # Error handling error() { echo "ERROR: $1" >&2 - exit 2 + exit 254 } # Show progress + # shellcheck disable=SC2329 progress() { # Bar + local x BAR x=$(($1 % ${#SPINNER} + 1)) - BAR=$(echo $SPINNER | awk "{ print substr(\$0, ${x}, 1) }") - if test -z "$PLAIN"; then + BAR=$(awk "{ print substr(\$0, ${x}, 1) }" <<< "${SPINNER}") + if [[ -z "${PLAIN}" ]]; then printf "\r "; fi # BAR as printf arg so that backslash will be litteraly interpreted - printf "[ %s %3s%% ] checking... %4s / $2 " "$BAR" $(($1 * 100 / $2)) "$1"; + printf "[ %s %3s%% ] checking... %4s / $2 " "${BAR}" $(($1 * 100 / $2)) "$1"; } # Resolve the IP resolve() { # IP already? - IP=$(echo "$1" | grep "^$REGEX_IP$") - if [ "$IP" ]; then - echo "$IP" + local IP + IP=$(grep "^${REGEX_IP}$" <<< "$1") + if [[ "${IP}" ]]; then + echo "${IP}" # Resolve domain else # Handle special resolve types case "$2" in - "ns" ) TYPE="ns"; REGEX="$REGEX_DOMAIN\.$";; - * ) TYPE="a"; REGEX="$REGEX_IP$";; + "ns" ) TYPE="ns"; REGEX="${REGEX_DOMAIN}\.$";; + * ) TYPE="a"; REGEX="${REGEX_IP}$";; esac - case "$CMD" in - $CMD_DIG ) "$CMD" $DNSSERVER +short -t "$TYPE" +time=$CONF_DNS_DURATION +tries=$CONF_DNS_TRIES "$1" | grep -om 1 "$REGEX";; - $CMD_HOST ) "$CMD" -t "$TYPE" -W $CONF_DNS_DURATION -R $CONF_DNS_TRIES "$1" $DNSSERVER | tail -n1 | grep -om 1 "$REGEX";; + # shellcheck disable=SC2086 + case "${CMD}" in + "${CMD_DIG}" ) "${CMD}" ${DNSSERVER} +short -t "${TYPE}" +time=${CONF_DNS_DURATION} +tries=${CONF_DNS_TRIES} "$1" | grep -om 1 "${REGEX}";; + "${CMD_HOST}" ) "${CMD}" -t "${TYPE}" -W ${CONF_DNS_DURATION} -R ${CONF_DNS_TRIES} "$1" ${DNSSERVER} | tail -n1 | grep -om 1 "${REGEX}";; esac fi } - # Load the blacklist from file - loadBlacklists() { + # Result info for IP + # shellcheck disable=SC2329 + text() { + # IP already? + local IP + IP=$(grep "^${REGEX_IP}$" <<< "$1") + if [[ -n "${IP}" ]]; then + echo "${IP}" + + # Resolve domain + else + + local TYPE="txt" + case "${CMD}" in + "${CMD_DIG}" ) "${CMD}" "${DNSSERVER}" +short -t "${TYPE}" +time="${CONF_DNS_DURATION}" +tries="${CONF_DNS_TRIES}" "$1" ;; + "${CMD_HOST}" ) "${CMD}" -t "${TYPE}" -W "${CONF_DNS_DURATION}" -R "${CONF_DNS_TRIES}" "$1" "${DNSSERVER}" | tail -n1 ;; + esac + fi + } + + # Load the blocklist from file + loadBlocklists() { # Make sure the file is readable - if [ ! "$1" ]; then + if [[ -z "$1" ]]; then error "Option -l requires an additional parameter"; - elif [ ! -r $1 ]; then + elif [[ ! -r $1 ]]; then error "File $1 cannot be opened for reading, make sure it exists and that you have appropriate privileges" fi - CONF_BLACKLISTS=$(cat "$1") + CONF_BLOCKLISTS=$(cat "$1") } # Show help showHelp() { - cat < - -Supplied domain must be full qualified domain name. -If the IP is supplied, the PTR check cannot be executed and will be skipped. - --d dnshost Use host as DNS server to make lookups --l file Load blacklists from file, separated by space or new line --c Warn if the top level domain of the blacklist has expired --v Verbose mode, can be used multiple times (up to -vvv) --q Quiet modem with absolutely no output (useful for scripts) --p Plain text output (no coloring, no interactive status) --h The help you are just reading - -Result of the script is the number of blacklisted entries. So if the supplied -IP is not blacklisted on any of the servers the result is 0. - -HELP + cat <<-HELP + blcheck [options] + + Supplied domain must be full qualified domain name. + If the IP is supplied, the PTR check cannot be executed and will be skipped. + + -d dnshost Use host as DNS server to make lookups + -l file Load blocklists from file, separated by space or new line + -c Warn if the top level domain of the blocklist has expired + -v Verbose mode, can be used multiple times (up to -vvv) + -q Quiet mode with absolutely no output (useful for scripts) + -p Plain text output (no coloring, no interactive status, useful + for tee'd output to a text file) + -j num The number of parallel processes to use (default is 200%, or + 2x system core count. See \`man parallel\` for details.) + -h The help you are just reading + + Exit code of the script is the number of blocklisted entries. So if the supplied + IP is not blocklisted on any of the servers the exit code is 0. If there are five + blocklist hits, then the exit code is 5. + + HELP exit; } #~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ } # Parse the params -while getopts :vqphcl:d: arg; do - case "$arg" in - d) DNSSERVER=$OPTARG;; - l) loadBlacklists $OPTARG;; +while getopts :vqphcj:l:d: arg; do + case "${arg}" in + d) DNSSERVER="${OPTARG}";; + l) loadBlocklists "${OPTARG}";; c) VERIFY_BL=TRUE;; - v) VERBOSE=$(( (VERBOSE + 1) % 4));; + v) VERBOSE=$((VERBOSE + 1));; q) VERBOSE=-1;; p) PLAIN=1 RED="" GREEN="" YELLOW="" CLEAR="" ;; h) showHelp;; - ?) error "Unknown option $OPTARG";; + j) j="${OPTARG}";; + ?) error "Unknown option ${OPTARG}";; esac done shift $((OPTIND - 1)) # Get the domain -if [ $# -eq 0 ]; then +if [[ $# -eq 0 ]]; then echo "Missing target domain or IP." showHelp fi TARGET=$1 +if ! which parallel &> /dev/null; then + echo "Parallel missing, trying to install..." + if which yum &> /dev/null; then + [ ! "$(yum -q provides parallel)" ] && yum -y -q --skip-broken install epel-release + yum -y -q --skip-broken install parallel + elif which dnf &> /dev/null; then + [ ! "$(dnf -q provides parallel)" ] && dnf -y -q --skip-broken install epel-release + dnf -y -q --skip-broken install parallel + else #assumed debian-like + apt-get -qq update + apt-get -y -q install parallel + fi + if which parallel &> /dev/null; then + mkdir ~/.parallel + touch ~/.parallel/will-cite + else + error "Failed to install parallel, exiting." + fi +fi + # Some shells disable parsing backslash in echo statements by default # Set the flag to enable echo to behave consistently across platforms shopt -s xpg_echo # Get the command we will use: dig or host CMD_DIG=$(which dig) -CMD_HOST=$(which host) -if [ "$CMD_DIG" ]; then - if [[ $DNSSERVER ]]; then - DNSSERVER="@$DNSSERVER" +if [[ "${CMD_DIG}" ]]; then + if [[ -n "${DNSSERVER}" ]]; then + DNSSERVER="@${DNSSERVER}" + fi + CMD=${CMD_DIG} +else + CMD_HOST=$(which host) + if [[ "${CMD_HOST}" ]]; then + CMD=${CMD_HOST} fi - CMD=$CMD_DIG -elif [ "$CMD_HOST" ]; then - CMD=$CMD_HOST fi -if [ ! "$CMD" ]; then +if [[ -z "${CMD}" ]]; then error "Either dig or host command is required." fi -info 3 "Using $CMD to reslove DNS queries" +info 3 "Using ${CMD} to reslove DNS queries" # Parse IP -IP=$(resolve "$TARGET") -if [ ! "$IP" ]; then - error "No DNS record found for $TARGET" -elif [ "$IP" != "$TARGET" ]; then - DOMAIN=$TARGET - info 2 "Using $TARGET for target, resolved to $IP" +IP=$(resolve "${TARGET}") +if [[ -z "${IP}" ]]; then + error "No DNS record found for ${TARGET}" +elif [[ "${IP}" != "${TARGET}" ]]; then + DOMAIN=${TARGET} + info 2 "Using ${TARGET} for target, resolved to ${IP}" + TARGET_TYPE="domain" else - info 3 "Using $TARGET for target" + info 3 "Using ${TARGET} for target" + TARGET_TYPE="ip" fi # Reverse the IP -REVERSED=$(echo "$IP" | sed -ne "s~^$REGEX_IP$~\4.\3.\2.\1~p") -info 3 "Using $REVERSED for reversed IP" +REVERSED=$(echo "${IP}" | sed -ne "s~^${REGEX_IP}$~\4.\3.\2.\1~p") +info 3 "Using ${REVERSED} for reversed IP" # Get the PTR info 3 "Checking the PTR record" -case "$CMD" in - $CMD_DIG ) PTR=$("$CMD" $DNSSERVER +short -x "$IP" | sed s/\.$//);; - $CMD_HOST ) PTR=$("$CMD" "$IP" $DNSSERVER | tail -n1 | grep -o '[^ ]\+$' | sed s/\.$//) +# shellcheck disable=SC2086 +case "${CMD}" in + "${CMD_DIG}" ) PTR=$("${CMD}" ${DNSSERVER} +short -x "${IP}" | sed s/\.$//);; + "${CMD_HOST}" ) PTR=$("${CMD}" "${IP}" ${DNSSERVER} | tail -n1 | grep -o '[^ ]\+$' | sed s/\.$//) esac # Validate PTR -if [ ! "$PTR" ]; then +if [[ -z "${PTR}" ]]; then info 0 "${YELLOW}Warning: PTR lookup failed${CLEAR}" else # Match against supplied domain - info 1 "PTR resolves to $PTR" - if [ "$DOMAIN" ]; then - if [ "$DOMAIN" != "$PTR" ]; then - info 0 "${YELLOW}Warning: PTR record does not match supplied domain: $TARGET != $PTR${CLEAR}" + info 1 "PTR resolves to ${PTR}" + if [[ -n "${DOMAIN}" ]]; then + if [[ "${DOMAIN}" != "${PTR}" ]]; then + info 0 "${YELLOW}Warning: PTR record does not match supplied domain: ${TARGET} != ${PTR}${CLEAR}" else - info 1 "${GREEN}PTR record matches supplied domain $PTR${CLEAR}" + info 1 "${GREEN}PTR record matches supplied domain ${PTR}${CLEAR}" fi fi fi -# Filter out the blacklists -BLACKLISTS="" -for BL in $CONF_BLACKLISTS; do - if [ "$BL" ]; then +# Filter out the blocklists +BLOCKLISTS="" +COUNT_DNSBL=0 +COUNT_DNSWL=0 +COUNT_URIBL=0 +COUNT=0 +for BL in ${CONF_BLOCKLISTS}; do + if [[ "${BL}" ]]; then + if [[ ${BL} =~ ${REGEX_LIST} ]]; then + DOMAIN="${BASH_REMATCH[1]}" + if [[ -n "${BASH_REMATCH[3]}" ]]; then + FILTER="${BASH_REMATCH[3]}" + else + FILTER='127\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})' + fi + if [[ -n "${BASH_REMATCH[6]}" ]]; then + TYPE="${BASH_REMATCH[6]}" + else + TYPE='DNSBL' + fi - # Make sure the domain is a proper one - DOMAIN=$(echo "$BL" | sed -e 's/^[ \t]*//' | grep ^"$REGEX_DOMAIN"$) - if [ ! "$DOMAIN" ]; then - info 0 "${YELLOW}Warning: blacklist '$BL' is not valid and will be ignored${CLEAR}" + # Make sure the domain is a proper one + # shellcheck disable=SC2001 + TMP=$(sed -e 's/^[ \t]*//' <<< "${DOMAIN}" | grep ^"${REGEX_DOMAIN}"$) + if [[ -z "${TMP}" ]]; then + info 0 "${YELLOW}Warning: domain '${DOMAIN}' is not valid and will be ignored${CLEAR}" - else - - # It is a proper blacklist - if [ "$BLACKLISTS" ]; then - BLACKLISTS=$(echo "$BLACKLISTS\n$DOMAIN") else - BLACKLISTS="$BL" + if [[ "${TARGET_TYPE}" == 'ip' && "${TYPE}" == 'URIBL' ]]; then + info 0 "${GREEN}Notice: URIBL entry '${DOMAIN}' will be ignore, because ${TARGET} is an IP address${CLEAR}" + else + case "${TYPE}" in + "DNSBL") COUNT_DNSBL=$((COUNT_DNSBL + 1)) ;; + "DNSWL") COUNT_DNSWL=$((COUNT_DNSWL + 1)) ;; + "URIBL") COUNT_URIBL=$((COUNT_URIBL + 1)) ;; + esac + COUNT=$((COUNT + 1)) + + # It is a proper blocklist + if [[ "${BLOCKLISTS}" ]]; then + BLOCKLISTS=$(echo -e "${BLOCKLISTS}\n${DOMAIN}=${FILTER}#${TYPE}") + else + BLOCKLISTS="${DOMAIN}=${FILTER}#${TYPE}" + fi + fi fi + else + info 0 "${YELLOW}Warning: list entry '${BL}' is not valid and will be ignored${CLEAR}" fi fi done -# Make sure we have at least one blacklist -COUNT=$(($(echo "$BLACKLISTS" | wc -l))) -if [ ! "$BLACKLISTS" ] || [ "$COUNT" -eq 0 ]; then - error "No blacklists have been specified" +# Make sure we have at least one blocklist +if [[ "${COUNT}" -eq 0 ]]; then + error "No blocklists have been specified" fi -info 1 "Matching against $COUNT blacklists" +info 1 "Matching against ${COUNT} entries:" +info 1 " - ${COUNT_DNSBL} DNS blocklists" +info 1 " - ${COUNT_DNSWL} DNS allowlists" +info 1 " - ${COUNT_URIBL} URI blocklists" + +# Iterate over all blocklists + +# shellcheck disable=SC2329 +parallel_check() { + local PREFIX= + local I=$1 + local BL=$2 + + # Parse list entry + if [[ ! ${BL} =~ ${REGEX_LIST} ]]; then + error "List entry ${BL} broken" + else + local DOMAIN="${BASH_REMATCH[1]}" + local FILTER="${BASH_REMATCH[3]}" + local TYPE="${BASH_REMATCH[6]}" + fi -# Initialize the counters -INVALID=0 -PASSED=0 -FAILED=0 -# Interate over all blacklists -I=0; -for BL in $BLACKLISTS; do - PREFIX= - I=$((I + 1)) # What should we test - TEST="$REVERSED.$BL." + if [[ "${TYPE}" != "URIBL" ]]; then + # IP based test + local TEST="${REVERSED}.${DOMAIN}." + else + # Domain name based test (URI) + local TEST="${TARGET}.${DOMAIN}." + fi + - # Make sure the info is shown if we are cheking the servers - if [ "$VERIFY_BL" ] && [ $VERBOSE -lt 1 ]; then - VERBOSE=1 + # Make sure the info is shown if we are checking the servers + if [[ "${VERIFY_BL}" && "${VERBOSE}" -lt 1 ]]; then + local VERBOSE=1 fi # For verbose output - if [ $VERBOSE -ge 1 ]; then + if [[ "${VERBOSE}" -ge 1 ]]; then # Show percentage - STATUS=$(printf " %3s" $((I * 100 / COUNT))) - STATUS="$STATUS%% " + local STATUS + STATUS="$(printf " %3s" $((I * 100 / COUNT)))% " # Show additional info - if [ $VERBOSE -ge 3 ]; then - PREFIX=$(printf "%-60s" "$TEST") + if [[ "${VERBOSE}" -ge 3 ]]; then + PREFIX=$(printf "%-60s" "${TEST}") else - PREFIX=$(printf "%-50s" "$BL") + PREFIX=$(printf "%-50s" "${DOMAIN}") fi - PREFIX="$STATUS $PREFIX" - if test -z "$PLAIN"; then - printf "%s \b" "$PREFIX" + PREFIX="${STATUS} ${PREFIX}" + if [[ -z "${PLAIN}" ]]; then + printf "%s \b" "${PREFIX}" fi - elif [ $VERBOSE -ge 0 ]; then - if test -z "$PLAIN"; then - progress "$I" "$COUNT" + elif [[ "${VERBOSE}" -ge 0 ]]; then + if [[ -z "${PLAIN}" ]]; then + progress "${I}" "${COUNT}" fi fi # Get the status - RESPONSE=$(resolve "$TEST") - START=$(echo "$RESPONSE" | cut -c1-4) + local RESPONSE + RESPONSE=$(resolve "${TEST}") - # Not blacklisted - if [ ! "$RESPONSE" ]; then + # Not blocklisted + if [[ -z "${RESPONSE}" ]]; then # Make sure the server is viable - ERROR="" - if [ "$VERIFY_BL" ]; then - TDL=$(echo "$BL" | grep -om 1 '\([a-zA-Z0-9]\+\(-[a-zA-Z0-9]\+\)*\.\)[a-zA-Z]\{2,\}$') - if [ ! "$(resolve "$TDL" ns)" ]; then - if test -z "$PLAIN"; then printf "\r"; fi - printf "%s%sUnreachable server%s\n" "$YELLOW" "$PREFIX" "$CLEAR"; - INVALID=$((INVALID + 1)) + local ERROR="" + if [[ "${VERIFY_BL}" ]]; then + local TDL + TDL=$(grep -om 1 "${REGEX_TDL}" <<< "${DOMAIN}") + if [[ ! "$(resolve "${TDL}" ns)" ]]; then + if [[ -z "${PLAIN}" ]]; then printf "\r"; fi + printf "%s%sUnreachable server%s\n" "${YELLOW}" "${PREFIX}" "${CLEAR}"; + echo "$BL" | cut -d= -f1 >> "$invalid_file" ERROR=TRUE fi fi - if [ ! "$ERROR" ]; then - if [ "$VERIFY_BL" ] || [ $VERBOSE -ge 1 ]; then - if test -z "$PLAIN"; then printf "\r"; fi - printf "%s%s✓%s\n" "$CLEAR" "$PREFIX" "$CLEAR"; + if [[ ! "${ERROR}" ]]; then + if [[ -n "${VERIFY_BL}" || "${VERBOSE}" -ge 1 ]]; then + if [[ -z "${PLAIN}" ]]; then printf "\r"; fi + printf "%s%s✓%s\n" "${CLEAR}" "${PREFIX}" "${CLEAR}"; fi - PASSED=$((PASSED + 1)) + cut -d= -f1 <<< "$BL" >> "$passed_file" fi; # Invalid response - elif [ "$START" != "127." ]; then - if [ $VERBOSE -ge 1 ]; then - if test -z "$PLAIN"; then printf "\r"; fi - printf "%s%sinvalid response (%s)%s\n" "$YELLOW" "$PREFIX" "$RESPONSE" "$CLEAR"; + elif [[ ! "${RESPONSE}" =~ ${FILTER} ]] || [[ ${RESPONSE:0:8} == "127.255." ]]; then + if [[ "${VERBOSE}" -ge 1 ]]; then + [[ -z "${PLAIN}" ]] && printf "\r" + printf "%s%sinvalid response (%s)%s\n" "${YELLOW}" "${PREFIX}" "${RESPONSE}" "${CLEAR}"; + printf "%s%${#PREFIX}s%s%s\n" "${YELLOW}" "TXT: " "$(text "${TEST}")" "${CLEAR}"; fi; - INVALID=$((INVALID + 1)) + cut -d= -f1 <<< "$BL" >> "$invalid_file" - # Blacklisted + # matched else - if [ $VERBOSE -ge 1 ]; then - if test -z "$PLAIN"; then printf "\r"; fi - printf "%s%sblacklisted (%s)%s\n" "$RED" "$PREFIX" "$RESPONSE" "$CLEAR"; - elif [ $VERBOSE -ge 0 ]; then - if test -z "$PLAIN"; then printf "\r "; printf "\r"; fi - printf "%s%s%s : %s\n" "$RED" "$BL" "$CLEAR" "$RESPONSE" + if [[ "${TYPE}" != "DNSWL" ]]; then + if [[ "${VERBOSE}" -ge 1 ]]; then + [[ -z "${PLAIN}" ]] && printf "\r" + printf "%s%sblocklisted (%s)%s\n" "${RED}" "${PREFIX}" "${RESPONSE}" "${CLEAR}"; + printf "%s%${#PREFIX}s%s%s\n" "${RED}" "TXT: " "$(text "${TEST}")" "${CLEAR}"; + elif [[ "${VERBOSE}" -ge 0 ]]; then + [[ -z "${PLAIN}" ]] && printf "\r%60s\r" " " + printf "%s%s%s : %s\n" "${RED}" "${DOMAIN}" "${CLEAR}" "${RESPONSE}" + fi + cut -d= -f1 <<< "$BL" >> "$failed_file" + else + if [[ "${VERBOSE}" -ge 1 ]]; then + [[ -z "${PLAIN}" ]] && printf "\r" + printf "%s%sallowlisted (%s)%s\n" "${GREEN}" "${PREFIX}" "${RESPONSE}" "${CLEAR}"; + printf "%s%${#PREFIX}s%s%s\n" "${GREEN}" "TXT: " "$(text "${TEST}")" "${CLEAR}"; + elif [[ "${VERBOSE}" -ge 0 ]]; then + [[ -z "${PLAIN}" ]] && printf "\r%60s\r" " " + printf "%s%s%s : %s\n" "${GREEN}" "${DOMAIN}" "${CLEAR}" "${RESPONSE}" + fi + cut -d= -f1 <<< "$BL" >> "$allow_file" fi - FAILED=$((FAILED + 1)) fi -done +} + +allow_file=$(mktemp) +failed_file=$(mktemp) +passed_file=$(mktemp) +invalid_file=$(mktemp) +export -f parallel_check resolve text progress +export REGEX_LIST REGEX_TDL REVERSED TARGET VERIFY_BL VERBOSE PLAIN COUNT YELLOW CLEAR RED GREEN SPINNER REGEX_IP REGEX_DOMAIN CMD_DIG DNSSERVER CONF_DNS_DURATION CONF_DNS_TRIES REGEX CMD_HOST CMD allow_file failed_file passed_file invalid_file +# shellcheck disable=SC2086 +parallel -j "$j" 'parallel_check {#} {}' ::: ${BLOCKLISTS} # Print results -if [ $VERBOSE -ge 0 ]; then - if test -z "$PLAIN"; then +# shellcheck disable=SC2002 +if [[ "${VERBOSE}" -ge 0 ]]; then + if [[ -z "${PLAIN}" ]]; then printf "\r \n" else printf " \n" fi echo "----------------------------------------------------------" - echo Results for "$TARGET" + echo Results for "${TARGET}" echo printf "%-15s" "Tested:"; echo "${COUNT}" - printf "%-15s" "Passed:"; echo "${GREEN}${PASSED}${CLEAR}" - printf "%-15s" "Invalid:"; echo "${YELLOW}${INVALID}${CLEAR}" - printf "%-15s" "Blacklisted:"; echo "${RED}${FAILED}${CLEAR}" + printf "%-15s" "Passed:"; echo "${GREEN}$(cat "$passed_file" | wc -l)${CLEAR}" + printf "%-15s" "Whitelisted:"; echo "${GREEN}$(cat "$allow_file" | wc -l)${CLEAR}" + printf "%-15s" "Invalid:"; echo "${YELLOW}$(cat "$invalid_file" | wc -l)${CLEAR}" + printf "%-15s" "Blocklisted:"; echo "${RED}$(cat "$failed_file" | wc -l)${CLEAR}" echo "----------------------------------------------------------" fi -exit $FAILED; +# shellcheck disable=SC2002 +failed=$(cat "$failed_file" | wc -l) +if [ "$failed" -gt 0 ]; then + touch $output + echo "${TARGET}" >> $output + cat "$failed_file" >> $output +fi + +rm -f "$passed_file" "$allow_file" "$invalid_file" "$failed_file" + +unset -f parallel_check resolve text progress +unset REGEX_LIST REGEX_TDL REVERSED TARGET VERIFY_BL VERBOSE PLAIN COUNT YELLOW CLEAR RED GREEN SPINNER REGEX_IP REGEX_DOMAIN CMD_DIG DNSSERVER CONF_DNS_DURATION CONF_DNS_TRIES REGEX CMD_HOST CMD allow_file failed_file passed_file invalid_file +exit "${failed}"