| 1 |
#!/bin/bash |
| 2 |
# |
| 3 |
# squidGuard blacklist update script v1.3 (incomplete!) |
| 4 |
# http://www.bn-paf.de/filter/ - filter@bn-paf.de |
| 5 |
# |
| 6 |
# Todo (somebody wants to help?): |
| 7 |
# - notify if: |
| 8 |
# downloaded files are broken |
| 9 |
# squidGuard goes into emergency mode for some reason |
| 10 |
|
| 11 |
SQGDIR=/usr/share/squidGuard-1.1.4 |
| 12 |
SQGLOG=/var/log/squidGuard/log |
| 13 |
CONFIG=/etc/squid/squidGuard/squidGuard.conf |
| 14 |
|
| 15 |
# Use "proxy" for Debian, otherwise usually "squid" |
| 16 |
USER=nobody |
| 17 |
|
| 18 |
# Copy blacklists locally for testing purposes |
| 19 |
# --- not implemented yet --- |
| 20 |
DEVEL=0 |
| 21 |
|
| 22 |
# Workaround for some installations having problems with diffs |
| 23 |
TEXT=0 |
| 24 |
|
| 25 |
|
| 26 |
|
| 27 |
if test -z $1; |
| 28 |
then echo; echo Please specifiy what to update: all, diffs or custom; echo; exit; |
| 29 |
fi |
| 30 |
|
| 31 |
|
| 32 |
|
| 33 |
custom-update () |
| 34 |
{ echo Compiling custom text files... |
| 35 |
#/usr/local/squidGuard/bin/squidGuard -C all -c $CONFIG 2>$SQGLOG/custom.log |
| 36 |
/usr/bin/squidGuard -C bad/domains -c $CONFIG 2>$SQGLOG/custom.log |
| 37 |
/usr/bin/squidGuard -C bad/urls -c $CONFIG 2>>$SQGLOG/custom.log |
| 38 |
/usr/bin/squidGuard -C good/domains -c $CONFIG 2>>$SQGLOG/custom.log |
| 39 |
/usr/bin/squidGuard -C good/urls -c $CONFIG 2>>$SQGLOG/custom.log; } |
| 40 |
|
| 41 |
if test "$1" = custom; then custom-update; /usr/sbin/squid -k reconfigure; exit; fi |
| 42 |
|
| 43 |
|
| 44 |
|
| 45 |
dircreate () { if test ! -d $1; then mkdir $1; fi; } |
| 46 |
filecreate () { if test ! -f $1; then echo > $1; fi; } |
| 47 |
|
| 48 |
dircreate $SQGDIR |
| 49 |
dircreate $SQGDIR/db |
| 50 |
dircreate $SQGDIR/db/custom |
| 51 |
dircreate $SQGDIR/db/custom/bad |
| 52 |
filecreate $SQGDIR/db/custom/bad/domains |
| 53 |
filecreate $SQGDIR/db/custom/bad/urls |
| 54 |
dircreate $SQGDIR/db/custom/good |
| 55 |
filecreate $SQGDIR/db/custom/good/domains |
| 56 |
filecreate $SQGDIR/db/custom/good/urls |
| 57 |
dircreate $SQGDIR/tmp |
| 58 |
dircreate $SQGLOG |
| 59 |
filecreate $SQGLOG/blocked.log |
| 60 |
|
| 61 |
|
| 62 |
|
| 63 |
rm -rf $SQGDIR/tmp/* |
| 64 |
cd $SQGDIR/tmp |
| 65 |
|
| 66 |
echo Downloading blacklists... |
| 67 |
|
| 68 |
if test "$1" = all -o ! -f ../blacklists.tar.gz; then |
| 69 |
wget -nv ftp://ftp.ost.eltele.no/pub/www/proxy/squidGuard/contrib/blacklists.tar.gz |
| 70 |
#cp $SQGDIR/devel/blacklists.tar.gz . |
| 71 |
mv blacklists.tar.gz .. |
| 72 |
fi |
| 73 |
tar xfz ../blacklists.tar.gz |
| 74 |
|
| 75 |
wget -nv http://www.bn-paf.de/filter/de-blacklists.tar.gz |
| 76 |
#cp $SQGDIR/devel/de-blacklists.tar.gz . |
| 77 |
tar xfz de-blacklists.tar.gz |
| 78 |
|
| 79 |
wget -nv ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/adult.tar.gz |
| 80 |
wget -nv ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/publicite.tar.gz |
| 81 |
wget -nv ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/redirector.tar.gz |
| 82 |
wget -nv ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/warez.tar.gz |
| 83 |
#cp $SQGDIR/devel/adult.tar.gz . |
| 84 |
#cp $SQGDIR/devel/publicite.tar.gz . |
| 85 |
#cp $SQGDIR/devel/redirector.tar.gz . |
| 86 |
#cp $SQGDIR/devel/warez.tar.gz . |
| 87 |
mkdir fr-blacklists |
| 88 |
cd fr-blacklists |
| 89 |
tar xfz ../adult.tar.gz |
| 90 |
tar xfz ../publicite.tar.gz |
| 91 |
tar xfz ../redirector.tar.gz |
| 92 |
tar xfz ../warez.tar.gz |
| 93 |
cd .. |
| 94 |
mv fr-blacklists/adult fr-blacklists/porn |
| 95 |
mv fr-blacklists/publicite fr-blacklists/ads |
| 96 |
mv fr-blacklists/redirector fr-blacklists/hacking |
| 97 |
|
| 98 |
wget -nv http://squidguard.mesd.k12.or.us/mesd-blacklists.tgz |
| 99 |
#cp $SQGDIR/devel/mesd-blacklists.tgz . |
| 100 |
tar xfz mesd-blacklists.tgz |
| 101 |
|
| 102 |
|
| 103 |
|
| 104 |
|
| 105 |
if ! test "$TEXT" = 1; then |
| 106 |
|
| 107 |
echo Compiling diffs... |
| 108 |
|
| 109 |
diffgen () |
| 110 |
{ if test -f $1; |
| 111 |
then sed 's/^/+/' $1 >> $SQGDIR/db/$filter/$2.diff; fi |
| 112 |
if test -f $1.exclude; |
| 113 |
then sed 's/^/-/' $1.exclude >> $SQGDIR/db/$filter/$2.diff; fi; } |
| 114 |
|
| 115 |
diffadd () |
| 116 |
{ cat $source-blacklists/$filter/$1.diff>>$SQGDIR/db/$filter/$1.diff; } |
| 117 |
|
| 118 |
for filter in ads aggressive audio-video drugs gambling hacking porn violence warez; |
| 119 |
do |
| 120 |
# Creates empty domain/url files to avoid emergency mode when compilling |
| 121 |
# custom lists later. |
| 122 |
dircreate $SQGDIR/db/$filter |
| 123 |
mv blacklists/$filter/*.db $SQGDIR/db/$filter/ |
| 124 |
filecreate $SQGDIR/db/$filter/domains; filecreate $SQGDIR/db/$filter/urls; |
| 125 |
for source in de fr mesd |
| 126 |
do |
| 127 |
if test -f $source-blacklists/$filter/domains.diff; |
| 128 |
then diffadd domains; |
| 129 |
else diffgen $source-blacklists/$filter/domains domains; |
| 130 |
fi |
| 131 |
if test -f $source-blacklists/$filter/urls.diff; |
| 132 |
then diffadd urls; |
| 133 |
else diffgen $source-blacklists/$filter/urls urls; |
| 134 |
fi |
| 135 |
done |
| 136 |
done |
| 137 |
|
| 138 |
else |
| 139 |
# Much slower than using diffs, creates plain text urls/domains |
| 140 |
# Thanks Wolfgang for this part! |
| 141 |
echo Creating text files... |
| 142 |
for filter in ads aggressive audio-video drugs gambling hacking porn violence warez; |
| 143 |
do |
| 144 |
if test ! -d $SQGDIR/db/$filter; then mkdir $SQGDIR/db/$filter; fi |
| 145 |
for addrtype in domains urls; |
| 146 |
do |
| 147 |
# We filter possible <LF>s using tr (comm is not working with that) and |
| 148 |
# comment lines which are confusing to read after sorting |
| 149 |
cat blacklists/$filter/$addrtype > temp.txt |
| 150 |
for source in de fr mesd |
| 151 |
do |
| 152 |
if test -f $source-blacklists/$filter/$addrtype; |
| 153 |
then cat $source-blacklists/$filter/$addrtype >> temp.txt; fi |
| 154 |
done |
| 155 |
comm -2 -3 <(tr -d '\r' < temp.txt | grep -v "^#" | sort) <(tr -d '\r' < de-blacklists/$filter/$addrtype.exclude | grep -v "^#" | sort) > $SQGDIR/db/$filter/$addrtype |
| 156 |
done |
| 157 |
done |
| 158 |
rm temp.txt |
| 159 |
|
| 160 |
fi |
| 161 |
|
| 162 |
|
| 163 |
|
| 164 |
echo Securing files... |
| 165 |
chownmod () { chown $1 $2 |
| 166 |
chmod ug+rw,o-rw $2 |
| 167 |
chown -R $1 $2/* |
| 168 |
chmod -R ug+rw,o-rw $2/*; } |
| 169 |
chownmod $USER:$USER $SQGDIR/db |
| 170 |
chownmod root:root $SQGDIR/tmp |
| 171 |
chownmod $USER:$USER $SQGLOG |
| 172 |
|
| 173 |
|
| 174 |
|
| 175 |
if ! test "$TEXT" = 1; then |
| 176 |
if ! test "$1" = custom; then |
| 177 |
echo Applying diffs... |
| 178 |
/usr/bin/squidGuard -u -c $CONFIG 2>$SQGLOG/update.log |
| 179 |
# Mandrake RPM: /usr/squidGuard/log/squidGuard.log will be used instead of |
| 180 |
# update.log if directory exists |
| 181 |
fi |
| 182 |
custom-update |
| 183 |
else |
| 184 |
echo Compiling text files... |
| 185 |
/usr/bin/squidGuard -C all -c $CONFIG 2>$SQGLOG/update.log |
| 186 |
fi |
| 187 |
|
| 188 |
|
| 189 |
echo Reconfiguring Squid...; /usr/sbin/squid -k reconfigure |
| 190 |
|
Parent Directory
| 
Revision Log