1a367d050d 2010-08-13 c.kworr@d4daf: #!/usr/bin/env python3.1
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: # This script converts SquidGuard database into format that can be imported to
1a367d050d 2010-08-13 c.kworr@d4daf: # squid-tagger. It should be run in SquidGuard database directory and it would
1a367d050d 2010-08-13 c.kworr@d4daf: # produce csv stream that can be redirected to squid-tagger for imports:
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: # cd /var/db/squidGuard ; path/to/sg_import.py | path/to/squid-tagger.py -l -f
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: # This one will flush squid-tagger's database and load selected SquidGuard
1a367d050d 2010-08-13 c.kworr@d4daf: # database.
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: import codecs, csv, os, re, sys
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: data = {}
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: for (path, names, files) in os.walk('.'):
1a367d050d 2010-08-13 c.kworr@d4daf: tag = path.lstrip('./')
1a367d050d 2010-08-13 c.kworr@d4daf: for file in files:
1a367d050d 2010-08-13 c.kworr@d4daf: if file in ('domains', 'expressions', 'urls'):
1a367d050d 2010-08-13 c.kworr@d4daf: with codecs.open(path + os.sep + file, 'r', 'L1') as source:
1a367d050d 2010-08-13 c.kworr@d4daf: for full_line in source:
1a367d050d 2010-08-13 c.kworr@d4daf: line = full_line.strip()
1a367d050d 2010-08-13 c.kworr@d4daf: if not re.compile('^(#|$)').match(line):
1a367d050d 2010-08-13 c.kworr@d4daf: regexp = None
1a367d050d 2010-08-13 c.kworr@d4daf: if file == 'expressions':
1a367d050d 2010-08-13 c.kworr@d4daf: regexp = line
1a367d050d 2010-08-13 c.kworr@d4daf: line = None
1a367d050d 2010-08-13 c.kworr@d4daf: if file == 'urls':
1a367d050d 2010-08-13 c.kworr@d4daf: (line, sep, regexp) = line.partition('/')
1a367d050d 2010-08-13 c.kworr@d4daf: regexp = '^' + re.escape(regexp)
1a367d050d 2010-08-13 c.kworr@d4daf: if line in data:
1a367d050d 2010-08-13 c.kworr@d4daf: if regexp in data[line]:
1a367d050d 2010-08-13 c.kworr@d4daf: data[line][regexp].add(tag)
1a367d050d 2010-08-13 c.kworr@d4daf: else:
1a367d050d 2010-08-13 c.kworr@d4daf: data[line][regexp] = set([tag])
1a367d050d 2010-08-13 c.kworr@d4daf: else:
1a367d050d 2010-08-13 c.kworr@d4daf: data[line] = {regexp: set([tag])}
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: cw = csv.writer(sys.stdout)
1a367d050d 2010-08-13 c.kworr@d4daf: cw.writerow(['site', 'tags', 'regexp'])
1a367d050d 2010-08-13 c.kworr@d4daf:
1a367d050d 2010-08-13 c.kworr@d4daf: for domain in data:
1a367d050d 2010-08-13 c.kworr@d4daf: for regexp in data[domain]:
1a367d050d 2010-08-13 c.kworr@d4daf: cw.writerow([domain, '{' + ','.join(data[domain][regexp]) + '}', regexp])