Index: squid-tagger.py ================================================================== --- squid-tagger.py +++ squid-tagger.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3.1 -import configparser, optparse, os, postgresql.api, re, sys, _thread +import postgresql.api, re, sys # wrapper around syslog, can be muted class Logger: __slots__ = frozenset(['_syslog']) @@ -23,11 +23,11 @@ if self._syslog: self._syslog.syslog(self._syslog.LOG_NOTICE, message) # wrapper around database class tagDB: - __slots__ = frozenset(('_check_stmt', '_db')) + __slots__ = frozenset(('_check_stmt', '_db', '_dump_stmt')) def __init__(self): config.section('database') self._db = postgresql.open( 'pq://{}:{}@{}/{}'.format( @@ -34,15 +34,23 @@ config['user'], config['password'], config['host'], config['database'], ) ) - self._check_stmt = self._db.prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc") + self._check_stmt = None + self._dump_stmt = None def check(self, site, ip_address): + if self._check_stmt == None: + self._check_stmt = self._db.prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc") return(self._check_stmt(site, ip_address)) + def dump(self): + if self._dump_stmt == None: + self._dump_stmt = self._db.prepare("select untrip(site), tag, regexp from urls natural join site natural join tag order by site, tag") + return(self._dump_stmt()) + # abstract class with basic checking functionality class Checker: __slots__ = frozenset(['_db', '_log']) def __init__(self): @@ -102,10 +110,12 @@ # threaded checking facility class CheckerThread(Checker): __slots__ = frozenset(['_lock', '_lock_exit', '_lock_queue', '_queue']) def __init__(self): + import _thread + # basic initialisation Checker.__init__(self) # Spin lock. Loop acquires it on start then releases it when holding queue # lock. This way the thread proceeds without stops while queue has data and @@ -226,11 +236,11 @@ self._queue.append((id, site, ip_address, url_path)) self._log.info('request {} queued ({})\n'.format(id, line)) # this classes processes config file and substitutes default values class Config: - __slots__ = frozenset(['_config', '_default', '_section']) + __slots__ = frozenset(['_config', '_default', '_section', 'options']) _default = { 'reactor': { 'reactor': 'thread', }, 'log': { @@ -241,21 +251,26 @@ 'database': 'squidTag', },} # function to read in config file def __init__(self): + import configparser, optparse, os + parser = optparse.OptionParser() parser.add_option('-c', '--config', dest = 'config', help = 'config file location', metavar = 'FILE', default = '/usr/local/etc/squid-tagger.conf') + parser.add_option('-d', '--dump', dest = 'dump', + help = 'dump database', action = 'store_true', metavar = 'bool', + default = False) - (options, args) = parser.parse_args() + (self.options, args) = parser.parse_args() - assert os.access(options.config, os.R_OK), "Fatal error: can't read {}".format(options.config) + assert os.access(self.options.config, os.R_OK), "Fatal error: can't read {}".format(self.options.config) self._config = configparser.ConfigParser() - self._config.readfp(open(options.config)) + self._config.readfp(open(self.options.config)) # function to select config file section or create one def section(self, section): if not self._config.has_section(section): self._config.add_section(section) @@ -275,14 +290,27 @@ return(self._config.get(self._section, name)) # initializing and reading in config file config = Config() -config.section('reactor') -if config['reactor'] == 'thread': - checker = CheckerThread() -elif config['reactor'] == 'plain': - checker = Checker() -elif config['reactor'] == 'kqueue': - checker = CheckerKqueue() - -checker.loop() +if config.options.dump: + # dumping database + import csv + + tagdb = tagDB() + + csv_writer = csv.writer(sys.stdout) + csv_writer.writerow(['site', 'tags', 'regexp']) + for row in tagdb.dump(): + csv_writer.writerow([row[0], '{' + ','.join(row[1]) + '}', row[2]]) + +else: + # main loop + config.section('reactor') + if config['reactor'] == 'thread': + checker = CheckerThread() + elif config['reactor'] == 'plain': + checker = Checker() + elif config['reactor'] == 'kqueue': + checker = CheckerKqueue() + + checker.loop() DELETED st-dump.py Index: st-dump.py ================================================================== --- st-dump.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3.1 - -import configparser, csv, optparse, os, postgresql.api, sys - -# wrapper around syslog, can be muted -class Logger: - __slots__ = frozenset(['_syslog']) - - def __init__(self): - config.section('log') - if config['silent'] == 'yes': - self._syslog = None - else: - import syslog - self._syslog = syslog - self._syslog.openlog('squidTag') - - def info(self, message): - if self._syslog: - self._syslog.syslog(self._syslog.LOG_INFO, message) - - def notice(self, message): - if self._syslog: - self._syslog.syslog(self._syslog.LOG_NOTICE, message) - -# wrapper around database -class tagDB: - __slots__ = frozenset(['_prepared', '_dump_stmt', '_db']) - - def __init__(self): - self._prepared = set() - self._db = False - self._dump_stmt = self._curs().prepare("select untrip(site), tag, regexp from urls natural join site natural join tag order by site, tag") - - def _curs(self): - if not self._db: - config.section('database') - self._db = postgresql.open( - 'pq://{}:{}@{}/{}'.format( - config['user'], - config['password'], - config['host'], - config['database'], - ) ) - return(self._db) - - def dump(self): - return(self._dump_stmt()) - -# this classes processes config file and substitutes default values -class Config: - __slots__ = frozenset(['_config', '_default', '_section']) - _default = { - 'reactor': { - 'reactor': 'thread', - }, - 'log': { - 'silent': 'no', - }, - 'database': { - 'host': 'localhost', - 'database': 'squidTag', - },} - - # function to read in config file - def __init__(self): - parser = optparse.OptionParser() - parser.add_option('-c', '--config', dest = 'config', - help = 'config file location', metavar = 'FILE', - default = '/usr/local/etc/squid-tagger.conf') - - (options, args) = parser.parse_args() - - if not os.access(options.config, os.R_OK): - print("Can't read {}: exitting".format(options.config)) - sys.exit(2) - - self._config = configparser.ConfigParser() - self._config.readfp(open(options.config)) - - # function to select config file section or create one - def section(self, section): - if not self._config.has_section(section): - self._config.add_section(section) - self._section = section - - # function to get config parameter, if parameter doesn't exists the default - # value or None is substituted - def __getitem__(self, name): - if not self._config.has_option(self._section, name): - if self._section in self._default: - if name in self._default[self._section]: - self._config.set(self._section, name, self._default[self._section][name]) - else: - self._config.set(self._section, name, None) - else: - self._config.set(self._section, name, None) - return(self._config.get(self._section, name)) - -# initializing and reading in config file -config = Config() - -tagdb = tagDB() - -csv_writer = csv.writer(sys.stdout) -csv_writer.writerow(['site', 'tags', 'regexp']) -for row in tagdb.dump(): - csv_writer.writerow([row[0], '{' + ','.join(row[1]) + '}', row[2]])