0ef24b1937 2011-04-06 1: #!/usr/bin/env python
0ef24b1937 2011-04-06 2:
0ef24b1937 2011-04-06 3: from __future__ import division, print_function, unicode_literals
0ef24b1937 2011-04-06 4:
0ef24b1937 2011-04-06 5: import gevent.monkey
0ef24b1937 2011-04-06 6: gevent.monkey.patch_all()
0ef24b1937 2011-04-06 7:
0ef24b1937 2011-04-06 8: import fcntl, gevent.core, gevent.pool, gevent.queue, gevent.socket, os, psycopg2, re, sys
0ef24b1937 2011-04-06 9:
0ef24b1937 2011-04-06 10: # //inclusion start
0ef24b1937 2011-04-06 11: # Copyright (C) 2010 Daniele Varrazzo <daniele.varrazzo@gmail.com>
0ef24b1937 2011-04-06 12: # and licensed under the MIT license:
0ef24b1937 2011-04-06 13:
0ef24b1937 2011-04-06 14: def gevent_wait_callback(conn, timeout=None):
0ef24b1937 2011-04-06 15: """A wait callback useful to allow gevent to work with Psycopg."""
0ef24b1937 2011-04-06 16: while 1:
0ef24b1937 2011-04-06 17: state = conn.poll()
0ef24b1937 2011-04-06 18: if state == psycopg2.extensions.POLL_OK:
0ef24b1937 2011-04-06 19: break
0ef24b1937 2011-04-06 20: elif state == psycopg2.extensions.POLL_READ:
0ef24b1937 2011-04-06 21: gevent.socket.wait_read(conn.fileno(), timeout=timeout)
0ef24b1937 2011-04-06 22: elif state == psycopg2.extensions.POLL_WRITE:
0ef24b1937 2011-04-06 23: gevent.socket.wait_write(conn.fileno(), timeout=timeout)
0ef24b1937 2011-04-06 24: else:
0ef24b1937 2011-04-06 25: raise psycopg2.OperationalError("Bad result from poll: %r" % state)
0ef24b1937 2011-04-06 26:
0ef24b1937 2011-04-06 27: if not hasattr(psycopg2.extensions, 'set_wait_callback'):
0ef24b1937 2011-04-06 28: raise ImportError("support for coroutines not available in this Psycopg version (%s)" % psycopg2.__version__)
0ef24b1937 2011-04-06 29: psycopg2.extensions.set_wait_callback(gevent_wait_callback)
0ef24b1937 2011-04-06 30:
0ef24b1937 2011-04-06 31: # //inclusion end
0ef24b1937 2011-04-06 32:
39b97ced92 2011-06-05 33: # this classes processes config file and substitutes default values
39b97ced92 2011-06-05 34: class Config:
39b97ced92 2011-06-05 35: __slots__ = frozenset(['_config', '_default', '_section', 'options'])
39b97ced92 2011-06-05 36: _default = {
39b97ced92 2011-06-05 37: 'log': {
39b97ced92 2011-06-05 38: 'silent': 'no',
39b97ced92 2011-06-05 39: },
39b97ced92 2011-06-05 40: 'database': {
39b97ced92 2011-06-05 41: 'host': 'localhost',
39b97ced92 2011-06-05 42: 'database': 'squidTag',
39b97ced92 2011-06-05 43: },}
39b97ced92 2011-06-05 44:
39b97ced92 2011-06-05 45: # function to read in config file
39b97ced92 2011-06-05 46: def __init__(self):
39b97ced92 2011-06-05 47: import ConfigParser, optparse, os
39b97ced92 2011-06-05 48:
39b97ced92 2011-06-05 49: parser = optparse.OptionParser()
39b97ced92 2011-06-05 50: parser.add_option('-c', '--config', dest = 'config',
39b97ced92 2011-06-05 51: help = 'config file location', metavar = 'FILE',
39b97ced92 2011-06-05 52: default = '/usr/local/etc/squid-tagger.conf')
39b97ced92 2011-06-05 53: parser.add_option('-d', '--dump', dest = 'dump',
39b97ced92 2011-06-05 54: help = 'dump database', action = 'store_true', metavar = 'bool',
39b97ced92 2011-06-05 55: default = False)
39b97ced92 2011-06-05 56: parser.add_option('-f', '--flush-database', dest = 'flush_db',
39b97ced92 2011-06-05 57: help = 'flush previous database on load', default = False,
39b97ced92 2011-06-05 58: action = 'store_true', metavar = 'bool')
39b97ced92 2011-06-05 59: parser.add_option('-l', '--load', dest = 'load',
39b97ced92 2011-06-05 60: help = 'load database', action = 'store_true', metavar = 'bool',
39b97ced92 2011-06-05 61: default = False)
39b97ced92 2011-06-05 62: parser.add_option('-D', '--dump-conf', dest = 'dump_conf',
39b97ced92 2011-06-05 63: help = 'dump filtering rules', default = False, metavar = 'bool',
39b97ced92 2011-06-05 64: action = 'store_true')
39b97ced92 2011-06-05 65: parser.add_option('-L', '--load-conf', dest = 'load_conf',
39b97ced92 2011-06-05 66: help = 'load filtering rules', default = False, metavar = 'bool',
39b97ced92 2011-06-05 67: action = 'store_true')
39b97ced92 2011-06-05 68:
39b97ced92 2011-06-05 69: (self.options, args) = parser.parse_args()
39b97ced92 2011-06-05 70:
39b97ced92 2011-06-05 71: assert os.access(self.options.config, os.R_OK), "Fatal error: can't read {}".format(self.options.config)
39b97ced92 2011-06-05 72:
39b97ced92 2011-06-05 73: self._config = ConfigParser.ConfigParser()
39b97ced92 2011-06-05 74: self._config.readfp(open(self.options.config))
39b97ced92 2011-06-05 75:
39b97ced92 2011-06-05 76: # function to select config file section or create one
39b97ced92 2011-06-05 77: def section(self, section):
39b97ced92 2011-06-05 78: if not self._config.has_section(section):
39b97ced92 2011-06-05 79: self._config.add_section(section)
39b97ced92 2011-06-05 80: self._section = section
39b97ced92 2011-06-05 81:
39b97ced92 2011-06-05 82: # function to get config parameter, if parameter doesn't exists the default
39b97ced92 2011-06-05 83: # value or None is substituted
39b97ced92 2011-06-05 84: def __getitem__(self, name):
39b97ced92 2011-06-05 85: if not self._config.has_option(self._section, name):
39b97ced92 2011-06-05 86: if self._section in self._default:
39b97ced92 2011-06-05 87: if name in self._default[self._section]:
39b97ced92 2011-06-05 88: self._config.set(self._section, name, self._default[self._section][name])
39b97ced92 2011-06-05 89: else:
39b97ced92 2011-06-05 90: self._config.set(self._section, name, None)
39b97ced92 2011-06-05 91: else:
39b97ced92 2011-06-05 92: self._config.set(self._section, name, None)
39b97ced92 2011-06-05 93: return(self._config.get(self._section, name))
39b97ced92 2011-06-05 94:
39b97ced92 2011-06-05 95: # initializing and reading in config file
39b97ced92 2011-06-05 96: config = Config()
39b97ced92 2011-06-05 97:
39b97ced92 2011-06-05 98: # wrapper around syslog, can be muted
39b97ced92 2011-06-05 99: class Logger(object):
39b97ced92 2011-06-05 100: __slots__ = frozenset(['_syslog'])
39b97ced92 2011-06-05 101:
39b97ced92 2011-06-05 102: def __init__(self):
39b97ced92 2011-06-05 103: config.section('log')
39b97ced92 2011-06-05 104: if config['silent'] == 'yes':
39b97ced92 2011-06-05 105: self._syslog = None
39b97ced92 2011-06-05 106: else:
39b97ced92 2011-06-05 107: import syslog
39b97ced92 2011-06-05 108: self._syslog = syslog
39b97ced92 2011-06-05 109: self._syslog.openlog(str('squidTag'))
39b97ced92 2011-06-05 110:
39b97ced92 2011-06-05 111: def info(self, message):
39b97ced92 2011-06-05 112: if self._syslog != None:
39b97ced92 2011-06-05 113: self._syslog.syslog(self._syslog.LOG_INFO, message)
39b97ced92 2011-06-05 114:
39b97ced92 2011-06-05 115: def notice(self, message):
39b97ced92 2011-06-05 116: if self._syslog != None:
39b97ced92 2011-06-05 117: self._syslog.syslog(self._syslog.LOG_NOTICE, message)
39b97ced92 2011-06-05 118:
39b97ced92 2011-06-05 119: logger = Logger()
39b97ced92 2011-06-05 120:
0ef24b1937 2011-04-06 121: # tiny wrapper around a file to make reads from it geventable
0ef24b1937 2011-04-06 122: # or should i move this somewhere?
0ef24b1937 2011-04-06 123:
0ef24b1937 2011-04-06 124: class FReadlineQueue(gevent.queue.Queue):
0ef24b1937 2011-04-06 125: # storing file descriptor, leftover
0ef24b1937 2011-04-06 126: __slots__ = frozenset(['_fd', '_tail'])
0ef24b1937 2011-04-06 127:
0ef24b1937 2011-04-06 128: def __init__(self, fd):
0ef24b1937 2011-04-06 129: # initialising class
0ef24b1937 2011-04-06 130: gevent.queue.Queue.__init__(self)
0ef24b1937 2011-04-06 131: # storing file descriptor
0ef24b1937 2011-04-06 132: self._fd = fd
0ef24b1937 2011-04-06 133: # using empty tail
0ef24b1937 2011-04-06 134: self._tail = ''
0ef24b1937 2011-04-06 135: # setting up event
0ef24b1937 2011-04-06 136: self._install_wait()
0ef24b1937 2011-04-06 137:
0ef24b1937 2011-04-06 138: def _install_wait(self):
0ef24b1937 2011-04-06 139: fileno = self._fd.fileno()
0ef24b1937 2011-04-06 140: # putting file to nonblocking mode
0ef24b1937 2011-04-06 141: fcntl.fcntl(fileno, fcntl.F_SETFL, fcntl.fcntl(fileno, fcntl.F_GETFL) | os.O_NONBLOCK)
0ef24b1937 2011-04-06 142: # installing event handler
0ef24b1937 2011-04-06 143: gevent.core.read_event(fileno, self._wait_helper)
0ef24b1937 2011-04-06 144:
0ef24b1937 2011-04-06 145: def _wait_helper(self, ev, evtype):
0ef24b1937 2011-04-06 146: # reading one buffer from stream
0ef24b1937 2011-04-06 147: buf = self._fd.read(4096)
0ef24b1937 2011-04-06 148: # splitting stream by line ends
39b97ced92 2011-06-05 149: rows = buf.decode('l1').split('\n')
0ef24b1937 2011-04-06 150: # adding tail to the first element if there is some tail
0ef24b1937 2011-04-06 151: if len(self._tail) > 0:
0ef24b1937 2011-04-06 152: rows[0] = self._tail + rows[0]
0ef24b1937 2011-04-06 153: # popping out last (incomplete) element
0ef24b1937 2011-04-06 154: self._tail = rows.pop(-1)
0ef24b1937 2011-04-06 155: # dropping all complete elements to the queue
0ef24b1937 2011-04-06 156: for row in rows:
0ef24b1937 2011-04-06 157: self.put_nowait(row)
39b97ced92 2011-06-05 158: logger.info('request: ' + row)
0ef24b1937 2011-04-06 159: if len(buf) > 0:
0ef24b1937 2011-04-06 160: # no EOF, reinstalling event handler
0ef24b1937 2011-04-06 161: gevent.core.read_event(self._fd.fileno(), self._wait_helper)
0ef24b1937 2011-04-06 162: else:
0ef24b1937 2011-04-06 163: # EOF found, sending EOF to queue
0ef24b1937 2011-04-06 164: self.put_nowait(None)
0ef24b1937 2011-04-06 165:
0ef24b1937 2011-04-06 166: stdin = FReadlineQueue(sys.stdin)
0ef24b1937 2011-04-06 167:
ddbf5288b9 2010-11-03 168: # wrapper around database
0ef24b1937 2011-04-06 169: class tagDB(object):
0ef24b1937 2011-04-06 170: __slots__ = frozenset(['_cursor', '_db'])
ddbf5288b9 2010-11-03 171:
ddbf5288b9 2010-11-03 172: def __init__(self):
ddbf5288b9 2010-11-03 173: config.section('database')
d2c7ba18a4 2011-09-14 174: if config['host'] == None:
d2c7ba18a4 2011-09-14 175: self._db = psycopg2.connect(
d2c7ba18a4 2011-09-14 176: database = config['database'],
d2c7ba18a4 2011-09-14 177: user = config['user'],
d2c7ba18a4 2011-09-14 178: password = config['password']
d2c7ba18a4 2011-09-14 179: )
d2c7ba18a4 2011-09-14 180: else:
d2c7ba18a4 2011-09-14 181: self._db = psycopg2.connect(
d2c7ba18a4 2011-09-14 182: database = config['database'],
d2c7ba18a4 2011-09-14 183: host = config['host'],
d2c7ba18a4 2011-09-14 184: user = config['user'],
d2c7ba18a4 2011-09-14 185: password = config['password']
d2c7ba18a4 2011-09-14 186: )
0ef24b1937 2011-04-06 187: self._cursor = self._db.cursor()
0ef24b1937 2011-04-06 188:
0ef24b1937 2011-04-06 189: def _field_names(self):
0ef24b1937 2011-04-06 190: names = []
0ef24b1937 2011-04-06 191: for record in self._cursor.description:
0ef24b1937 2011-04-06 192: names.append(record.name)
0ef24b1937 2011-04-06 193: return(names)
ddbf5288b9 2010-11-03 194:
ddbf5288b9 2010-11-03 195: def check(self, site, ip_address):
39b97ced92 2011-06-05 196: self._cursor.execute("select * from (select redirect_url, regexp from site_rule where site <@ tripdomain(%s) and netmask >>= %s order by array_length(site, 1) desc) a group by redirect_url, regexp", [site, ip_address])
0ef24b1937 2011-04-06 197: return(self._cursor.fetchall())
ddbf5288b9 2010-11-03 198:
ddbf5288b9 2010-11-03 199: def dump(self):
0ef24b1937 2011-04-06 200: self._cursor.execute("select untrip(site) as site, tag::text, regexp from urls order by site, tag")
0ef24b1937 2011-04-06 201: return(self._field_names(), self._cursor.fetchall())
ddbf5288b9 2010-11-03 202:
ddbf5288b9 2010-11-03 203: def load(self, data):
0ef24b1937 2011-04-06 204: if config.options.flush_db:
0ef24b1937 2011-04-06 205: self._cursor.execute('delete from urls;')
0ef24b1937 2011-04-06 206: bundle = []
0ef24b1937 2011-04-06 207: for row in data:
0ef24b1937 2011-04-06 208: if len(row) == 2:
0ef24b1937 2011-04-06 209: bundle.append([row[0], row[1], None])
0ef24b1937 2011-04-06 210: else:
0ef24b1937 2011-04-06 211: bundle.append([row[0], row[1], row[2]])
0ef24b1937 2011-04-06 212: self._cursor.executemany("insert into urls (site, tag, regexp) values (tripdomain(%s), %s, %s)", bundle)
0ef24b1937 2011-04-06 213: self._cursor.execute("update urls set regexp = NULL where regexp = ''")
0ef24b1937 2011-04-06 214: self._db.commit()
ddbf5288b9 2010-11-03 215:
ddbf5288b9 2010-11-03 216: def load_conf(self, csv_data):
0ef24b1937 2011-04-06 217: self._cursor.execute('delete from rules;')
0ef24b1937 2011-04-06 218: bundle = []
0ef24b1937 2011-04-06 219: for row in csv_data:
0ef24b1937 2011-04-06 220: bundle.append([row[0], row[1], int(row[2]), int(row[3]), row[4], row[5], row[6]])
0ef24b1937 2011-04-06 221: self._cursor.executemany("insert into rules (netmask, redirect_url, from_weekday, to_weekday, from_time, to_time, tag) values (%s::text::cidr, %s, %s, %s, %s::text::time, %s::text::time, %s::text::text[])", bundle)
0ef24b1937 2011-04-06 222: self._db.commit()
ddbf5288b9 2010-11-03 223:
ddbf5288b9 2010-11-03 224: def dump_conf(self):
0ef24b1937 2011-04-06 225: self._cursor.execute("select netmask, redirect_url, from_weekday, to_weekday, from_time, to_time, tag::text from rules")
0ef24b1937 2011-04-06 226: return(self._field_names(), self._cursor.fetchall())
ddbf5288b9 2010-11-03 227:
ddbf5288b9 2010-11-03 228: # abstract class with basic checking functionality
0ef24b1937 2011-04-06 229: class Checker(object):
0ef24b1937 2011-04-06 230: __slots__ = frozenset(['_db', '_log', '_queue', '_request'])
ddbf5288b9 2010-11-03 231:
39b97ced92 2011-06-05 232: def __init__(self, queue, logger):
ddbf5288b9 2010-11-03 233: self._db = tagDB()
39b97ced92 2011-06-05 234: self._log = logger
ddbf5288b9 2010-11-03 235: self._log.info('started\n')
ddbf5288b9 2010-11-03 236: self._request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$')
0ef24b1937 2011-04-06 237: self._queue = queue
ddbf5288b9 2010-11-03 238:
ddbf5288b9 2010-11-03 239: def process(self, id, site, ip_address, url_path, line = None):
0ef24b1937 2011-04-06 240: #self._log.info('trying {}\n'.format(site))
ddbf5288b9 2010-11-03 241: result = self._db.check(site, ip_address)
ddbf5288b9 2010-11-03 242: reply = None
39b97ced92 2011-06-05 243: #self._log.info('got {} lines from database'.format(len(result)))
ddbf5288b9 2010-11-03 244: for row in result:
ddbf5288b9 2010-11-03 245: if row != None and row[0] != None:
ddbf5288b9 2010-11-03 246: if row[1] != None:
ddbf5288b9 2010-11-03 247: self._log.info('trying regexp "{}" versus "{}"\n'.format(row[1], url_path))
ddbf5288b9 2010-11-03 248: try:
ddbf5288b9 2010-11-03 249: if re.compile(row[1]).match(url_path):
ddbf5288b9 2010-11-03 250: reply = row[0].format(url_path)
ddbf5288b9 2010-11-03 251: else:
ddbf5288b9 2010-11-03 252: continue
ddbf5288b9 2010-11-03 253: except:
ddbf5288b9 2010-11-03 254: self._log.info("can't compile regexp")
ddbf5288b9 2010-11-03 255: else:
ddbf5288b9 2010-11-03 256: reply = row[0].format(url_path)
ddbf5288b9 2010-11-03 257: if reply != None:
ddbf5288b9 2010-11-03 258: self.writeline('{} {}\n'.format(id, reply))
ddbf5288b9 2010-11-03 259: return(True)
ddbf5288b9 2010-11-03 260: self.writeline('{}\n'.format(id))
ddbf5288b9 2010-11-03 261:
0ef24b1937 2011-04-06 262: def check(self):
0ef24b1937 2011-04-06 263: while True:
0ef24b1937 2011-04-06 264: line = self._queue.get()
0ef24b1937 2011-04-06 265: if line == None:
0ef24b1937 2011-04-06 266: break
39b97ced92 2011-06-05 267: #self._log.info('request: ' + line)
0ef24b1937 2011-04-06 268: request = self._request.match(line)
0ef24b1937 2011-04-06 269: if request:
0ef24b1937 2011-04-06 270: id = request.group(1)
0ef24b1937 2011-04-06 271: #proto = request.group(2)
0ef24b1937 2011-04-06 272: site = request.group(3)
0ef24b1937 2011-04-06 273: url_path = request.group(4)
0ef24b1937 2011-04-06 274: ip_address = request.group(5)
0ef24b1937 2011-04-06 275: self.process(id, site, ip_address, url_path, line)
0ef24b1937 2011-04-06 276: else:
0ef24b1937 2011-04-06 277: self._log.info('bad request\n')
0ef24b1937 2011-04-06 278: self.writeline(line + '\n')
ddbf5288b9 2010-11-03 279:
ddbf5288b9 2010-11-03 280: def writeline(self, string):
ddbf5288b9 2010-11-03 281: self._log.info('sending: ' + string)
ddbf5288b9 2010-11-03 282: sys.stdout.write(string)
ddbf5288b9 2010-11-03 283: sys.stdout.flush()
ddbf5288b9 2010-11-03 284:
ddbf5288b9 2010-11-03 285: def loop(self):
0ef24b1937 2011-04-06 286: pool = gevent.pool.Pool()
0ef24b1937 2011-04-06 287: pool.spawn(self.check)
0ef24b1937 2011-04-06 288: pool.join()
d2c7ba18a4 2011-09-14 289:
d2c7ba18a4 2011-09-14 290: # this classes processes config file and substitutes default values
d2c7ba18a4 2011-09-14 291: class Config:
d2c7ba18a4 2011-09-14 292: __slots__ = frozenset(['_config', '_default', '_section', 'options'])
d2c7ba18a4 2011-09-14 293: _default = {
d2c7ba18a4 2011-09-14 294: 'log': {
d2c7ba18a4 2011-09-14 295: 'silent': 'no',
d2c7ba18a4 2011-09-14 296: },
d2c7ba18a4 2011-09-14 297: 'database': {
d2c7ba18a4 2011-09-14 298: 'host': None,
d2c7ba18a4 2011-09-14 299: 'database': 'squidTag',
d2c7ba18a4 2011-09-14 300: },}
d2c7ba18a4 2011-09-14 301:
d2c7ba18a4 2011-09-14 302: # function to read in config file
d2c7ba18a4 2011-09-14 303: def __init__(self):
d2c7ba18a4 2011-09-14 304: import ConfigParser, optparse, os
d2c7ba18a4 2011-09-14 305:
d2c7ba18a4 2011-09-14 306: parser = optparse.OptionParser()
d2c7ba18a4 2011-09-14 307: parser.add_option('-c', '--config', dest = 'config',
d2c7ba18a4 2011-09-14 308: help = 'config file location', metavar = 'FILE',
d2c7ba18a4 2011-09-14 309: default = '/usr/local/etc/squid-tagger.conf')
d2c7ba18a4 2011-09-14 310: parser.add_option('-d', '--dump', dest = 'dump',
d2c7ba18a4 2011-09-14 311: help = 'dump database', action = 'store_true', metavar = 'bool',
d2c7ba18a4 2011-09-14 312: default = False)
d2c7ba18a4 2011-09-14 313: parser.add_option('-f', '--flush-database', dest = 'flush_db',
d2c7ba18a4 2011-09-14 314: help = 'flush previous database on load', default = False,
d2c7ba18a4 2011-09-14 315: action = 'store_true', metavar = 'bool')
d2c7ba18a4 2011-09-14 316: parser.add_option('-l', '--load', dest = 'load',
d2c7ba18a4 2011-09-14 317: help = 'load database', action = 'store_true', metavar = 'bool',
d2c7ba18a4 2011-09-14 318: default = False)
d2c7ba18a4 2011-09-14 319: parser.add_option('-D', '--dump-conf', dest = 'dump_conf',
d2c7ba18a4 2011-09-14 320: help = 'dump filtering rules', default = False, metavar = 'bool',
d2c7ba18a4 2011-09-14 321: action = 'store_true')
d2c7ba18a4 2011-09-14 322: parser.add_option('-L', '--load-conf', dest = 'load_conf',
d2c7ba18a4 2011-09-14 323: help = 'load filtering rules', default = False, metavar = 'bool',
d2c7ba18a4 2011-09-14 324: action = 'store_true')
d2c7ba18a4 2011-09-14 325:
d2c7ba18a4 2011-09-14 326: (self.options, args) = parser.parse_args()
d2c7ba18a4 2011-09-14 327:
d2c7ba18a4 2011-09-14 328: assert os.access(self.options.config, os.R_OK), "Fatal error: can't read {}".format(self.options.config)
d2c7ba18a4 2011-09-14 329:
d2c7ba18a4 2011-09-14 330: self._config = ConfigParser.ConfigParser()
d2c7ba18a4 2011-09-14 331: self._config.readfp(open(self.options.config))
d2c7ba18a4 2011-09-14 332:
d2c7ba18a4 2011-09-14 333: # function to select config file section or create one
d2c7ba18a4 2011-09-14 334: def section(self, section):
d2c7ba18a4 2011-09-14 335: if not self._config.has_section(section):
d2c7ba18a4 2011-09-14 336: self._config.add_section(section)
d2c7ba18a4 2011-09-14 337: self._section = section
d2c7ba18a4 2011-09-14 338:
d2c7ba18a4 2011-09-14 339: # function to get config parameter, if parameter doesn't exists the default
d2c7ba18a4 2011-09-14 340: # value or None is substituted
d2c7ba18a4 2011-09-14 341: def __getitem__(self, name):
d2c7ba18a4 2011-09-14 342: if not self._config.has_option(self._section, name):
d2c7ba18a4 2011-09-14 343: if self._section in self._default:
d2c7ba18a4 2011-09-14 344: if name in self._default[self._section]:
d2c7ba18a4 2011-09-14 345: self._config.set(self._section, name, self._default[self._section][name])
d2c7ba18a4 2011-09-14 346: else:
d2c7ba18a4 2011-09-14 347: self._config.set(self._section, name, None)
d2c7ba18a4 2011-09-14 348: else:
d2c7ba18a4 2011-09-14 349: self._config.set(self._section, name, None)
d2c7ba18a4 2011-09-14 350: return(self._config.get(self._section, name))
d2c7ba18a4 2011-09-14 351:
d2c7ba18a4 2011-09-14 352: # initializing and reading in config file
d2c7ba18a4 2011-09-14 353: config = Config()
d500448801 2009-10-05 354:
d301d9adc6 2010-08-13 355: if config.options.dump or config.options.load or config.options.dump_conf or config.options.load_conf:
d301d9adc6 2010-08-13 356: import csv
d301d9adc6 2010-08-13 357:
d301d9adc6 2010-08-13 358: tagdb = tagDB()
bde51dc0c7 2010-08-26 359: data_fields = ['site', 'tag', 'regexp']
d301d9adc6 2010-08-13 360: conf_fields = ['netmask', 'redirect_url', 'from_weekday', 'to_weekday', 'from_time', 'to_time', 'tag']
d301d9adc6 2010-08-13 361:
d301d9adc6 2010-08-13 362: if config.options.dump or config.options.dump_conf:
0ef24b1937 2011-04-06 363: csv_writer = csv.writer(sys.stdout)
d301d9adc6 2010-08-13 364: if config.options.dump:
bde51dc0c7 2010-08-26 365: dump = tagdb.dump()
bde51dc0c7 2010-08-26 366: elif config.options.dump_conf:
bde51dc0c7 2010-08-26 367: dump = tagdb.dump_conf()
bde51dc0c7 2010-08-26 368:
0ef24b1937 2011-04-06 369: csv_writer.writerow(dump[0])
0ef24b1937 2011-04-06 370: for line in dump[1]:
0ef24b1937 2011-04-06 371: csv_writer.writerow(line)
d301d9adc6 2010-08-13 372:
d301d9adc6 2010-08-13 373: elif config.options.load or config.options.load_conf:
d301d9adc6 2010-08-13 374: csv_reader = csv.reader(sys.stdin)
d301d9adc6 2010-08-13 375: first_row = next(csv_reader)
d301d9adc6 2010-08-13 376:
d301d9adc6 2010-08-13 377: if config.options.load:
bde51dc0c7 2010-08-26 378: fields = data_fields
bde51dc0c7 2010-08-26 379: load = tagdb.load
bde51dc0c7 2010-08-26 380: elif config.options.load_conf:
bde51dc0c7 2010-08-26 381: fields = conf_fields
bde51dc0c7 2010-08-26 382: load = tagdb.load_conf
bde51dc0c7 2010-08-26 383:
bde51dc0c7 2010-08-26 384: assert first_row == fields, 'File must contain csv data with theese columns: ' + repr(fields)
bde51dc0c7 2010-08-26 385: load(csv_reader)
d301d9adc6 2010-08-13 386:
d301d9adc6 2010-08-13 387: else:
d301d9adc6 2010-08-13 388: # main loop
39b97ced92 2011-06-05 389: Checker(stdin, logger).loop()