d500448801 2009-10-05 c.kworr@d4daf: #!/usr/bin/env python3.1
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: import configparser, optparse, os, postgresql.api, re, sys, _thread
d500448801 2009-10-05 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: # wrapper around syslog, can be muted
d500448801 2009-10-05 c.kworr@d4daf: class Logger:
d500448801 2009-10-05 c.kworr@d4daf: __slots__ = frozenset(['_syslog'])
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def __init__(self):
d500448801 2009-10-05 c.kworr@d4daf: config.section('log')
d500448801 2009-10-05 c.kworr@d4daf: if config['silent'] == 'yes':
d500448801 2009-10-05 c.kworr@d4daf: self._syslog = None
d500448801 2009-10-05 c.kworr@d4daf: else:
d500448801 2009-10-05 c.kworr@d4daf: import syslog
d500448801 2009-10-05 c.kworr@d4daf: self._syslog = syslog
d500448801 2009-10-05 c.kworr@d4daf: self._syslog.openlog('squidTag')
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def info(self, message):
4b22e25f24 2009-10-07 c.kworr@d4daf: if self._syslog:
d500448801 2009-10-05 c.kworr@d4daf: self._syslog.syslog(self._syslog.LOG_INFO, message)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def notice(self, message):
4b22e25f24 2009-10-07 c.kworr@d4daf: if self._syslog:
d500448801 2009-10-05 c.kworr@d4daf: self._syslog.syslog(self._syslog.LOG_NOTICE, message)
d500448801 2009-10-05 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: # wrapper around database
d500448801 2009-10-05 c.kworr@d4daf: class tagDB:
d500448801 2009-10-05 c.kworr@d4daf: __slots__ = frozenset(['_prepared', '_check_stmt', '_db'])
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def __init__(self):
d500448801 2009-10-05 c.kworr@d4daf: self._prepared = set()
d500448801 2009-10-05 c.kworr@d4daf: self._db = False
88c03b5440 2009-10-09 c.kworr@d4daf: self._check_stmt = self._curs().prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc")
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def _curs(self):
d500448801 2009-10-05 c.kworr@d4daf: if not self._db:
d500448801 2009-10-05 c.kworr@d4daf: config.section('database')
d500448801 2009-10-05 c.kworr@d4daf: self._db = postgresql.open(
442d7bf53a 2009-10-12 c.kworr@d4daf: 'pq://{}:{}@{}/{}'.format(
d500448801 2009-10-05 c.kworr@d4daf: config['user'],
d500448801 2009-10-05 c.kworr@d4daf: config['password'],
d500448801 2009-10-05 c.kworr@d4daf: config['host'],
d500448801 2009-10-05 c.kworr@d4daf: config['database'],
d500448801 2009-10-05 c.kworr@d4daf: ) )
d500448801 2009-10-05 c.kworr@d4daf: return(self._db)
d500448801 2009-10-05 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: def check(self, site, ip_address):
b93dc49210 2009-10-13 c.kworr@d4daf: return(self._check_stmt(site, ip_address))
b93dc49210 2009-10-13 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: # abstract class with basic checking functionality
b93dc49210 2009-10-13 c.kworr@d4daf: class Checker:
ed7808827d 2009-10-14 c.kworr@d4daf: __slots__ = frozenset(['_db', '_log'])
b93dc49210 2009-10-13 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: def __init__(self):
b93dc49210 2009-10-13 c.kworr@d4daf: self._db = tagDB()
b93dc49210 2009-10-13 c.kworr@d4daf: self._log = Logger()
b93dc49210 2009-10-13 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: def process(self, id, site, ip_address, url_path, line = None):
b93dc49210 2009-10-13 c.kworr@d4daf: self._log.info('trying {}\n'.format(site))
b93dc49210 2009-10-13 c.kworr@d4daf: result = self._db.check(site, ip_address)
b93dc49210 2009-10-13 c.kworr@d4daf: #reply = '{}://{}/{}'.format(req[4], req[1], req[3])
b93dc49210 2009-10-13 c.kworr@d4daf: reply = '-'
b93dc49210 2009-10-13 c.kworr@d4daf: for row in result:
b93dc49210 2009-10-13 c.kworr@d4daf: if row != None and row[0] != None:
b93dc49210 2009-10-13 c.kworr@d4daf: if row[1] != None:
b93dc49210 2009-10-13 c.kworr@d4daf: self._log.info('trying regexp "{}" versus "{}"\n'.format(row[1], url_path))
d2c54d0451 2010-03-01 c.kworr@d4daf: try:
d2c54d0451 2010-03-01 c.kworr@d4daf: if re.compile(row[1]).match(url_path):
d2c54d0451 2010-03-01 c.kworr@d4daf: reply = '302:' + row[0]
d2c54d0451 2010-03-01 c.kworr@d4daf: break
d2c54d0451 2010-03-01 c.kworr@d4daf: else:
d2c54d0451 2010-03-01 c.kworr@d4daf: continue
d2c54d0451 2010-03-01 c.kworr@d4daf: except:
d2c54d0451 2010-03-01 c.kworr@d4daf: self._log.info("can't compile regexp")
b93dc49210 2009-10-13 c.kworr@d4daf: else:
b93dc49210 2009-10-13 c.kworr@d4daf: reply = '302:' + row[0]
b93dc49210 2009-10-13 c.kworr@d4daf: break
b93dc49210 2009-10-13 c.kworr@d4daf: self.writeline('{} {}\n'.format(id, reply))
7e3418d94f 2009-10-12 c.kworr@d4daf:
7e3418d94f 2009-10-12 c.kworr@d4daf: def check(self, line):
7e3418d94f 2009-10-12 c.kworr@d4daf: request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$').match(line)
7e3418d94f 2009-10-12 c.kworr@d4daf: if request:
7e3418d94f 2009-10-12 c.kworr@d4daf: id = request.group(1)
7e3418d94f 2009-10-12 c.kworr@d4daf: #proto = request.group(2)
7e3418d94f 2009-10-12 c.kworr@d4daf: site = request.group(3)
7e3418d94f 2009-10-12 c.kworr@d4daf: url_path = request.group(4)
7e3418d94f 2009-10-12 c.kworr@d4daf: ip_address = request.group(5)
ed7808827d 2009-10-14 c.kworr@d4daf: self.process(id, site, ip_address, url_path, line)
7e3418d94f 2009-10-12 c.kworr@d4daf: else:
7e3418d94f 2009-10-12 c.kworr@d4daf: self._log.info('bad request\n')
b93dc49210 2009-10-13 c.kworr@d4daf: self.writeline(line)
b93dc49210 2009-10-13 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: def writeline(self, string):
b93dc49210 2009-10-13 c.kworr@d4daf: self._log.info('sending: ' + string)
b93dc49210 2009-10-13 c.kworr@d4daf: sys.stdout.write(string)
b93dc49210 2009-10-13 c.kworr@d4daf: sys.stdout.flush()
b93dc49210 2009-10-13 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: def loop(self):
ed7808827d 2009-10-14 c.kworr@d4daf: while True:
ed7808827d 2009-10-14 c.kworr@d4daf: line = sys.stdin.readline()
ed7808827d 2009-10-14 c.kworr@d4daf: if len(line) == 0:
ed7808827d 2009-10-14 c.kworr@d4daf: break
ed7808827d 2009-10-14 c.kworr@d4daf: self.check(line)
ed7808827d 2009-10-14 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: # threaded checking facility
b93dc49210 2009-10-13 c.kworr@d4daf: class CheckerThread(Checker):
ed7808827d 2009-10-14 c.kworr@d4daf: __slots__ = frozenset(['_lock', '_lock_exit', '_lock_queue', '_queue'])
b93dc49210 2009-10-13 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: def __init__(self):
ed7808827d 2009-10-14 c.kworr@d4daf: # basic initialisation
b93dc49210 2009-10-13 c.kworr@d4daf: Checker.__init__(self)
ed7808827d 2009-10-14 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: # Spin lock. Loop acquires it on start then releases it when holding queue
b93dc49210 2009-10-13 c.kworr@d4daf: # lock. This way the thread proceeds without stops while queue has data and
b93dc49210 2009-10-13 c.kworr@d4daf: # gets stalled when no data present. The lock is released by queue writer
b93dc49210 2009-10-13 c.kworr@d4daf: # after storing something into the queue
b93dc49210 2009-10-13 c.kworr@d4daf: self._lock = _thread.allocate_lock()
ed7808827d 2009-10-14 c.kworr@d4daf: self._lock_exit = _thread.allocate_lock()
b93dc49210 2009-10-13 c.kworr@d4daf: self._lock_queue = _thread.allocate_lock()
b93dc49210 2009-10-13 c.kworr@d4daf: self._lock.acquire()
b93dc49210 2009-10-13 c.kworr@d4daf: self._queue = []
b93dc49210 2009-10-13 c.kworr@d4daf: _thread.start_new_thread(self._start, ())
b93dc49210 2009-10-13 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: def _start(self):
b93dc49210 2009-10-13 c.kworr@d4daf: while True:
b93dc49210 2009-10-13 c.kworr@d4daf: self._lock.acquire()
ed7808827d 2009-10-14 c.kworr@d4daf: with self._lock_queue:
ed7808827d 2009-10-14 c.kworr@d4daf: # yes this should be written this way, and yes, this is why I hate threading
ed7808827d 2009-10-14 c.kworr@d4daf: if len(self._queue) > 1:
ed7808827d 2009-10-14 c.kworr@d4daf: if self._lock.locked():
ed7808827d 2009-10-14 c.kworr@d4daf: self._lock.release()
ed7808827d 2009-10-14 c.kworr@d4daf: req = self._queue.pop(0)
ed7808827d 2009-10-14 c.kworr@d4daf: Checker.process(self, req[0], req[1], req[2], req[3])
ed7808827d 2009-10-14 c.kworr@d4daf: with self._lock_queue:
ed7808827d 2009-10-14 c.kworr@d4daf: if len(self._queue) == 0:
ed7808827d 2009-10-14 c.kworr@d4daf: if self._lock_exit.locked():
ed7808827d 2009-10-14 c.kworr@d4daf: self._lock_exit.release()
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: def process(self, id, site, ip_address, url_path, line):
ed7808827d 2009-10-14 c.kworr@d4daf: with self._lock_queue:
ed7808827d 2009-10-14 c.kworr@d4daf: self._queue.append((id, site, ip_address, url_path))
ed7808827d 2009-10-14 c.kworr@d4daf: self._log.info('request {} queued ({})\n'.format(id, line))
ed7808827d 2009-10-14 c.kworr@d4daf: if not self._lock_exit.locked():
ed7808827d 2009-10-14 c.kworr@d4daf: self._lock_exit.acquire()
ed7808827d 2009-10-14 c.kworr@d4daf: if self._lock.locked():
ed7808827d 2009-10-14 c.kworr@d4daf: self._lock.release()
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: def loop(self):
ed7808827d 2009-10-14 c.kworr@d4daf: while True:
ed7808827d 2009-10-14 c.kworr@d4daf: line = sys.stdin.readline()
ed7808827d 2009-10-14 c.kworr@d4daf: if len(line) == 0:
ed7808827d 2009-10-14 c.kworr@d4daf: break
ed7808827d 2009-10-14 c.kworr@d4daf: self.check(line)
ed7808827d 2009-10-14 c.kworr@d4daf: self._lock_exit.acquire()
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: # kqueue enable class for BSD's XXX broken for now
ed7808827d 2009-10-14 c.kworr@d4daf: class CheckerKqueue(Checker):
ed7808827d 2009-10-14 c.kworr@d4daf: __slots__ = frozenset(['_kq', '_select', '_queue'])
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: def __init__(self):
ed7808827d 2009-10-14 c.kworr@d4daf: # basic initialisation
ed7808827d 2009-10-14 c.kworr@d4daf: Checker.__init__(self)
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: # importing select module
ed7808827d 2009-10-14 c.kworr@d4daf: import select
ed7808827d 2009-10-14 c.kworr@d4daf: self._select = select
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: # kreating kqueue
ed7808827d 2009-10-14 c.kworr@d4daf: self._kq = self._select.kqueue()
ed7808827d 2009-10-14 c.kworr@d4daf: assert (self._kq.fileno() != -1)
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: # watching sys.stdin for data
ed7808827d 2009-10-14 c.kworr@d4daf: self._kq.control([self._select.kevent(sys.stdin, self._select.KQ_FILTER_READ, self._select.KQ_EV_ADD)], 0)
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: # creating data queue
ed7808827d 2009-10-14 c.kworr@d4daf: self._queue = []
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: def loop(self):
ed7808827d 2009-10-14 c.kworr@d4daf: # Wait for data by default
ed7808827d 2009-10-14 c.kworr@d4daf: timeout = None
ed7808827d 2009-10-14 c.kworr@d4daf: while True:
ed7808827d 2009-10-14 c.kworr@d4daf: # checking if there is any data
ed7808827d 2009-10-14 c.kworr@d4daf: kevs = self._kq.control(None, 1, timeout)
ed7808827d 2009-10-14 c.kworr@d4daf: if len(kevs) > 0:
ed7808827d 2009-10-14 c.kworr@d4daf: #kev = kevs[0]
ed7808827d 2009-10-14 c.kworr@d4daf: # XXX add some code to read only known data size and check for newlines
ed7808827d 2009-10-14 c.kworr@d4daf: line = sys.stdin.readline()
ed7808827d 2009-10-14 c.kworr@d4daf: # add data to the queue
ed7808827d 2009-10-14 c.kworr@d4daf: self.check(line)
ed7808827d 2009-10-14 c.kworr@d4daf: # don't wait for data, start processing
ed7808827d 2009-10-14 c.kworr@d4daf: timeout = 0
ed7808827d 2009-10-14 c.kworr@d4daf: else:
ed7808827d 2009-10-14 c.kworr@d4daf: req = self._queue.pop(0)
ed7808827d 2009-10-14 c.kworr@d4daf: Checker.process(self, req[0], req[1], req[2], req[3])
ed7808827d 2009-10-14 c.kworr@d4daf: if len(self._queue) == 0:
ed7808827d 2009-10-14 c.kworr@d4daf: # wait for data - we have nothing to process
ed7808827d 2009-10-14 c.kworr@d4daf: timeout = None
ed7808827d 2009-10-14 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: def process(self, id, site, ip_address, url_path, line):
ed7808827d 2009-10-14 c.kworr@d4daf: self._queue.append((id, site, ip_address, url_path))
ed7808827d 2009-10-14 c.kworr@d4daf: self._log.info('request {} queued ({})\n'.format(id, line))
7e3418d94f 2009-10-12 c.kworr@d4daf:
fc934cead1 2009-10-13 c.kworr@d4daf: # this classes processes config file and substitutes default values
d500448801 2009-10-05 c.kworr@d4daf: class Config:
b93dc49210 2009-10-13 c.kworr@d4daf: __slots__ = frozenset(['_config', '_default', '_section'])
b93dc49210 2009-10-13 c.kworr@d4daf: _default = {
b93dc49210 2009-10-13 c.kworr@d4daf: 'reactor': {
b93dc49210 2009-10-13 c.kworr@d4daf: 'reactor': 'thread',
b93dc49210 2009-10-13 c.kworr@d4daf: },
fc934cead1 2009-10-13 c.kworr@d4daf: 'log': {
fc934cead1 2009-10-13 c.kworr@d4daf: 'silent': 'no',
fc934cead1 2009-10-13 c.kworr@d4daf: },
fc934cead1 2009-10-13 c.kworr@d4daf: 'database': {
fc934cead1 2009-10-13 c.kworr@d4daf: 'host': 'localhost',
fc934cead1 2009-10-13 c.kworr@d4daf: 'database': 'squidTag',
fc934cead1 2009-10-13 c.kworr@d4daf: },}
d500448801 2009-10-05 c.kworr@d4daf:
fc934cead1 2009-10-13 c.kworr@d4daf: # function to read in config file
d500448801 2009-10-05 c.kworr@d4daf: def __init__(self):
d500448801 2009-10-05 c.kworr@d4daf: parser = optparse.OptionParser()
d500448801 2009-10-05 c.kworr@d4daf: parser.add_option('-c', '--config', dest = 'config',
d500448801 2009-10-05 c.kworr@d4daf: help = 'config file location', metavar = 'FILE',
d500448801 2009-10-05 c.kworr@d4daf: default = '/usr/local/etc/squid-tagger.conf')
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: (options, args) = parser.parse_args()
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: if not os.access(options.config, os.R_OK):
d500448801 2009-10-05 c.kworr@d4daf: print("Can't read {}: exitting".format(options.config))
d500448801 2009-10-05 c.kworr@d4daf: sys.exit(2)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: self._config = configparser.ConfigParser()
d500448801 2009-10-05 c.kworr@d4daf: self._config.readfp(open(options.config))
d500448801 2009-10-05 c.kworr@d4daf:
fc934cead1 2009-10-13 c.kworr@d4daf: # function to select config file section or create one
d500448801 2009-10-05 c.kworr@d4daf: def section(self, section):
fc934cead1 2009-10-13 c.kworr@d4daf: if not self._config.has_section(section):
fc934cead1 2009-10-13 c.kworr@d4daf: self._config.add_section(section)
d500448801 2009-10-05 c.kworr@d4daf: self._section = section
d500448801 2009-10-05 c.kworr@d4daf:
fc934cead1 2009-10-13 c.kworr@d4daf: # function to get config parameter, if parameter doesn't exists the default
fc934cead1 2009-10-13 c.kworr@d4daf: # value or None is substituted
d500448801 2009-10-05 c.kworr@d4daf: def __getitem__(self, name):
fc934cead1 2009-10-13 c.kworr@d4daf: if not self._config.has_option(self._section, name):
b93dc49210 2009-10-13 c.kworr@d4daf: if self._section in self._default:
b93dc49210 2009-10-13 c.kworr@d4daf: if name in self._default[self._section]:
fc934cead1 2009-10-13 c.kworr@d4daf: self._config.set(self._section, name, self._default[self._section][name])
fc934cead1 2009-10-13 c.kworr@d4daf: else:
fc934cead1 2009-10-13 c.kworr@d4daf: self._config.set(self._section, name, None)
fc934cead1 2009-10-13 c.kworr@d4daf: else:
fc934cead1 2009-10-13 c.kworr@d4daf: self._config.set(self._section, name, None)
b93dc49210 2009-10-13 c.kworr@d4daf: return(self._config.get(self._section, name))
d500448801 2009-10-05 c.kworr@d4daf:
fc934cead1 2009-10-13 c.kworr@d4daf: # initializing and reading in config file
d500448801 2009-10-05 c.kworr@d4daf: config = Config()
d500448801 2009-10-05 c.kworr@d4daf:
b93dc49210 2009-10-13 c.kworr@d4daf: config.section('reactor')
b93dc49210 2009-10-13 c.kworr@d4daf: if config['reactor'] == 'thread':
b93dc49210 2009-10-13 c.kworr@d4daf: checker = CheckerThread()
ed7808827d 2009-10-14 c.kworr@d4daf: elif config['reactor'] == 'plain':
ed7808827d 2009-10-14 c.kworr@d4daf: checker = Checker()
ed7808827d 2009-10-14 c.kworr@d4daf: elif config['reactor'] == 'kqueue':
ed7808827d 2009-10-14 c.kworr@d4daf: checker = CheckerKqueue()
b93dc49210 2009-10-13 c.kworr@d4daf:
ed7808827d 2009-10-14 c.kworr@d4daf: checker.loop()