d500448801 2009-10-05 c.kworr@d4daf: #!/usr/bin/env python3.1
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: import configparser, optparse, os, postgresql.api, re, sys, _thread
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: class Logger:
d500448801 2009-10-05 c.kworr@d4daf: __slots__ = frozenset(['_syslog'])
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def __init__(self):
d500448801 2009-10-05 c.kworr@d4daf: config.section('log')
d500448801 2009-10-05 c.kworr@d4daf: if config['silent'] == 'yes':
d500448801 2009-10-05 c.kworr@d4daf: self._syslog = None
d500448801 2009-10-05 c.kworr@d4daf: else:
d500448801 2009-10-05 c.kworr@d4daf: import syslog
d500448801 2009-10-05 c.kworr@d4daf: self._syslog = syslog
d500448801 2009-10-05 c.kworr@d4daf: self._syslog.openlog('squidTag')
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def info(self, message):
4b22e25f24 2009-10-07 c.kworr@d4daf: if self._syslog:
d500448801 2009-10-05 c.kworr@d4daf: self._syslog.syslog(self._syslog.LOG_INFO, message)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def notice(self, message):
4b22e25f24 2009-10-07 c.kworr@d4daf: if self._syslog:
d500448801 2009-10-05 c.kworr@d4daf: self._syslog.syslog(self._syslog.LOG_NOTICE, message)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: class tagDB:
d500448801 2009-10-05 c.kworr@d4daf: __slots__ = frozenset(['_prepared', '_check_stmt', '_db'])
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def __init__(self):
d500448801 2009-10-05 c.kworr@d4daf: self._prepared = set()
d500448801 2009-10-05 c.kworr@d4daf: self._db = False
d500448801 2009-10-05 c.kworr@d4daf: self._check_stmt = self._curs().prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc limit 1")
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def _curs(self):
d500448801 2009-10-05 c.kworr@d4daf: if not self._db:
d500448801 2009-10-05 c.kworr@d4daf: config.section('database')
d500448801 2009-10-05 c.kworr@d4daf: self._db = postgresql.open(
d500448801 2009-10-05 c.kworr@d4daf: 'pq://{0}:{1}@{2}/{3}'.format(
d500448801 2009-10-05 c.kworr@d4daf: config['user'],
d500448801 2009-10-05 c.kworr@d4daf: config['password'],
d500448801 2009-10-05 c.kworr@d4daf: config['host'],
d500448801 2009-10-05 c.kworr@d4daf: config['database'],
d500448801 2009-10-05 c.kworr@d4daf: ) )
d500448801 2009-10-05 c.kworr@d4daf: return(self._db)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def check(self, ip_address, site):
d500448801 2009-10-05 c.kworr@d4daf: result = self._check_stmt(site, ip_address)
d500448801 2009-10-05 c.kworr@d4daf: if len(result) > 0:
d500448801 2009-10-05 c.kworr@d4daf: return result[0]
d500448801 2009-10-05 c.kworr@d4daf: else:
d500448801 2009-10-05 c.kworr@d4daf: return None
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: class CheckerThread:
d500448801 2009-10-05 c.kworr@d4daf: __slots__ = frozenset(['_db', '_lock', '_lock_queue', '_log', '_queue'])
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def __init__(self, db, log):
d500448801 2009-10-05 c.kworr@d4daf: self._db = db
d500448801 2009-10-05 c.kworr@d4daf: self._log = log
d500448801 2009-10-05 c.kworr@d4daf: self._lock = _thread.allocate_lock()
d500448801 2009-10-05 c.kworr@d4daf: self._lock_queue = _thread.allocate_lock()
d500448801 2009-10-05 c.kworr@d4daf: self._lock.acquire()
d500448801 2009-10-05 c.kworr@d4daf: self._queue = []
d500448801 2009-10-05 c.kworr@d4daf: _thread.start_new_thread(self._start, ())
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def _start(self):
d500448801 2009-10-05 c.kworr@d4daf: while True:
d500448801 2009-10-05 c.kworr@d4daf: self._lock.acquire()
d500448801 2009-10-05 c.kworr@d4daf: self._lock_queue.acquire()
d500448801 2009-10-05 c.kworr@d4daf: if len(self._queue) > 1 and self._lock.locked():
d500448801 2009-10-05 c.kworr@d4daf: self._lock.release()
d500448801 2009-10-05 c.kworr@d4daf: req = self._queue.pop(0)
d500448801 2009-10-05 c.kworr@d4daf: self._lock_queue.release()
d500448801 2009-10-05 c.kworr@d4daf: self._log.info('trying %s\n'%req[1])
d500448801 2009-10-05 c.kworr@d4daf: row = self._db.check(req[2], req[1])
d500448801 2009-10-05 c.kworr@d4daf: if row != None and row[0] != None:
d500448801 2009-10-05 c.kworr@d4daf: if row[1] != None:
d500448801 2009-10-05 c.kworr@d4daf: self._log.info('trying regexp "{0}" versus "{1}"\n'.format(row[1], req[3]))
d500448801 2009-10-05 c.kworr@d4daf: if re.compile(row[1]).match(req[3]):
d500448801 2009-10-05 c.kworr@d4daf: writeline('%s 302:%s\n'%(req[0], row[0]))
d500448801 2009-10-05 c.kworr@d4daf: else:
d500448801 2009-10-05 c.kworr@d4daf: writeline('%s -\n'%req[0])
d500448801 2009-10-05 c.kworr@d4daf: else:
d500448801 2009-10-05 c.kworr@d4daf: writeline('%s 302:%s\n'%(req[0], row[0]))
d500448801 2009-10-05 c.kworr@d4daf: else:
d500448801 2009-10-05 c.kworr@d4daf: writeline('%s -\n'%req[0])
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def check(self, line):
d500448801 2009-10-05 c.kworr@d4daf: request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$').match(line)
d500448801 2009-10-05 c.kworr@d4daf: if request:
d500448801 2009-10-05 c.kworr@d4daf: id = request.group(1)
d500448801 2009-10-05 c.kworr@d4daf: site = request.group(3)
d500448801 2009-10-05 c.kworr@d4daf: url_path = request.group(4)
d500448801 2009-10-05 c.kworr@d4daf: ip_address = request.group(5)
d500448801 2009-10-05 c.kworr@d4daf: self._lock_queue.acquire()
d500448801 2009-10-05 c.kworr@d4daf: self._queue.append((id, site, ip_address, url_path))
d500448801 2009-10-05 c.kworr@d4daf: if self._lock.locked():
d500448801 2009-10-05 c.kworr@d4daf: self._lock.release()
d500448801 2009-10-05 c.kworr@d4daf: self._lock_queue.release()
d500448801 2009-10-05 c.kworr@d4daf: self._log.info('request %s queued (%s)\n'%(id, line))
d500448801 2009-10-05 c.kworr@d4daf: else:
d500448801 2009-10-05 c.kworr@d4daf: self._log.info('bad request\n')
d500448801 2009-10-05 c.kworr@d4daf: writeline(line)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def writeline(string):
d500448801 2009-10-05 c.kworr@d4daf: log.info('sending: %s'%string)
d500448801 2009-10-05 c.kworr@d4daf: sys.stdout.write(string)
d500448801 2009-10-05 c.kworr@d4daf: sys.stdout.flush()
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: class Config:
d500448801 2009-10-05 c.kworr@d4daf: __slots__ = frozenset(['_config', '_section'])
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def __init__(self):
d500448801 2009-10-05 c.kworr@d4daf: parser = optparse.OptionParser()
d500448801 2009-10-05 c.kworr@d4daf: parser.add_option('-c', '--config', dest = 'config',
d500448801 2009-10-05 c.kworr@d4daf: help = 'config file location', metavar = 'FILE',
d500448801 2009-10-05 c.kworr@d4daf: default = '/usr/local/etc/squid-tagger.conf')
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: (options, args) = parser.parse_args()
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: if not os.access(options.config, os.R_OK):
d500448801 2009-10-05 c.kworr@d4daf: print("Can't read {}: exitting".format(options.config))
d500448801 2009-10-05 c.kworr@d4daf: sys.exit(2)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: self._config = configparser.ConfigParser()
d500448801 2009-10-05 c.kworr@d4daf: self._config.readfp(open(options.config))
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def section(self, section):
d500448801 2009-10-05 c.kworr@d4daf: self._section = section
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: def __getitem__(self, name):
d500448801 2009-10-05 c.kworr@d4daf: return self._config.get(self._section, name)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: config = Config()
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: log = Logger()
d500448801 2009-10-05 c.kworr@d4daf: db = tagDB()
d500448801 2009-10-05 c.kworr@d4daf: checker = CheckerThread(db,log)
d500448801 2009-10-05 c.kworr@d4daf:
d500448801 2009-10-05 c.kworr@d4daf: while True:
d500448801 2009-10-05 c.kworr@d4daf: line = sys.stdin.readline()
d500448801 2009-10-05 c.kworr@d4daf: if len(line) == 0:
d500448801 2009-10-05 c.kworr@d4daf: break
d500448801 2009-10-05 c.kworr@d4daf: checker.check(line)