d500448801 2009-10-05 1: #!/usr/bin/env python3.1
d500448801 2009-10-05 2:
d500448801 2009-10-05 3: import configparser, optparse, os, postgresql.api, re, sys, _thread
d500448801 2009-10-05 4:
d500448801 2009-10-05 5: class Logger:
d500448801 2009-10-05 6: __slots__ = frozenset(['_syslog'])
d500448801 2009-10-05 7:
d500448801 2009-10-05 8: def __init__(self):
d500448801 2009-10-05 9: config.section('log')
d500448801 2009-10-05 10: if config['silent'] == 'yes':
d500448801 2009-10-05 11: self._syslog = None
d500448801 2009-10-05 12: else:
d500448801 2009-10-05 13: import syslog
d500448801 2009-10-05 14: self._syslog = syslog
d500448801 2009-10-05 15: self._syslog.openlog('squidTag')
d500448801 2009-10-05 16:
d500448801 2009-10-05 17: def info(self, message):
4b22e25f24 2009-10-07 18: if self._syslog:
d500448801 2009-10-05 19: self._syslog.syslog(self._syslog.LOG_INFO, message)
d500448801 2009-10-05 20:
d500448801 2009-10-05 21: def notice(self, message):
4b22e25f24 2009-10-07 22: if self._syslog:
d500448801 2009-10-05 23: self._syslog.syslog(self._syslog.LOG_NOTICE, message)
d500448801 2009-10-05 24:
d500448801 2009-10-05 25: class tagDB:
d500448801 2009-10-05 26: __slots__ = frozenset(['_prepared', '_check_stmt', '_db'])
d500448801 2009-10-05 27:
d500448801 2009-10-05 28: def __init__(self):
d500448801 2009-10-05 29: self._prepared = set()
d500448801 2009-10-05 30: self._db = False
88c03b5440 2009-10-09 31: self._check_stmt = self._curs().prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc")
d500448801 2009-10-05 32:
d500448801 2009-10-05 33: def _curs(self):
d500448801 2009-10-05 34: if not self._db:
d500448801 2009-10-05 35: config.section('database')
d500448801 2009-10-05 36: self._db = postgresql.open(
442d7bf53a 2009-10-12 37: 'pq://{}:{}@{}/{}'.format(
d500448801 2009-10-05 38: config['user'],
d500448801 2009-10-05 39: config['password'],
d500448801 2009-10-05 40: config['host'],
d500448801 2009-10-05 41: config['database'],
d500448801 2009-10-05 42: ) )
d500448801 2009-10-05 43: return(self._db)
d500448801 2009-10-05 44:
d500448801 2009-10-05 45: def check(self, ip_address, site):
88c03b5440 2009-10-09 46: return self._check_stmt(site, ip_address)
d500448801 2009-10-05 47:
d500448801 2009-10-05 48: class CheckerThread:
d500448801 2009-10-05 49: __slots__ = frozenset(['_db', '_lock', '_lock_queue', '_log', '_queue'])
d500448801 2009-10-05 50:
d500448801 2009-10-05 51: def __init__(self, db, log):
d500448801 2009-10-05 52: self._db = db
d500448801 2009-10-05 53: self._log = log
88c03b5440 2009-10-09 54: # Spin lock. Loop acquires it on start then releases it when holding queue
88c03b5440 2009-10-09 55: # lock. This way the thread proceeds without stops while queue has data and
88c03b5440 2009-10-09 56: # gets stalled when no data present. The lock is released by queue writer
88c03b5440 2009-10-09 57: # after storing something into the queue
d500448801 2009-10-05 58: self._lock = _thread.allocate_lock()
d500448801 2009-10-05 59: self._lock_queue = _thread.allocate_lock()
d500448801 2009-10-05 60: self._lock.acquire()
d500448801 2009-10-05 61: self._queue = []
d500448801 2009-10-05 62: _thread.start_new_thread(self._start, ())
d500448801 2009-10-05 63:
d500448801 2009-10-05 64: def _start(self):
d500448801 2009-10-05 65: while True:
d500448801 2009-10-05 66: self._lock.acquire()
d500448801 2009-10-05 67: self._lock_queue.acquire()
88c03b5440 2009-10-09 68: # yes this should be written this way, and yes, this is why I hate threading
d500448801 2009-10-05 69: if len(self._queue) > 1 and self._lock.locked():
d500448801 2009-10-05 70: self._lock.release()
d500448801 2009-10-05 71: req = self._queue.pop(0)
d500448801 2009-10-05 72: self._lock_queue.release()
442d7bf53a 2009-10-12 73: self._log.info('trying {}\n'.format(req[1]))
88c03b5440 2009-10-09 74: result = self._db.check(req[2], req[1])
7e3418d94f 2009-10-12 75: #reply = '{}://{}/{}'.format(req[4], req[1], req[3])
7e3418d94f 2009-10-12 76: reply = '-'
88c03b5440 2009-10-09 77: for row in result:
88c03b5440 2009-10-09 78: if row != None and row[0] != None:
88c03b5440 2009-10-09 79: if row[1] != None:
442d7bf53a 2009-10-12 80: self._log.info('trying regexp "{}" versus "{}"\n'.format(row[1], req[3]))
88c03b5440 2009-10-09 81: if re.compile(row[1]).match(req[3]):
7e3418d94f 2009-10-12 82: reply = '302:' + row[0]
88c03b5440 2009-10-09 83: break
88c03b5440 2009-10-09 84: else:
88c03b5440 2009-10-09 85: continue
88c03b5440 2009-10-09 86: else:
7e3418d94f 2009-10-12 87: reply = '302:' + row[0]
88c03b5440 2009-10-09 88: break
7e3418d94f 2009-10-12 89: writeline('{} {}\n'.format(req[0], reply))
d500448801 2009-10-05 90:
d500448801 2009-10-05 91: def check(self, line):
d500448801 2009-10-05 92: request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$').match(line)
d500448801 2009-10-05 93: if request:
d500448801 2009-10-05 94: id = request.group(1)
7e3418d94f 2009-10-12 95: #proto = request.group(2)
d500448801 2009-10-05 96: site = request.group(3)
d500448801 2009-10-05 97: url_path = request.group(4)
d500448801 2009-10-05 98: ip_address = request.group(5)
d500448801 2009-10-05 99: self._lock_queue.acquire()
7e3418d94f 2009-10-12 100: self._queue.append((id, site, ip_address, url_path))
d500448801 2009-10-05 101: if self._lock.locked():
d500448801 2009-10-05 102: self._lock.release()
d500448801 2009-10-05 103: self._lock_queue.release()
442d7bf53a 2009-10-12 104: self._log.info('request {} queued ({})\n'.format(id, line))
d500448801 2009-10-05 105: else:
d500448801 2009-10-05 106: self._log.info('bad request\n')
d500448801 2009-10-05 107: writeline(line)
d500448801 2009-10-05 108:
d500448801 2009-10-05 109: def writeline(string):
442d7bf53a 2009-10-12 110: log.info('sending: ' + string)
d500448801 2009-10-05 111: sys.stdout.write(string)
d500448801 2009-10-05 112: sys.stdout.flush()
d500448801 2009-10-05 113:
fc934cead1 2009-10-13 114: # this classes processes config file and substitutes default values
d500448801 2009-10-05 115: class Config:
fc934cead1 2009-10-13 116: __slots__ = frozenset(['_config', '_defaults', '_section'])
fc934cead1 2009-10-13 117: _defaults = {
fc934cead1 2009-10-13 118: 'log': {
fc934cead1 2009-10-13 119: 'silent': 'no',
fc934cead1 2009-10-13 120: },
fc934cead1 2009-10-13 121: 'database': {
fc934cead1 2009-10-13 122: 'host': 'localhost',
fc934cead1 2009-10-13 123: 'database': 'squidTag',
fc934cead1 2009-10-13 124: },}
d500448801 2009-10-05 125:
fc934cead1 2009-10-13 126: # function to read in config file
d500448801 2009-10-05 127: def __init__(self):
d500448801 2009-10-05 128: parser = optparse.OptionParser()
d500448801 2009-10-05 129: parser.add_option('-c', '--config', dest = 'config',
d500448801 2009-10-05 130: help = 'config file location', metavar = 'FILE',
d500448801 2009-10-05 131: default = '/usr/local/etc/squid-tagger.conf')
d500448801 2009-10-05 132:
d500448801 2009-10-05 133: (options, args) = parser.parse_args()
d500448801 2009-10-05 134:
d500448801 2009-10-05 135: if not os.access(options.config, os.R_OK):
d500448801 2009-10-05 136: print("Can't read {}: exitting".format(options.config))
d500448801 2009-10-05 137: sys.exit(2)
d500448801 2009-10-05 138:
d500448801 2009-10-05 139: self._config = configparser.ConfigParser()
d500448801 2009-10-05 140: self._config.readfp(open(options.config))
d500448801 2009-10-05 141:
fc934cead1 2009-10-13 142: # function to select config file section or create one
d500448801 2009-10-05 143: def section(self, section):
fc934cead1 2009-10-13 144: if not self._config.has_section(section):
fc934cead1 2009-10-13 145: self._config.add_section(section)
d500448801 2009-10-05 146: self._section = section
d500448801 2009-10-05 147:
fc934cead1 2009-10-13 148: # function to get config parameter, if parameter doesn't exists the default
fc934cead1 2009-10-13 149: # value or None is substituted
d500448801 2009-10-05 150: def __getitem__(self, name):
fc934cead1 2009-10-13 151: if not self._config.has_option(self._section, name):
fc934cead1 2009-10-13 152: if self._default.has_key(self._section):
fc934cead1 2009-10-13 153: if self._default[self._section].has_key(name):
fc934cead1 2009-10-13 154: self._config.set(self._section, name, self._default[self._section][name])
fc934cead1 2009-10-13 155: else:
fc934cead1 2009-10-13 156: self._config.set(self._section, name, None)
fc934cead1 2009-10-13 157: else:
fc934cead1 2009-10-13 158: self._config.set(self._section, name, None)
d500448801 2009-10-05 159: return self._config.get(self._section, name)
d500448801 2009-10-05 160:
fc934cead1 2009-10-13 161: # initializing and reading in config file
d500448801 2009-10-05 162: config = Config()
d500448801 2009-10-05 163:
d500448801 2009-10-05 164: log = Logger()
d500448801 2009-10-05 165: db = tagDB()
d500448801 2009-10-05 166: checker = CheckerThread(db,log)
d500448801 2009-10-05 167:
d500448801 2009-10-05 168: while True:
d500448801 2009-10-05 169: line = sys.stdin.readline()
d500448801 2009-10-05 170: if len(line) == 0:
d500448801 2009-10-05 171: break
d500448801 2009-10-05 172: checker.check(line)