Overview
Comment: | Some code rearrengements for future alternate reactors support. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | master | trunk |
Files: | files | file ages | folders |
SHA3-256: |
b93dc49210bfabae3e656bc2a85d4f64 |
User & Date: | c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 on 2009-10-13 15:21:38.000 |
Other Links: | branch diff | manifest | tags |
Context
2009-10-14
| ||
09:04 | Added 'or replace' for better function updating, reordered funtions to work as found in file. check-in: 67e762b39b user: c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 tags: master, trunk | |
2009-10-13
| ||
15:21 | Some code rearrengements for future alternate reactors support. check-in: b93dc49210 user: c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 tags: master, trunk | |
13:20 | Default values for config file processing, now script doesn't brakes if some fields are not present. check-in: fc934cead1 user: c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 tags: master, trunk | |
Changes
Modified squid-tagger.py
from [4e6c4ae05d]
to [b2dec98d3a].
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | #!/usr/bin/env python3.1 import configparser, optparse, os, postgresql.api, re, sys, _thread class Logger: __slots__ = frozenset(['_syslog']) def __init__(self): config.section('log') if config['silent'] == 'yes': self._syslog = None else: import syslog self._syslog = syslog self._syslog.openlog('squidTag') def info(self, message): if self._syslog: self._syslog.syslog(self._syslog.LOG_INFO, message) def notice(self, message): if self._syslog: self._syslog.syslog(self._syslog.LOG_NOTICE, message) class tagDB: __slots__ = frozenset(['_prepared', '_check_stmt', '_db']) def __init__(self): self._prepared = set() self._db = False self._check_stmt = self._curs().prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc") def _curs(self): if not self._db: config.section('database') self._db = postgresql.open( 'pq://{}:{}@{}/{}'.format( config['user'], config['password'], config['host'], config['database'], ) ) return(self._db) | > > | | > | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < | < < < < < < < < < < < < < < < < < < < < < | | | | | | < < < < < < < < < | | > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | #!/usr/bin/env python3.1 import configparser, optparse, os, postgresql.api, re, sys, _thread # wrapper around syslog, can be muted class Logger: __slots__ = frozenset(['_syslog']) def __init__(self): config.section('log') if config['silent'] == 'yes': self._syslog = None else: import syslog self._syslog = syslog self._syslog.openlog('squidTag') def info(self, message): if self._syslog: self._syslog.syslog(self._syslog.LOG_INFO, message) def notice(self, message): if self._syslog: self._syslog.syslog(self._syslog.LOG_NOTICE, message) # wrapper around database class tagDB: __slots__ = frozenset(['_prepared', '_check_stmt', '_db']) def __init__(self): self._prepared = set() self._db = False self._check_stmt = self._curs().prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc") def _curs(self): if not self._db: config.section('database') self._db = postgresql.open( 'pq://{}:{}@{}/{}'.format( config['user'], config['password'], config['host'], config['database'], ) ) return(self._db) def check(self, site, ip_address): return(self._check_stmt(site, ip_address)) # abstract class with basic checking functionality class Checker: __slots__ = frozenset(['_db', '_log', '_queue']) def __init__(self): self._db = tagDB() self._log = Logger() def process(self, id, site, ip_address, url_path): self._log.info('trying {}\n'.format(site)) result = self._db.check(site, ip_address) #reply = '{}://{}/{}'.format(req[4], req[1], req[3]) reply = '-' for row in result: if row != None and row[0] != None: if row[1] != None: self._log.info('trying regexp "{}" versus "{}"\n'.format(row[1], url_path)) if re.compile(row[1]).match(url_path): reply = '302:' + row[0] break else: continue else: reply = '302:' + row[0] break self.writeline('{} {}\n'.format(id, reply)) def check(self, line): request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$').match(line) if request: id = request.group(1) #proto = request.group(2) site = request.group(3) url_path = request.group(4) ip_address = request.group(5) self.insert(id, site, ip_address, url_path) self._log.info('request {} queued ({})\n'.format(id, line)) else: self._log.info('bad request\n') self.writeline(line) def insert(self, id, site, ip_address, url_path): self._queue.append((id, site, ip_address, url_path)) def writeline(self, string): self._log.info('sending: ' + string) sys.stdout.write(string) sys.stdout.flush() # threaded checking facility class CheckerThread(Checker): __slots__ = frozenset(['_lock', '_lock_queue']) def __init__(self): Checker.__init__(self) # Spin lock. Loop acquires it on start then releases it when holding queue # lock. This way the thread proceeds without stops while queue has data and # gets stalled when no data present. The lock is released by queue writer # after storing something into the queue self._lock = _thread.allocate_lock() self._lock_queue = _thread.allocate_lock() self._lock.acquire() self._queue = [] _thread.start_new_thread(self._start, ()) def _start(self): while True: self._lock.acquire() self._lock_queue.acquire() # yes this should be written this way, and yes, this is why I hate threading if len(self._queue) > 1 and self._lock.locked(): self._lock.release() req = self._queue.pop(0) self._lock_queue.release() self.process(req[0], req[1], req[2], req[3]) def insert(self, id, site, ip_address, url_path): self._lock_queue.acquire() Checker.insert(self, id, site, ip_address, url_path) if self._lock.locked(): self._lock.release() self._lock_queue.release() # this classes processes config file and substitutes default values class Config: __slots__ = frozenset(['_config', '_default', '_section']) _default = { 'reactor': { 'reactor': 'thread', }, 'log': { 'silent': 'no', }, 'database': { 'host': 'localhost', 'database': 'squidTag', },} |
︙ | ︙ | |||
145 146 147 148 149 150 151 | self._config.add_section(section) self._section = section # function to get config parameter, if parameter doesn't exists the default # value or None is substituted def __getitem__(self, name): if not self._config.has_option(self._section, name): | | | | | | | | 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | self._config.add_section(section) self._section = section # function to get config parameter, if parameter doesn't exists the default # value or None is substituted def __getitem__(self, name): if not self._config.has_option(self._section, name): if self._section in self._default: if name in self._default[self._section]: self._config.set(self._section, name, self._default[self._section][name]) else: self._config.set(self._section, name, None) else: self._config.set(self._section, name, None) return(self._config.get(self._section, name)) # initializing and reading in config file config = Config() config.section('reactor') if config['reactor'] == 'thread': checker = CheckerThread() while True: line = sys.stdin.readline() if len(line) == 0: break checker.check(line) |