Overview
Comment: | logging fully rewritten to use sockets instead of syscalls |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | master | trunk |
Files: | files | file ages | folders |
SHA3-256: |
fad48b740cb1d0b7b377af11b52108db |
User & Date: | arcade@b1t.name on 2012-07-07 15:08:27.000 |
Other Links: | branch diff | manifest | tags |
Context
2012-07-09
| ||
11:26 | added gevent wrapper for SysLogHandler automatically recode all log lines from utf-8 to str simplify invoking added some comment check-in: 2654b86697 user: arcade@b1t.name tags: master, trunk | |
2012-07-07
| ||
15:08 | logging fully rewritten to use sockets instead of syscalls check-in: fad48b740c user: arcade@b1t.name tags: master, trunk | |
13:24 | new class for writing output asynchronously check-in: d823fa83dd user: c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 tags: master, trunk | |
Changes
Modified squid-tagger.py
from [ab921f1976]
to [251db6bc04].
︙ | ︙ | |||
91 92 93 94 95 96 97 | else: self._config.set(self._section, name, None) return(self._config.get(self._section, name)) # initializing and reading in config file config = Config() | < < < < < < < < < | < | | < < < | < < < | | | 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | else: self._config.set(self._section, name, None) return(self._config.get(self._section, name)) # initializing and reading in config file config = Config() import logging, logging.handlers logger = logging.getLogger('squidTag') logger.setLevel(logging.INFO) handler = logging.handlers.SysLogHandler('/dev/log') handler.setFormatter(logging.Formatter(str('squidTag[%(process)s]: %(message)s'))) logger.addHandler(handler) # tiny wrapper around a file to make reads from it geventable # or should i move this somewhere? class FReadlineQueue(gevent.queue.Queue): # storing file descriptor, leftover __slots__ = frozenset(['_fd', '_tail']) |
︙ | ︙ | |||
151 152 153 154 155 156 157 | if len(self._tail) > 0: rows[0] = self._tail + rows[0] # popping out last (incomplete) element self._tail = rows.pop(-1) # dropping all complete elements to the queue for row in rows: self.put_nowait(row) | | | 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | if len(self._tail) > 0: rows[0] = self._tail + rows[0] # popping out last (incomplete) element self._tail = rows.pop(-1) # dropping all complete elements to the queue for row in rows: self.put_nowait(row) logger.info(str('< ' + row)) if len(buf) > 0: # no EOF, reinstalling event handler gevent.core.read_event(self._fd.fileno(), self._wait_helper) else: # EOF found, sending EOF to queue self.put_nowait(None) |
︙ | ︙ | |||
279 280 281 282 283 284 285 | # abstract class with basic checking functionality class Checker(object): __slots__ = frozenset(['_db', '_log', '_queue', '_request', '_stdout']) def __init__(self, queue, logger): self._db = tagDB() self._log = logger | | | | | | | | | | 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 | # abstract class with basic checking functionality class Checker(object): __slots__ = frozenset(['_db', '_log', '_queue', '_request', '_stdout']) def __init__(self, queue, logger): self._db = tagDB() self._log = logger self._log.info(str('started')) self._request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$') self._queue = queue self._stdout = FWritelineQueue(sys.stdout, False) def process(self, id, site, ip_address, url_path, line = None): #self._log.info(str('trying {}'.format(site))) result = self._db.check(site, ip_address) reply = None #self._log.info(str('got {} lines from database'.format(len(result)))) for row in result: if row != None and row[0] != None: if row[1] != None: self._log.info(str('trying regexp "{}" versus "{}"'.format(row[1], url_path))) try: if re.compile(row[1]).match(url_path): reply = row[0].format(url_path) else: continue except: self._log.info(str("can't compile regexp")) else: reply = row[0].format(url_path) if reply != None: self.writeline('{} {}'.format(id, reply)) return(True) self.writeline('{}'.format(id)) def check(self): while True: line = self._queue.get() if line == None: break #self._log.info(str('request: ' + line)) request = self._request.match(line) if request: id = request.group(1) #proto = request.group(2) site = request.group(3) url_path = request.group(4) ip_address = request.group(5) self.process(id, site, ip_address, url_path, line) else: self._log.info(str('bad request')) self.writeline(line) def writeline(self, string): self._log.info(str('> ' + string)) self._stdout.put(string) def loop(self): pool = gevent.pool.Pool() pool.spawn(self.check) pool.join() |
︙ | ︙ |