Overview
Comment: | added gevent wrapper for SysLogHandler automatically recode all log lines from utf-8 to str simplify invoking added some comment |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | master | trunk |
Files: | files | file ages | folders |
SHA3-256: |
2654b866976c1e1a160bdfafd3cae179 |
User & Date: | arcade@b1t.name on 2012-07-09 11:26:55.000 |
Other Links: | branch diff | manifest | tags |
Context
2012-07-09
| ||
14:25 | switched sg_import to python3.2 reworked SysLogHandlerQueue to stick to one socket reworked FReadlineQueue to use io.FileIO for file operation check-in: 67e8b3309d user: arcade@b1t.name tags: master, trunk | |
11:26 | added gevent wrapper for SysLogHandler automatically recode all log lines from utf-8 to str simplify invoking added some comment check-in: 2654b86697 user: arcade@b1t.name tags: master, trunk | |
2012-07-07
| ||
15:08 | logging fully rewritten to use sockets instead of syscalls check-in: fad48b740c user: arcade@b1t.name tags: master, trunk | |
Changes
Modified squid-tagger.py
from [251db6bc04]
to [faa884292f].
︙ | |||
92 93 94 95 96 97 98 99 100 | 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + | self._config.set(self._section, name, None) return(self._config.get(self._section, name)) # initializing and reading in config file config = Config() import logging, logging.handlers # wrapper around logging handler to make it queue records and don't stall when sending them class SysLogHandlerQueue(logging.handlers.SysLogHandler): __slots__ = frozenset(['_event', '_tail', '_workers']) def __init__(self): logging.handlers.SysLogHandler.__init__(self, '/dev/log') self._event = gevent.event.Event() self._event.set() self._tail = gevent.queue.Queue() self._workers = set() def emit(self, record): # my syslog is broken and cannot into UTF-8 BOM record.msg = str(record.msg) self._tail.put(record) if self._tail.qsize() != 0: # in case queue is empty we will spawn new worker # all workers are logged so we can kill them on close() self._workers.add(gevent.spawn(self._writer)) def _writer(self): # here we are locking the queue so we can be sure we are the only one self._event.wait() self._event.clear() while not self._tail.empty(): logging.handlers.SysLogHandler.emit(self, self._tail.get()) self._event.set() self._workers.remove(gevent.getcurrent()) def close(self): for worker in self._workers: gevent.kill(worker) logging.handlers.SysLogHandler.close(self) logger = logging.getLogger('squidTag') logger.setLevel(logging.INFO) |
︙ | |||
135 136 137 138 139 140 141 | 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | - + + + | if len(self._tail) > 0: rows[0] = self._tail + rows[0] # popping out last (incomplete) element self._tail = rows.pop(-1) # dropping all complete elements to the queue for row in rows: self.put_nowait(row) |
︙ | |||
263 264 265 266 267 268 269 | 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 | - + - + - + - + - + - + - + - + - + - - - - - | # abstract class with basic checking functionality class Checker(object): __slots__ = frozenset(['_db', '_log', '_queue', '_request', '_stdout']) def __init__(self, queue, logger): self._db = tagDB() self._log = logger |
︙ |