Squid url redirector

Annotation For squid-tagger.py
anonymous

Annotation For squid-tagger.py

Origin for each line in squid-tagger.py from check-in ddbf5288b9:

d500448801 2009-10-05    1: #!/usr/bin/env python3.1
d500448801 2009-10-05    2: 
ae30851739 2010-08-12    3: import postgresql.api, re, sys
d500448801 2009-10-05    4: 
b93dc49210 2009-10-13    5: # wrapper around syslog, can be muted
d500448801 2009-10-05    6: class Logger:
d500448801 2009-10-05    7: 	__slots__ = frozenset(['_syslog'])
d500448801 2009-10-05    8: 
d500448801 2009-10-05    9: 	def __init__(self):
d500448801 2009-10-05   10: 		config.section('log')
d500448801 2009-10-05   11: 		if config['silent'] == 'yes':
d500448801 2009-10-05   12: 			self._syslog = None
d500448801 2009-10-05   13: 		else:
d500448801 2009-10-05   14: 			import syslog
d500448801 2009-10-05   15: 			self._syslog = syslog
d500448801 2009-10-05   16: 			self._syslog.openlog('squidTag')
d500448801 2009-10-05   17: 
d500448801 2009-10-05   18: 	def info(self, message):
4b22e25f24 2009-10-07   19: 		if self._syslog:
d500448801 2009-10-05   20: 			self._syslog.syslog(self._syslog.LOG_INFO, message)
d500448801 2009-10-05   21: 
d500448801 2009-10-05   22: 	def notice(self, message):
4b22e25f24 2009-10-07   23: 		if self._syslog:
d500448801 2009-10-05   24: 			self._syslog.syslog(self._syslog.LOG_NOTICE, message)
d500448801 2009-10-05   25: 
b93dc49210 2009-10-13   26: # wrapper around database
d500448801 2009-10-05   27: class tagDB:
d301d9adc6 2010-08-13   28: 	__slots__ = frozenset(('_check_stmt', '_db'))
b93dc49210 2009-10-13   29: 
b93dc49210 2009-10-13   30: 	def __init__(self):
9450c03d41 2010-08-07   31: 		config.section('database')
9450c03d41 2010-08-07   32: 		self._db = postgresql.open(
9450c03d41 2010-08-07   33: 			'pq://{}:{}@{}/{}'.format(
9450c03d41 2010-08-07   34: 				config['user'],
9450c03d41 2010-08-07   35: 				config['password'],
9450c03d41 2010-08-07   36: 				config['host'],
9450c03d41 2010-08-07   37: 				config['database'],
9450c03d41 2010-08-07   38: 		) )
ae30851739 2010-08-12   39: 		self._check_stmt = None
b93dc49210 2009-10-13   40: 
b93dc49210 2009-10-13   41: 	def check(self, site, ip_address):
ae30851739 2010-08-12   42: 		if self._check_stmt == None:
ae30851739 2010-08-12   43: 			self._check_stmt = self._db.prepare("select redirect_url, regexp from site_rule where site <@ tripdomain($1) and netmask >> $2::text::inet order by array_length(site, 1) desc")
b93dc49210 2009-10-13   44: 		return(self._check_stmt(site, ip_address))
ae30851739 2010-08-12   45: 
ae30851739 2010-08-12   46: 	def dump(self):
bde51dc0c7 2010-08-26   47: 		return(self._db.prepare("copy (select untrip(site) as site, tag, regexp from urls order by site, tag) to stdout csv header")())
31e69c4237 2010-08-12   48: 
bde51dc0c7 2010-08-26   49: 	def load(self, data):
31e69c4237 2010-08-12   50: 		with self._db.xact():
31e69c4237 2010-08-12   51: 			if config.options.flush_db:
31e69c4237 2010-08-12   52: 				self._db.execute('delete from urls;')
bde51dc0c7 2010-08-26   53: 			insert = self._db.prepare("insert into urls (site, tag, regexp) values (tripdomain($1), $2::text::text[], $3)")
bde51dc0c7 2010-08-26   54: 			for row in data:
bde51dc0c7 2010-08-26   55: 				if len(row) == 2:
bde51dc0c7 2010-08-26   56: 					insert(row[0], row[1], None)
31e69c4237 2010-08-12   57: 				else:
bde51dc0c7 2010-08-26   58: 					insert(row[0], row[1], row[2])
bde51dc0c7 2010-08-26   59: 			self._db.execute("update urls set regexp = NULL where regexp = ''")
31e69c4237 2010-08-12   60: 		self._db.execute('vacuum analyze urls;')
d301d9adc6 2010-08-13   61: 
d301d9adc6 2010-08-13   62: 	def load_conf(self, csv_data):
d301d9adc6 2010-08-13   63: 		with self._db.xact():
d301d9adc6 2010-08-13   64: 			self._db.execute('delete from rules;')
bde51dc0c7 2010-08-26   65: 			insertconf = self._db.prepare("insert into rules (netmask, redirect_url, from_weekday, to_weekday, from_time, to_time, tag) values ($1::text::cidr, $2, $3, $4, $5::text::time, $6::text::time, $7::text::text[])")
d301d9adc6 2010-08-13   66: 			for row in csv_data:
d301d9adc6 2010-08-13   67: 				insertconf(row[0], row[1], int(row[2]), int(row[3]), row[4], row[5], row[6])
d301d9adc6 2010-08-13   68: 		self._db.execute('vacuum analyze rules;')
d301d9adc6 2010-08-13   69: 
d301d9adc6 2010-08-13   70: 	def dump_conf(self):
bde51dc0c7 2010-08-26   71: 		return(self._db.prepare("copy (select netmask, redirect_url, from_weekday, to_weekday, from_time, to_time, tag from rules) to stdout csv header")())
b93dc49210 2009-10-13   72: 
b93dc49210 2009-10-13   73: # abstract class with basic checking functionality
b93dc49210 2009-10-13   74: class Checker:
bde51dc0c7 2010-08-26   75: 	__slots__ = frozenset(['_db', '_log', '_request'])
b93dc49210 2009-10-13   76: 
b93dc49210 2009-10-13   77: 	def __init__(self):
b93dc49210 2009-10-13   78: 		self._db = tagDB()
b93dc49210 2009-10-13   79: 		self._log = Logger()
7c13294e9f 2010-08-07   80: 		self._log.info('started\n')
bde51dc0c7 2010-08-26   81: 		self._request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$')
b93dc49210 2009-10-13   82: 
ed7808827d 2009-10-14   83: 	def process(self, id, site, ip_address, url_path, line = None):
b93dc49210 2009-10-13   84: 		self._log.info('trying {}\n'.format(site))
b93dc49210 2009-10-13   85: 		result = self._db.check(site, ip_address)
ddbf5288b9 2010-11-03   86: 		reply = None
b93dc49210 2009-10-13   87: 		for row in result:
b93dc49210 2009-10-13   88: 			if row != None and row[0] != None:
b93dc49210 2009-10-13   89: 				if row[1] != None:
b93dc49210 2009-10-13   90: 					self._log.info('trying regexp "{}" versus "{}"\n'.format(row[1], url_path))
d2c54d0451 2010-03-01   91: 					try:
d2c54d0451 2010-03-01   92: 						if re.compile(row[1]).match(url_path):
1fa8a88371 2010-07-14   93: 							reply = row[0].format(url_path)
d2c54d0451 2010-03-01   94: 						else:
d2c54d0451 2010-03-01   95: 							continue
d2c54d0451 2010-03-01   96: 					except:
d2c54d0451 2010-03-01   97: 						self._log.info("can't compile regexp")
d2c54d0451 2010-03-01   98: 				else:
1fa8a88371 2010-07-14   99: 					reply = row[0].format(url_path)
ddbf5288b9 2010-11-03  100: 			if reply != None:
ddbf5288b9 2010-11-03  101: 				self.writeline('{} {}\n'.format(id, reply))
ddbf5288b9 2010-11-03  102: 				return(True)
ddbf5288b9 2010-11-03  103: 		self.writeline('{}\n'.format(id))
b93dc49210 2009-10-13  104: 
b93dc49210 2009-10-13  105: 	def check(self, line):
bde51dc0c7 2010-08-26  106: 		request = self._request.match(line)
b93dc49210 2009-10-13  107: 		if request:
b93dc49210 2009-10-13  108: 			id = request.group(1)
b93dc49210 2009-10-13  109: 			#proto = request.group(2)
b93dc49210 2009-10-13  110: 			site = request.group(3)
b93dc49210 2009-10-13  111: 			url_path = request.group(4)
b93dc49210 2009-10-13  112: 			ip_address = request.group(5)
ed7808827d 2009-10-14  113: 			self.process(id, site, ip_address, url_path, line)
26fc9b34d9 2010-08-07  114: 			return(True)
b93dc49210 2009-10-13  115: 		else:
b93dc49210 2009-10-13  116: 			self._log.info('bad request\n')
b93dc49210 2009-10-13  117: 			self.writeline(line)
26fc9b34d9 2010-08-07  118: 			return(False)
b93dc49210 2009-10-13  119: 
b93dc49210 2009-10-13  120: 	def writeline(self, string):
b93dc49210 2009-10-13  121: 		self._log.info('sending: ' + string)
b93dc49210 2009-10-13  122: 		sys.stdout.write(string)
b93dc49210 2009-10-13  123: 		sys.stdout.flush()
b93dc49210 2009-10-13  124: 
ed7808827d 2009-10-14  125: 	def loop(self):
ed7808827d 2009-10-14  126: 		while True:
ed7808827d 2009-10-14  127: 			line = sys.stdin.readline()
ed7808827d 2009-10-14  128: 			if len(line) == 0:
ed7808827d 2009-10-14  129: 				break
ed7808827d 2009-10-14  130: 			self.check(line)
ed7808827d 2009-10-14  131: 
b93dc49210 2009-10-13  132: # threaded checking facility
b93dc49210 2009-10-13  133: class CheckerThread(Checker):
ed7808827d 2009-10-14  134: 	__slots__ = frozenset(['_lock', '_lock_exit', '_lock_queue', '_queue'])
d500448801 2009-10-05  135: 
d500448801 2009-10-05  136: 	def __init__(self):
ae30851739 2010-08-12  137: 		import _thread
ae30851739 2010-08-12  138: 
ed7808827d 2009-10-14  139: 		# basic initialisation
b93dc49210 2009-10-13  140: 		Checker.__init__(self)
ed7808827d 2009-10-14  141: 
88c03b5440 2009-10-09  142: 		# Spin lock. Loop acquires it on start then releases it when holding queue
88c03b5440 2009-10-09  143: 		# lock. This way the thread proceeds without stops while queue has data and
88c03b5440 2009-10-09  144: 		# gets stalled when no data present. The lock is released by queue writer
88c03b5440 2009-10-09  145: 		# after storing something into the queue
d500448801 2009-10-05  146: 		self._lock = _thread.allocate_lock()
ed7808827d 2009-10-14  147: 		self._lock_exit = _thread.allocate_lock()
d500448801 2009-10-05  148: 		self._lock_queue = _thread.allocate_lock()
d500448801 2009-10-05  149: 		self._lock.acquire()
d500448801 2009-10-05  150: 		self._queue = []
d500448801 2009-10-05  151: 		_thread.start_new_thread(self._start, ())
d500448801 2009-10-05  152: 
d500448801 2009-10-05  153: 	def _start(self):
d500448801 2009-10-05  154: 		while True:
d500448801 2009-10-05  155: 			self._lock.acquire()
ed7808827d 2009-10-14  156: 			with self._lock_queue:
ed7808827d 2009-10-14  157: 				# yes this should be written this way, and yes, this is why I hate threading
ed7808827d 2009-10-14  158: 				if len(self._queue) > 1:
ed7808827d 2009-10-14  159: 					if self._lock.locked():
ed7808827d 2009-10-14  160: 						self._lock.release()
ed7808827d 2009-10-14  161: 				req = self._queue.pop(0)
ed7808827d 2009-10-14  162: 			Checker.process(self, req[0], req[1], req[2], req[3])
ed7808827d 2009-10-14  163: 			with self._lock_queue:
ed7808827d 2009-10-14  164: 				if len(self._queue) == 0:
ed7808827d 2009-10-14  165: 					if self._lock_exit.locked():
ed7808827d 2009-10-14  166: 						self._lock_exit.release()
ed7808827d 2009-10-14  167: 
ed7808827d 2009-10-14  168: 	def process(self, id, site, ip_address, url_path, line):
ed7808827d 2009-10-14  169: 		with self._lock_queue:
ed7808827d 2009-10-14  170: 			self._queue.append((id, site, ip_address, url_path))
ed7808827d 2009-10-14  171: 			self._log.info('request {} queued ({})\n'.format(id, line))
ed7808827d 2009-10-14  172: 			if not self._lock_exit.locked():
ed7808827d 2009-10-14  173: 				self._lock_exit.acquire()
ed7808827d 2009-10-14  174: 			if self._lock.locked():
ed7808827d 2009-10-14  175: 				self._lock.release()
ed7808827d 2009-10-14  176: 
ed7808827d 2009-10-14  177: 	def loop(self):
ed7808827d 2009-10-14  178: 		while True:
ed7808827d 2009-10-14  179: 			line = sys.stdin.readline()
ed7808827d 2009-10-14  180: 			if len(line) == 0:
ed7808827d 2009-10-14  181: 				break
ed7808827d 2009-10-14  182: 			self.check(line)
ed7808827d 2009-10-14  183: 		self._lock_exit.acquire()
ed7808827d 2009-10-14  184: 
26fc9b34d9 2010-08-07  185: # kqueue enabled class for BSD's
ed7808827d 2009-10-14  186: class CheckerKqueue(Checker):
ed7808827d 2009-10-14  187: 	__slots__ = frozenset(['_kq', '_select', '_queue'])
ed7808827d 2009-10-14  188: 
ed7808827d 2009-10-14  189: 	def __init__(self):
ed7808827d 2009-10-14  190: 		# basic initialisation
ed7808827d 2009-10-14  191: 		Checker.__init__(self)
ed7808827d 2009-10-14  192: 
ed7808827d 2009-10-14  193: 		# importing select module
ed7808827d 2009-10-14  194: 		import select
ed7808827d 2009-10-14  195: 		self._select = select
ed7808827d 2009-10-14  196: 
ed7808827d 2009-10-14  197: 		# kreating kqueue
ed7808827d 2009-10-14  198: 		self._kq = self._select.kqueue()
7c13294e9f 2010-08-07  199: 		assert self._kq.fileno() != -1, "Fatal error: can't initialise kqueue."
ed7808827d 2009-10-14  200: 
ed7808827d 2009-10-14  201: 		# watching sys.stdin for data
ed7808827d 2009-10-14  202: 		self._kq.control([self._select.kevent(sys.stdin, self._select.KQ_FILTER_READ, self._select.KQ_EV_ADD)], 0)
ed7808827d 2009-10-14  203: 
ed7808827d 2009-10-14  204: 		# creating data queue
ed7808827d 2009-10-14  205: 		self._queue = []
ed7808827d 2009-10-14  206: 
ed7808827d 2009-10-14  207: 	def loop(self):
ed7808827d 2009-10-14  208: 		# Wait for data by default
ed7808827d 2009-10-14  209: 		timeout = None
26fc9b34d9 2010-08-07  210: 		eof = False
26fc9b34d9 2010-08-07  211: 		buffer = ''
ed7808827d 2009-10-14  212: 		while True:
26fc9b34d9 2010-08-07  213: 			# checking if there is any data or witing for data to arrive
ed7808827d 2009-10-14  214: 			kevs = self._kq.control(None, 1, timeout)
7c13294e9f 2010-08-07  215: 
ae1c0114c1 2010-08-09  216: 			for kev in kevs:
ae1c0114c1 2010-08-09  217: 				if kev.filter == self._select.KQ_FILTER_READ and kev.data > 0:
ae1c0114c1 2010-08-09  218: 					# reading data in
ae1c0114c1 2010-08-09  219: 					new_buffer = sys.stdin.read(kev.data)
ae1c0114c1 2010-08-09  220: 					# if no data was sent - we have reached end of file
ae1c0114c1 2010-08-09  221: 					if len(new_buffer) == 0:
ae1c0114c1 2010-08-09  222: 						eof = True
ae1c0114c1 2010-08-09  223: 					else:
ae1c0114c1 2010-08-09  224: 						# adding current buffer to old buffer remains
ae1c0114c1 2010-08-09  225: 						buffer += new_buffer
ae1c0114c1 2010-08-09  226: 						# splitting to lines
ae1c0114c1 2010-08-09  227: 						lines = buffer.split('\n')
ae1c0114c1 2010-08-09  228: 						# last line that was not terminate by newline returns to buffer
ae1c0114c1 2010-08-09  229: 						buffer = lines[-1]
ae1c0114c1 2010-08-09  230: 						# an only if there was at least one newline
ae1c0114c1 2010-08-09  231: 						if len(lines) > 1:
ae1c0114c1 2010-08-09  232: 							for line in lines[:-1]:
ae1c0114c1 2010-08-09  233: 								# add data to the queue
ae1c0114c1 2010-08-09  234: 								if self.check(line + '\n'):
ae1c0114c1 2010-08-09  235: 									# don't wait for more data, start processing
ae1c0114c1 2010-08-09  236: 									timeout = 0
ae1c0114c1 2010-08-09  237: 
ae1c0114c1 2010-08-09  238: 				# detect end of stream and exit if possible
ae1c0114c1 2010-08-09  239: 				if kev.flags >> 15 == 1:
ae1c0114c1 2010-08-09  240: 					self._kq.control([self._select.kevent(sys.stdin, self._select.KQ_FILTER_READ, self._select.KQ_EV_DELETE)], 0)
ae1c0114c1 2010-08-09  241: 					eof = True
bde51dc0c7 2010-08-26  242: 					#timeout = 0
ae1c0114c1 2010-08-09  243: 
ae1c0114c1 2010-08-09  244: 			if len(kevs) == 0:
7c13294e9f 2010-08-07  245: 				if len(self._queue) > 0:
7c13294e9f 2010-08-07  246: 					# get one request and process it
26fc9b34d9 2010-08-07  247: 					req = self._queue.pop(0)
26fc9b34d9 2010-08-07  248: 					Checker.process(self, req[0], req[1], req[2], req[3])
26fc9b34d9 2010-08-07  249: 					if len(self._queue) == 0:
26fc9b34d9 2010-08-07  250: 						# wait for data - we have nothing to process
26fc9b34d9 2010-08-07  251: 						timeout = None
7c13294e9f 2010-08-07  252: 
7c13294e9f 2010-08-07  253: 			# if queue is empty and we reached end of stream - we can exit
7c13294e9f 2010-08-07  254: 			if len(self._queue) == 0 and eof:
7c13294e9f 2010-08-07  255: 				break
ed7808827d 2009-10-14  256: 
ed7808827d 2009-10-14  257: 	def process(self, id, site, ip_address, url_path, line):
26fc9b34d9 2010-08-07  258: 		# simply adding data to the queue
ed7808827d 2009-10-14  259: 		self._queue.append((id, site, ip_address, url_path))
ed7808827d 2009-10-14  260: 		self._log.info('request {} queued ({})\n'.format(id, line))
fc934cead1 2009-10-13  261: 
fc934cead1 2009-10-13  262: # this classes processes config file and substitutes default values
d500448801 2009-10-05  263: class Config:
ae30851739 2010-08-12  264: 	__slots__ = frozenset(['_config', '_default', '_section', 'options'])
b93dc49210 2009-10-13  265: 	_default = {
b93dc49210 2009-10-13  266: 		'reactor': {
b93dc49210 2009-10-13  267: 			'reactor': 'thread',
b93dc49210 2009-10-13  268: 		},
fc934cead1 2009-10-13  269: 		'log': {
fc934cead1 2009-10-13  270: 			'silent': 'no',
fc934cead1 2009-10-13  271: 		},
fc934cead1 2009-10-13  272: 		'database': {
fc934cead1 2009-10-13  273: 			'host': 'localhost',
fc934cead1 2009-10-13  274: 			'database': 'squidTag',
fc934cead1 2009-10-13  275: 	},}
d500448801 2009-10-05  276: 
fc934cead1 2009-10-13  277: 	# function to read in config file
d500448801 2009-10-05  278: 	def __init__(self):
ae30851739 2010-08-12  279: 		import configparser, optparse, os
ae30851739 2010-08-12  280: 
d500448801 2009-10-05  281: 		parser = optparse.OptionParser()
d500448801 2009-10-05  282: 		parser.add_option('-c', '--config', dest = 'config',
d500448801 2009-10-05  283: 			help = 'config file location', metavar = 'FILE',
d500448801 2009-10-05  284: 			default = '/usr/local/etc/squid-tagger.conf')
ae30851739 2010-08-12  285: 		parser.add_option('-d', '--dump', dest = 'dump',
ae30851739 2010-08-12  286: 			help = 'dump database', action = 'store_true', metavar = 'bool',
ae30851739 2010-08-12  287: 			default = False)
31e69c4237 2010-08-12  288: 		parser.add_option('-f', '--flush-database', dest = 'flush_db',
31e69c4237 2010-08-12  289: 			help = 'flush previous database on load', default = False,
31e69c4237 2010-08-12  290: 			action = 'store_true', metavar = 'bool')
31e69c4237 2010-08-12  291: 		parser.add_option('-l', '--load', dest = 'load',
31e69c4237 2010-08-12  292: 			help = 'load database', action = 'store_true', metavar = 'bool',
31e69c4237 2010-08-12  293: 			default = False)
d301d9adc6 2010-08-13  294: 		parser.add_option('-D', '--dump-conf', dest = 'dump_conf',
d301d9adc6 2010-08-13  295: 			help = 'dump filtering rules', default = False, metavar = 'bool',
d301d9adc6 2010-08-13  296: 			action = 'store_true')
d301d9adc6 2010-08-13  297: 		parser.add_option('-L', '--load-conf', dest = 'load_conf',
d301d9adc6 2010-08-13  298: 			help = 'load filtering rules', default = False, metavar = 'bool',
d301d9adc6 2010-08-13  299: 			action = 'store_true')
d500448801 2009-10-05  300: 
ae30851739 2010-08-12  301: 		(self.options, args) = parser.parse_args()
d500448801 2009-10-05  302: 
ae30851739 2010-08-12  303: 		assert os.access(self.options.config, os.R_OK), "Fatal error: can't read {}".format(self.options.config)
d500448801 2009-10-05  304: 
d500448801 2009-10-05  305: 		self._config = configparser.ConfigParser()
ae30851739 2010-08-12  306: 		self._config.readfp(open(self.options.config))
d500448801 2009-10-05  307: 
fc934cead1 2009-10-13  308: 	# function to select config file section or create one
d500448801 2009-10-05  309: 	def section(self, section):
fc934cead1 2009-10-13  310: 		if not self._config.has_section(section):
fc934cead1 2009-10-13  311: 			self._config.add_section(section)
d500448801 2009-10-05  312: 		self._section = section
d500448801 2009-10-05  313: 
fc934cead1 2009-10-13  314: 	# function to get config parameter, if parameter doesn't exists the default
fc934cead1 2009-10-13  315: 	# value or None is substituted
d500448801 2009-10-05  316: 	def __getitem__(self, name):
fc934cead1 2009-10-13  317: 		if not self._config.has_option(self._section, name):
b93dc49210 2009-10-13  318: 			if self._section in self._default:
b93dc49210 2009-10-13  319: 				if name in self._default[self._section]:
fc934cead1 2009-10-13  320: 					self._config.set(self._section, name, self._default[self._section][name])
fc934cead1 2009-10-13  321: 				else:
fc934cead1 2009-10-13  322: 					self._config.set(self._section, name, None)
fc934cead1 2009-10-13  323: 			else:
fc934cead1 2009-10-13  324: 				self._config.set(self._section, name, None)
b93dc49210 2009-10-13  325: 		return(self._config.get(self._section, name))
d500448801 2009-10-05  326: 
fc934cead1 2009-10-13  327: # initializing and reading in config file
d500448801 2009-10-05  328: config = Config()
d500448801 2009-10-05  329: 
d301d9adc6 2010-08-13  330: if config.options.dump or config.options.load or config.options.dump_conf or config.options.load_conf:
d301d9adc6 2010-08-13  331: 	import csv
d301d9adc6 2010-08-13  332: 
d301d9adc6 2010-08-13  333: 	tagdb = tagDB()
bde51dc0c7 2010-08-26  334: 	data_fields = ['site', 'tag', 'regexp']
d301d9adc6 2010-08-13  335: 	conf_fields = ['netmask', 'redirect_url', 'from_weekday', 'to_weekday', 'from_time', 'to_time', 'tag']
d301d9adc6 2010-08-13  336: 
d301d9adc6 2010-08-13  337: 	if config.options.dump or config.options.dump_conf:
bde51dc0c7 2010-08-26  338: 		if config.options.dump:
bde51dc0c7 2010-08-26  339: 			dump = tagdb.dump()
bde51dc0c7 2010-08-26  340: 		elif config.options.dump_conf:
bde51dc0c7 2010-08-26  341: 			dump = tagdb.dump_conf()
bde51dc0c7 2010-08-26  342: 
bde51dc0c7 2010-08-26  343: 		for line in dump:
bde51dc0c7 2010-08-26  344: 			sys.stdout.write(line.decode('utf-8'))
d301d9adc6 2010-08-13  345: 
d301d9adc6 2010-08-13  346: 	elif config.options.load or config.options.load_conf:
d301d9adc6 2010-08-13  347: 		csv_reader = csv.reader(sys.stdin)
d301d9adc6 2010-08-13  348: 		first_row = next(csv_reader)
d301d9adc6 2010-08-13  349: 
d301d9adc6 2010-08-13  350: 		if config.options.load:
bde51dc0c7 2010-08-26  351: 			fields = data_fields
bde51dc0c7 2010-08-26  352: 			load = tagdb.load
bde51dc0c7 2010-08-26  353: 		elif config.options.load_conf:
bde51dc0c7 2010-08-26  354: 			fields = conf_fields
bde51dc0c7 2010-08-26  355: 			load = tagdb.load_conf
bde51dc0c7 2010-08-26  356: 
bde51dc0c7 2010-08-26  357: 		assert first_row == fields, 'File must contain csv data with theese columns: ' + repr(fields)
bde51dc0c7 2010-08-26  358: 		load(csv_reader)
ae30851739 2010-08-12  359: 
ae30851739 2010-08-12  360: else:
ae30851739 2010-08-12  361: 	# main loop
ae30851739 2010-08-12  362: 	config.section('reactor')
ae30851739 2010-08-12  363: 	if config['reactor'] == 'thread':
ae30851739 2010-08-12  364: 		checker = CheckerThread()
ae30851739 2010-08-12  365: 	elif config['reactor'] == 'plain':
ae30851739 2010-08-12  366: 		checker = Checker()
ae30851739 2010-08-12  367: 	elif config['reactor'] == 'kqueue':
ae30851739 2010-08-12  368: 		checker = CheckerKqueue()
ae30851739 2010-08-12  369: 
ae30851739 2010-08-12  370: 	checker.loop()