Squid url redirector

Check-in [d823fa83dd]
anonymous

Check-in [d823fa83dd]

Overview
Comment:new class for writing output asynchronously
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: d823fa83dd34743be6f86ebd51decc20a56cc104d9991eecc2a69655c38bdefa
User & Date: c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 on 2012-07-07 13:24:15.000
Other Links: branch diff | manifest | tags
Context
2012-07-07
15:08
logging fully rewritten to use sockets instead of syscalls check-in: fad48b740c user: arcade@b1t.name tags: master, trunk
13:24
new class for writing output asynchronously check-in: d823fa83dd user: c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 tags: master, trunk
2011-09-14
07:46
there class was relocated, drop extra declaration check-in: 5ff0e0514c user: c.kworr@d4daf22a-8aaf-11de-a64d-234b64dd91b4 tags: master, trunk
Changes
161
162
163
164
165
166
167



















































168
169
170
171
172
173
174
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







			gevent.core.read_event(self._fd.fileno(), self._wait_helper)
		else:
			# EOF found, sending EOF to queue
			self.put_nowait(None)

stdin = FReadlineQueue(sys.stdin)

class FWritelineQueue(gevent.queue.JoinableQueue):
	# storing fileno, io interface, leftover
	__slots__ = frozenset(['_fileno', '_io', '_tail'])

	def __init__(self, fd, closefd = True):
		import io
		# initialising class
		gevent.queue.JoinableQueue.__init__(self)
		# storing fileno
		self._fileno = fd.fileno()
		# creating interface
		self._io = io.FileIO(self._fileno, 'w', closefd)
		# using empty tail
		self._tail = None
		# putting file to nonblocking mode
		fcntl.fcntl(self._fileno, fcntl.F_SETFL, fcntl.fcntl(self._fileno, fcntl.F_GETFL)  | os.O_NONBLOCK)

	def __del__(self):
		# purge queue before deleting
		if not self.empty():
			self.join()

	def put(self, item, block=True, timeout=None):
		# calling real put
		gevent.queue.JoinableQueue.put(self, item, block, timeout)
		# installing event handler
		gevent.core.write_event(self._fileno, self._wait_helper)

	def _wait_helper(self, ev, evtype):
		# XXX ev, evtype checking?
		# checking leftover
		while True:
			if self._tail == None:
				try:
					self._tail = str(self.get_nowait()).encode('utf-8') + '\n'
				except gevent.queue.Empty:
					self._tail = None
					return
			# writing tail
			written = self._io.write(self._tail)
			length = len(self._tail)
			if written == length:
				self._tail = None
			elif written < length:
				self._tail = self._tail[written:]
				break
			else:
				break
		# reinstalling event handler
		gevent.core.write_event(self._fileno, self._wait_helper)

# wrapper around database
class tagDB(object):
	__slots__ = frozenset(['_cursor', '_db'])

	def __init__(self):
		config.section('database')
		if config['host'] == None:
223
224
225
226
227
228
229
230

231
232
233
234
235

236
237

238
239
240

241
242
243
244
245
246
247

248
249
250
251
252
253
254
255
256
257
258

259
260

261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278

279
280
281
282
283

284
285
286
287
288
289
290
274
275
276
277
278
279
280

281
282
283
284
285

286
287
288
289
290
291

292
293
294
295
296
297
298

299
300
301
302
303
304
305
306
307
308
309

310
311

312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329

330
331
332
333


334
335
336
337
338
339
340
341







-
+




-
+


+


-
+






-
+










-
+

-
+

















-
+



-
-
+








	def dump_conf(self):
		self._cursor.execute("select netmask, redirect_url, from_weekday, to_weekday, from_time, to_time, tag::text from rules")
		return(self._field_names(), self._cursor.fetchall())

# abstract class with basic checking functionality
class Checker(object):
	__slots__ = frozenset(['_db', '_log', '_queue', '_request'])
	__slots__ = frozenset(['_db', '_log', '_queue', '_request', '_stdout'])

	def __init__(self, queue, logger):
		self._db = tagDB()
		self._log = logger
		self._log.info('started\n')
		self._log.info('started')
		self._request = re.compile('^([0-9]+)\ (http|ftp):\/\/([-\w.:]+)\/([^ ]*)\ ([0-9.]+)\/(-|[\w\.]+)\ (-|\w+)\ (-|GET|HEAD|POST).*$')
		self._queue = queue
		self._stdout = FWritelineQueue(sys.stdout, False)

	def process(self, id, site, ip_address, url_path, line = None):
		#self._log.info('trying {}\n'.format(site))
		#self._log.info('trying {}'.format(site))
		result = self._db.check(site, ip_address)
		reply = None
		#self._log.info('got {} lines from database'.format(len(result)))
		for row in result:
			if row != None and row[0] != None:
				if row[1] != None:
					self._log.info('trying regexp "{}" versus "{}"\n'.format(row[1], url_path))
					self._log.info('trying regexp "{}" versus "{}"'.format(row[1], url_path))
					try:
						if re.compile(row[1]).match(url_path):
							reply = row[0].format(url_path)
						else:
							continue
					except:
						self._log.info("can't compile regexp")
				else:
					reply = row[0].format(url_path)
			if reply != None:
				self.writeline('{} {}\n'.format(id, reply))
				self.writeline('{} {}'.format(id, reply))
				return(True)
		self.writeline('{}\n'.format(id))
		self.writeline('{}'.format(id))

	def check(self):
		while True:
			line = self._queue.get()
			if line == None:
				break
			#self._log.info('request: ' + line)
			request = self._request.match(line)
			if request:
				id = request.group(1)
				#proto = request.group(2)
				site = request.group(3)
				url_path = request.group(4)
				ip_address = request.group(5)
				self.process(id, site, ip_address, url_path, line)
			else:
				self._log.info('bad request\n')
				self.writeline(line + '\n')
				self.writeline(line)

	def writeline(self, string):
		self._log.info('sending: ' + string)
		sys.stdout.write(string)
		sys.stdout.flush()
		self._stdout.put(string)

	def loop(self):
		pool = gevent.pool.Pool()
		pool.spawn(self.check)
		pool.join()

if config.options.dump or config.options.load or config.options.dump_conf or config.options.load_conf: