Samesite - proxy that can cache partial transfers

Check-in [f57e6e032b]
anonymous

Check-in [f57e6e032b]

Overview
Comment:less logs, more headers, don't saturate disk with writes
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: f57e6e032bfff0324eadefc3721aa2aad771934903da94f4f859d310f611e9c5
User & Date: arcade@b1t.name on 2013-11-05 16:21:14.000
Other Links: branch diff | manifest | tags
Context
2014-01-13
17:25
DNT header check-in: 4fece04acc user: arcade@b1t.name tags: master, trunk
2013-11-05
16:21
less logs, more headers, don't saturate disk with writes check-in: f57e6e032b user: arcade@b1t.name tags: master, trunk
2013-08-23
12:00
better error handling check-in: 6cf3431e69 user: arcade@b1t.name tags: master, trunk
Changes
43
44
45
46
47
48
49
50

51
52
53
54
55
56
57
43
44
45
46
47
48
49

50
51
52
53
54
55
56
57







-
+







	'Accept-Ranges', 'Age',
	'Cache-Control', 'Connection', 'Content-Type',
	'Date',
	'Expires',
	'Referer',
	'Server',
	'Via',
	'X-Cache', 'X-Cache-Lookup', 'X-Livetool', 'X-Powered-By',
	'X-CCC', 'X-CID', 'X-Cache', 'X-Cache-Lookup', 'X-Livetool', 'X-Powered-By',
])

block_size = 8192

import bsddb3.dbshelve, copy, datetime, http.server, spacemap, urllib.request, urllib.error

class MyRequestHandler(http.server.BaseHTTPRequestHandler):
296
297
298
299
300
301
302

303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322


323
324



325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345

346
347
348
349
350
351
352
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325


326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348

349
350
351
352
353
354
355
356







+




















+
+
-
-
+
+
+




















-
+







						with open(empty_name, 'w+b') as some_file:
							pass
						os.renames(empty_name, temp_name)
					temp_file = open(temp_name, 'r+b')
					if requested_ranges == None and needed == None:
						needed = new_record['_parts']
					needed.rewind()
					countdown = 16
					while True:
						# XXX can make this implicit - one request per range
						(start, end) = needed.pop()
						if start == None:
							break
						stream_last = start
						old_record = copy.copy(new_record)
						if end - start < block_size:
							req_block_size = end - start
						else:
							req_block_size = block_size
						buffer = source.read(req_block_size)
						length = len(buffer)
						while length > 0 and stream_last < end:
							stream_pos = stream_last + length
							assert stream_pos <= end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
							temp_file.seek(stream_last)
							temp_file.write(buffer)
							x = new_record['_parts'] - spacemap.SpaceMap({stream_last: stream_pos})
							new_record['_parts'] = new_record['_parts'] - spacemap.SpaceMap({stream_last: stream_pos})
							countdown -= 1
							if countdown == 0:
							index[my_path_b] = old_record
							index.sync()
								index[my_path_b] = old_record
								index.sync()
								countdown = 16
							old_record = copy.copy(new_record)
							stream_last = stream_pos
							if end - stream_last < block_size:
								req_block_size = end - stream_last
							buffer = source.read(req_block_size)
							length = len(buffer)
					# moving downloaded data to real file
					temp_file.close()

				index[my_path_b] = new_record
				index.sync()

			except urllib.error.HTTPError as error:
				# in case of error we don't need to do anything actually,
				# if file download stalls or fails the file would not be moved to it's location
				self.send_response(error.code)
				self.end_headers()
				print(error, repr(my_headers))
				return

		print(index[my_path_b])
		#print(index[my_path_b])

		if not os.access(file_name, os.R_OK) and os.access(temp_name, os.R_OK) and '_parts' in index[my_path_b] and index[my_path_b]['_parts'] == spacemap.SpaceMap():
			# just moving
			# drop old dirs XXX
			print('Moving temporary file to new destination.')
			os.renames(temp_name, file_name)