Samesite - proxy that can cache partial transfers

Check-in [9a8a46bcf0]
anonymous

Check-in [9a8a46bcf0]

Overview
Comment:another round of small fixes in logic, ignored fields and caching files without known length
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: 9a8a46bcf0dbe844b80ef0f6bf4835d1dc274fec9586b1e349ec26ffd9dbe352
User & Date: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 on 2011-09-06 15:09:32.000
Other Links: branch diff | manifest | tags
Context
2011-12-14
08:54
Fixed logic around old file rechecking. check-in: 8425e2e393 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2011-09-06
15:09
another round of small fixes in logic, ignored fields and caching files without known length check-in: 9a8a46bcf0 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2011-03-06
09:39
cleaning up a bit check-in: 90160dbf50 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
Changes
1

2
3
4
5
6
7
8

1
2
3
4
5
6
7
8
-
+







#!/usr/bin/env python3.1
#!/usr/bin/env python3.2

import datetime, http.cookiejar, os, sys, shelve, spacemap, re, urllib.request

class Config:
	__slots__ = frozenset(['_config', '_default', '_section', 'options', 'root'])
	_default = {
		'general': {
81
82
83
84
85
86
87
88

89
90
91
92
93
94
95
81
82
83
84
85
86
87

88
89
90
91
92
93
94
95







-
+







	'Accept-Ranges', 'Age',
	'Cache-Control', 'Connection', 'Content-Type',
	'Date',
	'Expires',
	'Referer',
	'Server',
	'Via',
	'X-Cache', 'X-Cache-Lookup', 'X-Powered-By'
	'X-Cache', 'X-Cache-Lookup', 'X-Powered-By',
])

block_size = 4096

import http.server

class MyRequestHandler(http.server.BaseHTTPRequestHandler):
138
139
140
141
142
143
144
145

146
147
148
149
150
151
152
138
139
140
141
142
143
144

145
146
147
148
149
150
151
152







-
+







			'Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language',
			'Cache-Control', 'Connection', 'Content-Length', 'Cookie',
			'Host',
			'If-Modified-Since', 'If-Unmodified-Since',
			'Referer',
			'User-Agent',
			'Via',
			'X-Forwarded-For', 'X-REMOVED',
			'X-Forwarded-For', 'X-Last-HR', 'X-Last-HTTP-Status-Code', 'X-REMOVED', 'X-Real-IP', 'X-Retry-Count',
		])

		print('===============[ {} request ]==='.format(self.command))

		for header in self.headers:
			if header in proxy_ignored:
				pass
226
227
228
229
230
231
232
233

234
235
236
237
238
239
240
226
227
228
229
230
231
232

233
234
235
236
237
238
239
240







-
+







						my_headers[header] = self.headers[header]

				needed = None
				if '_parts' in record and record['_parts'] != None:
					if config['noparts'] != 'no' or requested_ranges == None or requested_ranges == spacemap.SpaceMap():
						needed = record['_parts']
					else:
						needed = record['_parts'] | requested_ranges
						needed = record['_parts'] & requested_ranges
				elif config['noparts'] =='no' and requested_ranges != None and requested_ranges != spacemap.SpaceMap():
					needed = requested_ranges
				ranges = ()
				print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
				if needed != None and len(needed) > 0:
					needed.rewind()
					while True:
293
294
295
296
297
298
299

300



301
302
303
304
305
306
307
308
309
310
311
312


313
314
315
316
317
318
319
293
294
295
296
297
298
299
300

301
302
303
304
305
306
307
308
309
310
311
312
313


314
315
316
317
318
319
320
321
322







+
-
+
+
+










-
-
+
+








					if reload:
						print('Reloading.')
						if os.access(temp_name, os.R_OK):
							os.unlink(temp_name)
						if os.access(file_name, os.R_OK):
							os.unlink(file_name)
						if 'Content-Length' in new_record:
						new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
							new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
						else:
							new_record['_parts'] = spacemap.SpaceMap()
					print(new_record)

					# downloading file or segment
					if 'Content-Length' in new_record:
						if needed == None:
							needed = new_record['_parts']
						else:
							if len(needed) > 1:
								print("Multipart requests currently not supported.")
								assert False, 'Skip this one for now.'
					else:
						assert False, 'No Content-Length or Content-Range header.'
					#else:
						#assert False, 'No Content-Length or Content-Range header.'

					new_record['_time'] = datetime.datetime.now()
					if self.command not in ('HEAD'):
						# file is created at temporary location and moved in place only when download completes
						if not os.access(temp_name, os.R_OK):
							empty_name = config['dir'] + os.sep + '.tmp'
							with open(empty_name, 'w+b') as some_file:
408
409
410
411
412
413
414


415



416
417
418
419
420
421
422
411
412
413
414
415
416
417
418
419

420
421
422
423
424
425
426
427
428
429







+
+
-
+
+
+







				self.end_headers()
				if self.command in ('GET'):
					if len(requested_ranges) > 0:
						requested_ranges.rewind()
						(start, end) = requested_ranges.pop()
					else:
						start = 0
						# XXX ugly hack
						if 'Content-Length' in index[my_path]:
						end = index[my_path]['Content-Length']
							end = index[my_path]['Content-Length']
						else:
							end = 0
					real_file.seek(start)
					if block_size > end - start:
						req_block_size = end - start
					else:
						req_block_size = block_size
					buffer = real_file.read(req_block_size)
					length = len(buffer)