Samesite - proxy that can cache partial transfers

Check-in [9a8a46bcf0]
anonymous

Check-in [9a8a46bcf0]

Overview
Comment:another round of small fixes in logic, ignored fields and caching files without known length
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: 9a8a46bcf0dbe844b80ef0f6bf4835d1dc274fec9586b1e349ec26ffd9dbe352
User & Date: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 on 2011-09-06 15:09:32.000
Other Links: branch diff | manifest | tags
Context
2011-12-14
08:54
Fixed logic around old file rechecking. check-in: 8425e2e393 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2011-09-06
15:09
another round of small fixes in logic, ignored fields and caching files without known length check-in: 9a8a46bcf0 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2011-03-06
09:39
cleaning up a bit check-in: 90160dbf50 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
Changes
1
2
3
4
5
6
7
8
#!/usr/bin/env python3.1

import datetime, http.cookiejar, os, sys, shelve, spacemap, re, urllib.request

class Config:
	__slots__ = frozenset(['_config', '_default', '_section', 'options', 'root'])
	_default = {
		'general': {
|







1
2
3
4
5
6
7
8
#!/usr/bin/env python3.2

import datetime, http.cookiejar, os, sys, shelve, spacemap, re, urllib.request

class Config:
	__slots__ = frozenset(['_config', '_default', '_section', 'options', 'root'])
	_default = {
		'general': {
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
	'Accept-Ranges', 'Age',
	'Cache-Control', 'Connection', 'Content-Type',
	'Date',
	'Expires',
	'Referer',
	'Server',
	'Via',
	'X-Cache', 'X-Cache-Lookup', 'X-Powered-By'
])

block_size = 4096

import http.server

class MyRequestHandler(http.server.BaseHTTPRequestHandler):







|







81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
	'Accept-Ranges', 'Age',
	'Cache-Control', 'Connection', 'Content-Type',
	'Date',
	'Expires',
	'Referer',
	'Server',
	'Via',
	'X-Cache', 'X-Cache-Lookup', 'X-Powered-By',
])

block_size = 4096

import http.server

class MyRequestHandler(http.server.BaseHTTPRequestHandler):
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
			'Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language',
			'Cache-Control', 'Connection', 'Content-Length', 'Cookie',
			'Host',
			'If-Modified-Since', 'If-Unmodified-Since',
			'Referer',
			'User-Agent',
			'Via',
			'X-Forwarded-For', 'X-REMOVED',
		])

		print('===============[ {} request ]==='.format(self.command))

		for header in self.headers:
			if header in proxy_ignored:
				pass







|







138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
			'Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language',
			'Cache-Control', 'Connection', 'Content-Length', 'Cookie',
			'Host',
			'If-Modified-Since', 'If-Unmodified-Since',
			'Referer',
			'User-Agent',
			'Via',
			'X-Forwarded-For', 'X-Last-HR', 'X-Last-HTTP-Status-Code', 'X-REMOVED', 'X-Real-IP', 'X-Retry-Count',
		])

		print('===============[ {} request ]==='.format(self.command))

		for header in self.headers:
			if header in proxy_ignored:
				pass
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
						my_headers[header] = self.headers[header]

				needed = None
				if '_parts' in record and record['_parts'] != None:
					if config['noparts'] != 'no' or requested_ranges == None or requested_ranges == spacemap.SpaceMap():
						needed = record['_parts']
					else:
						needed = record['_parts'] | requested_ranges
				elif config['noparts'] =='no' and requested_ranges != None and requested_ranges != spacemap.SpaceMap():
					needed = requested_ranges
				ranges = ()
				print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
				if needed != None and len(needed) > 0:
					needed.rewind()
					while True:







|







226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
						my_headers[header] = self.headers[header]

				needed = None
				if '_parts' in record and record['_parts'] != None:
					if config['noparts'] != 'no' or requested_ranges == None or requested_ranges == spacemap.SpaceMap():
						needed = record['_parts']
					else:
						needed = record['_parts'] & requested_ranges
				elif config['noparts'] =='no' and requested_ranges != None and requested_ranges != spacemap.SpaceMap():
					needed = requested_ranges
				ranges = ()
				print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
				if needed != None and len(needed) > 0:
					needed.rewind()
					while True:
293
294
295
296
297
298
299

300


301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319

					if reload:
						print('Reloading.')
						if os.access(temp_name, os.R_OK):
							os.unlink(temp_name)
						if os.access(file_name, os.R_OK):
							os.unlink(file_name)

						new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})


					print(new_record)

					# downloading file or segment
					if 'Content-Length' in new_record:
						if needed == None:
							needed = new_record['_parts']
						else:
							if len(needed) > 1:
								print("Multipart requests currently not supported.")
								assert False, 'Skip this one for now.'
					else:
						assert False, 'No Content-Length or Content-Range header.'

					new_record['_time'] = datetime.datetime.now()
					if self.command not in ('HEAD'):
						# file is created at temporary location and moved in place only when download completes
						if not os.access(temp_name, os.R_OK):
							empty_name = config['dir'] + os.sep + '.tmp'
							with open(empty_name, 'w+b') as some_file:







>
|
>
>










|
|







293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322

					if reload:
						print('Reloading.')
						if os.access(temp_name, os.R_OK):
							os.unlink(temp_name)
						if os.access(file_name, os.R_OK):
							os.unlink(file_name)
						if 'Content-Length' in new_record:
							new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
						else:
							new_record['_parts'] = spacemap.SpaceMap()
					print(new_record)

					# downloading file or segment
					if 'Content-Length' in new_record:
						if needed == None:
							needed = new_record['_parts']
						else:
							if len(needed) > 1:
								print("Multipart requests currently not supported.")
								assert False, 'Skip this one for now.'
					#else:
						#assert False, 'No Content-Length or Content-Range header.'

					new_record['_time'] = datetime.datetime.now()
					if self.command not in ('HEAD'):
						# file is created at temporary location and moved in place only when download completes
						if not os.access(temp_name, os.R_OK):
							empty_name = config['dir'] + os.sep + '.tmp'
							with open(empty_name, 'w+b') as some_file:
408
409
410
411
412
413
414


415


416
417
418
419
420
421
422
				self.end_headers()
				if self.command in ('GET'):
					if len(requested_ranges) > 0:
						requested_ranges.rewind()
						(start, end) = requested_ranges.pop()
					else:
						start = 0


						end = index[my_path]['Content-Length']


					real_file.seek(start)
					if block_size > end - start:
						req_block_size = end - start
					else:
						req_block_size = block_size
					buffer = real_file.read(req_block_size)
					length = len(buffer)







>
>
|
>
>







411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
				self.end_headers()
				if self.command in ('GET'):
					if len(requested_ranges) > 0:
						requested_ranges.rewind()
						(start, end) = requested_ranges.pop()
					else:
						start = 0
						# XXX ugly hack
						if 'Content-Length' in index[my_path]:
							end = index[my_path]['Content-Length']
						else:
							end = 0
					real_file.seek(start)
					if block_size > end - start:
						req_block_size = end - start
					else:
						req_block_size = block_size
					buffer = real_file.read(req_block_size)
					length = len(buffer)