Samesite - proxy that can cache partial transfers

Check-in [fb10031536]
anonymous

Check-in [fb10031536]

Overview
Comment:added license and todo many fixes in web server, now it's mostly functional web server was tested by Windows Update (win7, WinXP, Win2003)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: fb1003153617e0fb2ee594a7e23ad817f012431885cde1fbbc08d9991e7769b8
User & Date: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 on 2010-08-21 10:59:04.000
Other Links: branch diff | manifest | tags
Context
2010-08-25
15:16
multiconfiguration - working with many sites at ones fullfile - avoiding partial transfers Pragma support in requests files are checked both in full dir and part dir file is moved to full dir even if it's left untouched check-in: e7b837a681 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2010-08-21
10:59
added license and todo many fixes in web server, now it's mostly functional web server was tested by Windows Update (win7, WinXP, Win2003) check-in: fb10031536 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2010-08-20
14:25
small fix for striping unneeded request parts from file path check-in: d0071bdbc7 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
Changes
Added LICENSE version [7fe6229bc8].
Added TODO version [d7b006a56b].
162
163
164
165
166
167
168

169
170
171
172
173
174

175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218

219
220
221
222
223
224
225
162
163
164
165
166
167
168
169
170
171
172
173
174

175

176
177
178
179
180
181
182
183
184
185
186
187
188
189

190
191
192
193
194
195
196
197
198
199
200
201

202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224







+





-
+
-














-












-















+







				my_path = myPath.group(1)
			else:
				my_path = self.path

			proxy_ignored = ('Accept', 'Accept-Encoding',
				'Cache-Control', 'Connection',
				'Host',
				'If-Modified-Since', 'If-Unmodified-Since',
				'User-Agent',
				'Via',
				'X-Forwarded-For',
			)

			print('===============[ Request ]===')
			print('===============[ {} request ]==='.format(self.command))
			print('Command:', self.command)

			for header in self.headers:
				if header in proxy_ignored:
					pass
				elif header in ('Range'):
					isRange = re.compile('bytes=(\d+)-(\d+)').match(self.headers[header])
					if isRange:
						requested_ranges = SpaceMap({int(isRange.group(1)): int(isRange.group(2)) + 1})
					else:
						return()
				else:
					print('Unknown header - ', header, ': ', self.headers[header], sep='')
					return()
				print(header, self.headers[header])
			print(my_path)

			# creating empty placeholder in index
			# if there's no space map and there's no file in real directory - we have no file
			# if there's an empty space map - file is full
			# space map generally covers every bit of file we don't posess currently
			if not my_path in index:
				info += '\nThis one is new.'
				reload = True
				record = {'_parts': None}
			else:
				record = index[my_path]
				if '_parts' in index[my_path]:
					print(record['_parts'])
					if index[my_path]['_parts'] == {0: -1}:
						index[my_path]['_parts'] = None

			# creating file name from my_path
			file_name = options.dir + os.sep + re.compile('%20').sub(' ', my_path)
			# partial file or unfinished download
			temp_name = options.dir + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)

			# forcibly checking file if no file present
			if os.access(file_name, os.R_OK):
				file_stat = os.stat(file_name)
			elif '_parts' in record and os.access(temp_name, os.R_OK):
				file_stat = os.stat(temp_name)
			elif not reload:
				info += '\nFile not found or inaccessible.'
				record = {'_parts': None}
				reload = True

			# forcibly checking file if file size doesn't match with index data
			if not reload:
				if '_parts' in record and record['_parts'] == SpaceMap():
					if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
						info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
235
236
237
238
239
240
241

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258









259
260
261
262
263
264
265
266
267
268
269
270
271
272

273
274
275
276
277
278

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306

307
308
309
310
311
312
313
314



315
316
317
318
319


320
321

322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348





















349
350
351
352
353




354
355
356
357



358
359
360
361
362
363
364
365







366
367

368
369
370
371
372
373
374
375
376
377
378











379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401

402

403
404
405
406
407
408
409
410
411
412

413

414
415

416
417
418

419
420
421





422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
234
235
236
237
238
239
240
241
242
243
244
245
246
247











248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266

267
268

269
270
271
272
273
274

275
276
277
278
279
280




281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314


315
316
317

318
319
320
321
322
323






















324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344





345
346
347
348




349
350
351








352
353
354
355
356
357
358


359











360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376

377

378
379
380
381
382
383
384
385
386
387
388
389
390
391
392

393


394
395
396
397
398
399
400

401
402
403
404
405
406
407
408
409
410



411
412
413
414
415
416
417
418
419
420

421
422
423
424
425
426
427
428
429

430
431
432
433
434
435
436







+






-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+










-


-
+





-
+





-
-
-
-



















+








+
+
+



-
-
+
+

-
+





-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
-
-
-
-
+
+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
+
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+






-

-














+
-
+
-
-







-
+

+


+



+
-
-
-
+
+
+
+
+





-









-







				recheck = True

			print(info)
			if reload or recheck:

				try:
					request = options.root + my_path
					needed = None
					if requested_ranges != None:
						if '_parts' in record and record['_parts'] != None:
							needed = record['_parts'] & requested_ranges
						else:
							needed = requested_ranges
						ranges = ()
						print('Requesting ranges:', ranges)
						print('Not stored ranges:', record['_parts'])
						print('Requested ranges:', requested_ranges)
						print('Needed ranges:', needed)
						needed.rewind()
						while True:
							range = needed.pop()
							if range[0] == None:
								break
							ranges += '{}-{}'.format(range[0], range[1] - 1),
						request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
						print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
						if len(needed) > 0:
							needed.rewind()
							while True:
								range = needed.pop()
								if range[0] == None:
									break
								ranges += '{}-{}'.format(range[0], range[1] - 1),
							request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})

					with urllib.request.urlopen(request) as source:
						new_record = {}
						new_record['_parts'] = record['_parts']
						headers = source.info()

						# stripping unneeded headers (XXX make this inplace?)
						for header in headers:
							if header in desc_fields:
								#if header == 'Pragma' and headers[header] != 'no-cache':
								print(header, headers[header])
								if header == 'Content-Length':
									if 'Content-Range' not in headers:
										new_record[header] = headers[header]
										new_record[header] = int(headers[header])
								else:
									new_record[header] = headers[header]
							elif header == 'Content-Range':
								range = re.compile('^bytes (\d+)-(\d+)/(\d+)$').match(headers[header])
								if range:
									new_record['Content-Length'] = range.group(3)
									new_record['Content-Length'] = int(range.group(3))
								else:	
									assert False, 'Content-Range unrecognized.'
							elif not header in ignore_fields:
								print('Undefined header "', header, '": ', headers[header], sep='')

						if new_record['_parts'] == None:
							new_record['_parts'] = SpaceMap({0: int(new_record['Content-Length'])})
						print(new_record)

						# comparing headers with data found in index
						# if any header has changed (except Pragma) file is fully downloaded
						# same if we get more or less headers
						old_keys = set(record.keys())
						old_keys.discard('_time')
						old_keys.discard('Pragma')
						more_keys = set(new_record.keys()) - old_keys
						more_keys.discard('Pragma')
						less_keys = old_keys - set(new_record.keys())
						if len(more_keys) > 0:
							if not len(old_keys) == 0:
								print('More headers appear:', more_keys)
							reload = True
						elif len(less_keys) > 0:
							print('Less headers appear:', less_keys)
						else:
							for key in record.keys():
								if key[0] != '_' and key != 'Pragma' and not record[key] == new_record[key]:
									print('Header "', key, '" changed from [', record[key], '] to [', new_record[key], ']', sep='')
									print(type(record[key]), type(new_record[key]))
									reload = True

						if reload:
							print('Reloading.')
							if os.access(temp_name, os.R_OK):
								os.unlink(temp_name)
							if os.access(file_name, os.R_OK):
								os.unlink(file_name)
						if new_record['_parts'] == None or reload:
							new_record['_parts'] = SpaceMap({0: int(new_record['Content-Length'])})
						print(new_record)

						# downloading file or segment
						if 'Content-Length' in new_record:
							if requested_ranges == None:
								requested_ranges = new_record['_parts']
							if needed == None:
								needed = new_record['_parts']
							else:
								if len(requested_ranges) > 1:
								if len(needed) > 1:
									print("Multipart requests currently not supported.")
									assert False, 'Skip this one for now.'
						else:
							assert False, 'No Content-Length or Content-Range header.'

						if reload:
							new_record['_time'] = datetime.datetime.now()
							if self.command not in ('HEAD'):
								# file is created at temporary location and moved in place only when download completes
								if not os.access(temp_name, os.R_OK):
									empty_name = options.dir + os.sep + '.tmp'
									with open(empty_name, 'w+b') as some_file:
										pass
									os.renames(empty_name, temp_name)
								temp_file = open(temp_name, 'r+b')
								requested_ranges.rewind()
								while True:
									(start, end) = requested_ranges.pop()
									if start == None:
										break
									stream_last = start
									old_record = new_record
									if end - start < block_size:
										req_block_size = end - start
									else:
										req_block_size = block_size
									buffer = source.read(req_block_size)
						new_record['_time'] = datetime.datetime.now()
						if self.command not in ('HEAD'):
							# file is created at temporary location and moved in place only when download completes
							if not os.access(temp_name, os.R_OK):
								empty_name = options.dir + os.sep + '.tmp'
								with open(empty_name, 'w+b') as some_file:
									pass
								os.renames(empty_name, temp_name)
							temp_file = open(temp_name, 'r+b')
							needed.rewind()
							while True:
								(start, end) = needed.pop()
								if start == None:
									break
								stream_last = start
								old_record = new_record
								if end - start < block_size:
									req_block_size = end - start
								else:
									req_block_size = block_size
								buffer = source.read(req_block_size)
									print(buffer)
									length = len(buffer)
									while length > 0 and stream_last < end:
										stream_pos = stream_last + length
										assert not stream_pos > end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
								length = len(buffer)
								while length > 0 and stream_last < end:
									stream_pos = stream_last + length
									assert not stream_pos > end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
										print('Writing', length, 'bytes to temp file at position', stream_last)
										temp_file.seek(stream_last)
										temp_file.write(buffer)
										new_record['_parts'] = new_record['_parts'] - SpaceMap({stream_last: stream_pos})
									temp_file.seek(stream_last)
									temp_file.write(buffer)
									new_record['_parts'] = new_record['_parts'] - SpaceMap({stream_last: stream_pos})
										print(new_record)
										index[my_path] = old_record
										index.sync()
										old_record = new_record
										stream_last = stream_pos
										if end - stream_last < block_size:
											req_block_size = end - stream_last
										buffer = source.read(req_block_size)
									index[my_path] = old_record
									index.sync()
									old_record = new_record
									stream_last = stream_pos
									if end - stream_last < block_size:
										req_block_size = end - stream_last
									buffer = source.read(req_block_size)
										print(buffer)
										length = len(buffer)
									length = len(buffer)
								print(new_record)
								index[my_path] = new_record
								index.sync()
								temp_file.close()

								# moving downloaded data to real file
								if new_record['_parts'] == SpaceMap():
									# just moving
									# drop old dirs XXX
									print('Moving temporary file to new destination.')
									os.renames(temp_name, file_name)
							# moving downloaded data to real file
							temp_file.close()
							if new_record['_parts'] == SpaceMap():
								# just moving
								# drop old dirs XXX
								print('Moving temporary file to new destination.')
								os.renames(temp_name, file_name)

						print(new_record)
						index[my_path] = new_record
						index.sync()

				except urllib.error.HTTPError as error:
					# in case of error we don't need to do anything actually,
					# if file download stalls or fails the file would not be moved to it's location
					print(error)

			print('Sending response.')
			if self.command == 'HEAD':
				print('Sending HEAD response.')
				self.send_response(200)
				if 'Content-Length' in index[my_path]:
					self.send_header('Content-Length', index[my_path]['Content-Length'])
				self.send_header('Accept-Ranges', 'bytes')
				self.send_header('Content-Type', 'application/octet-stream')
				if 'Last-Modified' in index[my_path]:
					self.send_header('Last-Modified', index[my_path]['Last-Modified'])
				self.end_headers()
			else:
				if index[my_path]['_parts'] != SpaceMap():
					file_name = temp_name

				with open(file_name, 'rb') as real_file:
					file_stat = os.stat(file_name)
					if 'Range' in self.headers:
					self.send_response(200)
						self.send_response(206)
					self.send_header('Last-Modified', index[my_path]['Last-Modified'])
					if requested_ranges != None:
						ranges = ()
						requested_ranges.rewind()
						while True:
							pair = requested_ranges.pop()
							if pair[0] == None:
								break
							ranges += '{}-{}'.format(pair[0], str(pair[1] - 1)),
						self.send_header('Content-Range', 'bytes ' + ','.join(ranges) + '/' + index[my_path]['Content-Length'])
						self.send_header('Content-Range', 'bytes {}/{}'.format(','.join(ranges), index[my_path]['Content-Length']))
					else:
						self.send_response(200)
						self.send_header('Content-Length', str(file_stat.st_size))
						requested_ranges = SpaceMap({0: file_stat.st_size})
					self.send_header('Last-Modified', index[my_path]['Last-Modified'])
					self.send_header('Content-Type', 'application/octet-stream')
					self.end_headers()
					if self.command in ('GET'):
						if len(requested_ranges) > 0:
						requested_ranges.rewind()
						(start, end) = requested_ranges.pop()
						print('Seeking file to position', start)
							requested_ranges.rewind()
							(start, end) = requested_ranges.pop()
						else:
							start = 0
							end = index[my_path]['Content-Length']
						real_file.seek(start)
						if block_size > end - start:
							req_block_size = end - start
						else:
							req_block_size = block_size
						print('block_size is', req_block_size)
						buffer = real_file.read(req_block_size)
						length = len(buffer)
						while length > 0:
							self.wfile.write(buffer)
							start += len(buffer)
							if req_block_size > end - start:
								req_block_size = end - start
							if req_block_size == 0:
								break
							print('block_size is', req_block_size)
							buffer = real_file.read(req_block_size)
							length = len(buffer)
					
		def do_HEAD(self):
			return self.__process()
		def do_GET(self):
			return self.__process()