Samesite - proxy that can cache partial transfers

Diff
anonymous

Diff

Differences From Artifact [f1c4262f2e]:

To Artifact [ed1a5254d3]:


244
245
246
247
248
249
250





251
252
253
254
255
256
257
258
259









260


261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285

286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309


310
311
312
313
314
315
316
317
318
319
320
					if my_path in index:
						index[my_path][header] = self.headers[header]
				else:
					print('Unknown header - ', header, ': ', self.headers[header], sep='')
					return()
				print(header, self.headers[header])






			# creating empty placeholder in index
			# if there's no space map and there's no file in real directory - we have no file
			# if there's an empty space map - file is full
			# space map generally covers every bit of file we don't posess currently
			if not my_path in index:
				info += '\nThis one is new.'
				reload = True
				record = {}
			else:









				record = index[my_path]



			if not '_parts' in record:
				record['_parts'] = None

			# creating file name from my_path
			file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
			# partial file or unfinished download
			temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)

			# forcibly checking file if no file present
			if os.access(file_name, os.R_OK):
				file_stat = os.stat(file_name)
			elif '_parts' in record and os.access(temp_name, os.R_OK):
				file_stat = os.stat(temp_name)
			elif not reload:
				print(record)
				info += '\nFile not found or inaccessible.'
				record['_parts'] = None
				reload = True

			# forcibly checking file if file size doesn't match with index data
			if not reload:
				if '_parts' in record and record['_parts'] == spacemap.SpaceMap():
					if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
						info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])

						reload = True

			# forcibly checking file if index holds Pragma header
			if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache':
				info +='\nPragma on: recheck imminent.'
				recheck = True

			# skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
			if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0:
				recheck = True

			print(info)
			if reload or recheck:

				try:
					request = 'http://' + config['root'] + my_path
					needed = None
					# XXX and if we specify full file we don't go partial?
					if requested_ranges != None:
						if '_parts' in record and record['_parts'] != None:
							needed = record['_parts'] & requested_ranges
						elif config['noparts']:
							needed = record['_parts']
						else:


							needed = requested_ranges
						ranges = ()
						print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
						if len(needed) > 0:
							needed.rewind()
							while True:
								range = needed.pop()
								if range[0] == None:
									break
								ranges += '{}-{}'.format(range[0], range[1] - 1),
							request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})







>
>
>
>
>









>
>
>
>
>
>
>
>
>

>
>




<
<
<
<
<
<
<
<
<
<
<
<
<
|
|






>




















<
|
|
|
>
>



|







244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280













281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309

310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
					if my_path in index:
						index[my_path][header] = self.headers[header]
				else:
					print('Unknown header - ', header, ': ', self.headers[header], sep='')
					return()
				print(header, self.headers[header])

			# creating file name from my_path
			file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
			# partial file or unfinished download
			temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)

			# creating empty placeholder in index
			# if there's no space map and there's no file in real directory - we have no file
			# if there's an empty space map - file is full
			# space map generally covers every bit of file we don't posess currently
			if not my_path in index:
				info += '\nThis one is new.'
				reload = True
				record = {}
			else:
				# forcibly checking file if no file present
				if os.access(file_name, os.R_OK):
					file_stat = os.stat(file_name)
				elif '_parts' in index[my_path] and os.access(temp_name, os.R_OK):
					file_stat = os.stat(temp_name)
				else:
					info += '\nFile not found or inaccessible.'
					index[my_path]['_parts'] = None
					reload = True
				record = index[my_path]

			print(record)

			if not '_parts' in record:
				record['_parts'] = None














			if record['_parts'] == None:
				recheck = True

			# forcibly checking file if file size doesn't match with index data
			if not reload:
				if '_parts' in record and record['_parts'] == spacemap.SpaceMap():
					if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
						info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
						record['_parts'] = None
						reload = True

			# forcibly checking file if index holds Pragma header
			if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache':
				info +='\nPragma on: recheck imminent.'
				recheck = True

			# skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
			if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0:
				recheck = True

			print(info)
			if reload or recheck:

				try:
					request = 'http://' + config['root'] + my_path
					needed = None
					# XXX and if we specify full file we don't go partial?
					if requested_ranges != None:
						if '_parts' in record and record['_parts'] != None:

							if config['noparts']:
								needed = record['_parts']
							else:
								needed = record['_parts'] | requested_ranges
						elif not config['noparts']:
							needed = requested_ranges
						ranges = ()
						print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
						if needed != None and len(needed) > 0:
							needed.rewind()
							while True:
								range = needed.pop()
								if range[0] == None:
									break
								ranges += '{}-{}'.format(range[0], range[1] - 1),
							request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380

						if reload:
							print('Reloading.')
							if os.access(temp_name, os.R_OK):
								os.unlink(temp_name)
							if os.access(file_name, os.R_OK):
								os.unlink(file_name)
						if new_record['_parts'] == None or reload:
							new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
						print(new_record)

						# downloading file or segment
						if 'Content-Length' in new_record:
							if needed == None:
								needed = new_record['_parts']







<







371
372
373
374
375
376
377

378
379
380
381
382
383
384

						if reload:
							print('Reloading.')
							if os.access(temp_name, os.R_OK):
								os.unlink(temp_name)
							if os.access(file_name, os.R_OK):
								os.unlink(file_name)

							new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
						print(new_record)

						# downloading file or segment
						if 'Content-Length' in new_record:
							if needed == None:
								needed = new_record['_parts']