Samesite - proxy that can cache partial transfers

Check-in [b0975a28fb]
anonymous

Check-in [b0975a28fb]

Overview
Comment:optimized remains detection
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: b0975a28fbc7c10f43d18cace9f764b8ccaabbc1b33ccc8f8834f58ba3e7b097
User & Date: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 on 2010-08-26 15:06:19.000
Other Links: branch diff | manifest | tags
Context
2010-09-06
20:53
More features: * path substitution * more fields ignored in requests; * request now uses original path - not modified by stripping or substituting; * some headers are proxied to request; * when there is no file to send 502 error is returned; * recheck file before moving it to full dir; * Last-Modified only sent when present. check-in: cab908195f user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2010-08-26
15:06
optimized remains detection check-in: b0975a28fb user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
2010-08-25
15:16
multiconfiguration - working with many sites at ones fullfile - avoiding partial transfers Pragma support in requests files are checked both in full dir and part dir file is moved to full dir even if it's left untouched check-in: e7b837a681 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk
Changes
244
245
246
247
248
249
250





251
252
253
254
255
256
257
258
259









260


261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279


280
281
282
283
284
285

286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309





310
311
312
313

314
315
316
317
318
319
320
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280















281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309




310
311
312
313
314
315
316
317

318
319
320
321
322
323
324
325







+
+
+
+
+









+
+
+
+
+
+
+
+
+

+
+




-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+






+




















-
-
-
-
+
+
+
+
+



-
+







					if my_path in index:
						index[my_path][header] = self.headers[header]
				else:
					print('Unknown header - ', header, ': ', self.headers[header], sep='')
					return()
				print(header, self.headers[header])

			# creating file name from my_path
			file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
			# partial file or unfinished download
			temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)

			# creating empty placeholder in index
			# if there's no space map and there's no file in real directory - we have no file
			# if there's an empty space map - file is full
			# space map generally covers every bit of file we don't posess currently
			if not my_path in index:
				info += '\nThis one is new.'
				reload = True
				record = {}
			else:
				# forcibly checking file if no file present
				if os.access(file_name, os.R_OK):
					file_stat = os.stat(file_name)
				elif '_parts' in index[my_path] and os.access(temp_name, os.R_OK):
					file_stat = os.stat(temp_name)
				else:
					info += '\nFile not found or inaccessible.'
					index[my_path]['_parts'] = None
					reload = True
				record = index[my_path]

			print(record)

			if not '_parts' in record:
				record['_parts'] = None

			# creating file name from my_path
			file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
			# partial file or unfinished download
			temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)

			# forcibly checking file if no file present
			if os.access(file_name, os.R_OK):
				file_stat = os.stat(file_name)
			elif '_parts' in record and os.access(temp_name, os.R_OK):
				file_stat = os.stat(temp_name)
			elif not reload:
				print(record)
				info += '\nFile not found or inaccessible.'
				record['_parts'] = None
				reload = True
			if record['_parts'] == None:
				recheck = True

			# forcibly checking file if file size doesn't match with index data
			if not reload:
				if '_parts' in record and record['_parts'] == spacemap.SpaceMap():
					if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
						info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
						record['_parts'] = None
						reload = True

			# forcibly checking file if index holds Pragma header
			if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache':
				info +='\nPragma on: recheck imminent.'
				recheck = True

			# skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
			if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0:
				recheck = True

			print(info)
			if reload or recheck:

				try:
					request = 'http://' + config['root'] + my_path
					needed = None
					# XXX and if we specify full file we don't go partial?
					if requested_ranges != None:
						if '_parts' in record and record['_parts'] != None:
							needed = record['_parts'] & requested_ranges
						elif config['noparts']:
							needed = record['_parts']
						else:
							if config['noparts']:
								needed = record['_parts']
							else:
								needed = record['_parts'] | requested_ranges
						elif not config['noparts']:
							needed = requested_ranges
						ranges = ()
						print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
						if len(needed) > 0:
						if needed != None and len(needed) > 0:
							needed.rewind()
							while True:
								range = needed.pop()
								if range[0] == None:
									break
								ranges += '{}-{}'.format(range[0], range[1] - 1),
							request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
371
372
373
374
375
376
377

378
379
380
381
382
383
384







-








						if reload:
							print('Reloading.')
							if os.access(temp_name, os.R_OK):
								os.unlink(temp_name)
							if os.access(file_name, os.R_OK):
								os.unlink(file_name)
						if new_record['_parts'] == None or reload:
							new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
						print(new_record)

						# downloading file or segment
						if 'Content-Length' in new_record:
							if needed == None:
								needed = new_record['_parts']