162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
+
-
+
-
-
-
+
|
my_path = myPath.group(1)
else:
my_path = self.path
proxy_ignored = ('Accept', 'Accept-Encoding',
'Cache-Control', 'Connection',
'Host',
'If-Modified-Since', 'If-Unmodified-Since',
'User-Agent',
'Via',
'X-Forwarded-For',
)
print('===============[ Request ]===')
print('===============[ {} request ]==='.format(self.command))
print('Command:', self.command)
for header in self.headers:
if header in proxy_ignored:
pass
elif header in ('Range'):
isRange = re.compile('bytes=(\d+)-(\d+)').match(self.headers[header])
if isRange:
requested_ranges = SpaceMap({int(isRange.group(1)): int(isRange.group(2)) + 1})
else:
return()
else:
print('Unknown header - ', header, ': ', self.headers[header], sep='')
return()
print(header, self.headers[header])
print(my_path)
# creating empty placeholder in index
# if there's no space map and there's no file in real directory - we have no file
# if there's an empty space map - file is full
# space map generally covers every bit of file we don't posess currently
if not my_path in index:
info += '\nThis one is new.'
reload = True
record = {'_parts': None}
else:
record = index[my_path]
if '_parts' in index[my_path]:
print(record['_parts'])
if index[my_path]['_parts'] == {0: -1}:
index[my_path]['_parts'] = None
# creating file name from my_path
file_name = options.dir + os.sep + re.compile('%20').sub(' ', my_path)
# partial file or unfinished download
temp_name = options.dir + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)
# forcibly checking file if no file present
if os.access(file_name, os.R_OK):
file_stat = os.stat(file_name)
elif '_parts' in record and os.access(temp_name, os.R_OK):
file_stat = os.stat(temp_name)
elif not reload:
info += '\nFile not found or inaccessible.'
record = {'_parts': None}
reload = True
# forcibly checking file if file size doesn't match with index data
if not reload:
if '_parts' in record and record['_parts'] == SpaceMap():
if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
|
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
|
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
|
+
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-
-
+
-
+
-
-
-
-
+
+
+
+
-
-
+
+
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
-
-
-
-
+
+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
+
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-
+
-
+
-
-
-
+
+
+
+
-
-
-
+
+
+
+
+
-
-
|
recheck = True
print(info)
if reload or recheck:
try:
request = options.root + my_path
needed = None
if requested_ranges != None:
if '_parts' in record and record['_parts'] != None:
needed = record['_parts'] & requested_ranges
else:
needed = requested_ranges
ranges = ()
print('Requesting ranges:', ranges)
print('Not stored ranges:', record['_parts'])
print('Requested ranges:', requested_ranges)
print('Needed ranges:', needed)
needed.rewind()
while True:
range = needed.pop()
if range[0] == None:
break
ranges += '{}-{}'.format(range[0], range[1] - 1),
request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
if len(needed) > 0:
needed.rewind()
while True:
range = needed.pop()
if range[0] == None:
break
ranges += '{}-{}'.format(range[0], range[1] - 1),
request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
with urllib.request.urlopen(request) as source:
new_record = {}
new_record['_parts'] = record['_parts']
headers = source.info()
# stripping unneeded headers (XXX make this inplace?)
for header in headers:
if header in desc_fields:
#if header == 'Pragma' and headers[header] != 'no-cache':
print(header, headers[header])
if header == 'Content-Length':
if 'Content-Range' not in headers:
new_record[header] = headers[header]
new_record[header] = int(headers[header])
else:
new_record[header] = headers[header]
elif header == 'Content-Range':
range = re.compile('^bytes (\d+)-(\d+)/(\d+)$').match(headers[header])
if range:
new_record['Content-Length'] = range.group(3)
new_record['Content-Length'] = int(range.group(3))
else:
assert False, 'Content-Range unrecognized.'
elif not header in ignore_fields:
print('Undefined header "', header, '": ', headers[header], sep='')
if new_record['_parts'] == None:
new_record['_parts'] = SpaceMap({0: int(new_record['Content-Length'])})
print(new_record)
# comparing headers with data found in index
# if any header has changed (except Pragma) file is fully downloaded
# same if we get more or less headers
old_keys = set(record.keys())
old_keys.discard('_time')
old_keys.discard('Pragma')
more_keys = set(new_record.keys()) - old_keys
more_keys.discard('Pragma')
less_keys = old_keys - set(new_record.keys())
if len(more_keys) > 0:
if not len(old_keys) == 0:
print('More headers appear:', more_keys)
reload = True
elif len(less_keys) > 0:
print('Less headers appear:', less_keys)
else:
for key in record.keys():
if key[0] != '_' and key != 'Pragma' and not record[key] == new_record[key]:
print('Header "', key, '" changed from [', record[key], '] to [', new_record[key], ']', sep='')
print(type(record[key]), type(new_record[key]))
reload = True
if reload:
print('Reloading.')
if os.access(temp_name, os.R_OK):
os.unlink(temp_name)
if os.access(file_name, os.R_OK):
os.unlink(file_name)
if new_record['_parts'] == None or reload:
new_record['_parts'] = SpaceMap({0: int(new_record['Content-Length'])})
print(new_record)
# downloading file or segment
if 'Content-Length' in new_record:
if requested_ranges == None:
requested_ranges = new_record['_parts']
if needed == None:
needed = new_record['_parts']
else:
if len(requested_ranges) > 1:
if len(needed) > 1:
print("Multipart requests currently not supported.")
assert False, 'Skip this one for now.'
else:
assert False, 'No Content-Length or Content-Range header.'
if reload:
new_record['_time'] = datetime.datetime.now()
if self.command not in ('HEAD'):
# file is created at temporary location and moved in place only when download completes
if not os.access(temp_name, os.R_OK):
empty_name = options.dir + os.sep + '.tmp'
with open(empty_name, 'w+b') as some_file:
pass
os.renames(empty_name, temp_name)
temp_file = open(temp_name, 'r+b')
requested_ranges.rewind()
while True:
(start, end) = requested_ranges.pop()
if start == None:
break
stream_last = start
old_record = new_record
if end - start < block_size:
req_block_size = end - start
else:
req_block_size = block_size
buffer = source.read(req_block_size)
new_record['_time'] = datetime.datetime.now()
if self.command not in ('HEAD'):
# file is created at temporary location and moved in place only when download completes
if not os.access(temp_name, os.R_OK):
empty_name = options.dir + os.sep + '.tmp'
with open(empty_name, 'w+b') as some_file:
pass
os.renames(empty_name, temp_name)
temp_file = open(temp_name, 'r+b')
needed.rewind()
while True:
(start, end) = needed.pop()
if start == None:
break
stream_last = start
old_record = new_record
if end - start < block_size:
req_block_size = end - start
else:
req_block_size = block_size
buffer = source.read(req_block_size)
print(buffer)
length = len(buffer)
while length > 0 and stream_last < end:
stream_pos = stream_last + length
assert not stream_pos > end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
length = len(buffer)
while length > 0 and stream_last < end:
stream_pos = stream_last + length
assert not stream_pos > end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
print('Writing', length, 'bytes to temp file at position', stream_last)
temp_file.seek(stream_last)
temp_file.write(buffer)
new_record['_parts'] = new_record['_parts'] - SpaceMap({stream_last: stream_pos})
temp_file.seek(stream_last)
temp_file.write(buffer)
new_record['_parts'] = new_record['_parts'] - SpaceMap({stream_last: stream_pos})
print(new_record)
index[my_path] = old_record
index.sync()
old_record = new_record
stream_last = stream_pos
if end - stream_last < block_size:
req_block_size = end - stream_last
buffer = source.read(req_block_size)
index[my_path] = old_record
index.sync()
old_record = new_record
stream_last = stream_pos
if end - stream_last < block_size:
req_block_size = end - stream_last
buffer = source.read(req_block_size)
print(buffer)
length = len(buffer)
length = len(buffer)
print(new_record)
index[my_path] = new_record
index.sync()
temp_file.close()
# moving downloaded data to real file
if new_record['_parts'] == SpaceMap():
# just moving
# drop old dirs XXX
print('Moving temporary file to new destination.')
os.renames(temp_name, file_name)
# moving downloaded data to real file
temp_file.close()
if new_record['_parts'] == SpaceMap():
# just moving
# drop old dirs XXX
print('Moving temporary file to new destination.')
os.renames(temp_name, file_name)
print(new_record)
index[my_path] = new_record
index.sync()
except urllib.error.HTTPError as error:
# in case of error we don't need to do anything actually,
# if file download stalls or fails the file would not be moved to it's location
print(error)
print('Sending response.')
if self.command == 'HEAD':
print('Sending HEAD response.')
self.send_response(200)
if 'Content-Length' in index[my_path]:
self.send_header('Content-Length', index[my_path]['Content-Length'])
self.send_header('Accept-Ranges', 'bytes')
self.send_header('Content-Type', 'application/octet-stream')
if 'Last-Modified' in index[my_path]:
self.send_header('Last-Modified', index[my_path]['Last-Modified'])
self.end_headers()
else:
if index[my_path]['_parts'] != SpaceMap():
file_name = temp_name
with open(file_name, 'rb') as real_file:
file_stat = os.stat(file_name)
if 'Range' in self.headers:
self.send_response(200)
self.send_response(206)
self.send_header('Last-Modified', index[my_path]['Last-Modified'])
if requested_ranges != None:
ranges = ()
requested_ranges.rewind()
while True:
pair = requested_ranges.pop()
if pair[0] == None:
break
ranges += '{}-{}'.format(pair[0], str(pair[1] - 1)),
self.send_header('Content-Range', 'bytes ' + ','.join(ranges) + '/' + index[my_path]['Content-Length'])
self.send_header('Content-Range', 'bytes {}/{}'.format(','.join(ranges), index[my_path]['Content-Length']))
else:
self.send_response(200)
self.send_header('Content-Length', str(file_stat.st_size))
requested_ranges = SpaceMap({0: file_stat.st_size})
self.send_header('Last-Modified', index[my_path]['Last-Modified'])
self.send_header('Content-Type', 'application/octet-stream')
self.end_headers()
if self.command in ('GET'):
if len(requested_ranges) > 0:
requested_ranges.rewind()
(start, end) = requested_ranges.pop()
print('Seeking file to position', start)
requested_ranges.rewind()
(start, end) = requested_ranges.pop()
else:
start = 0
end = index[my_path]['Content-Length']
real_file.seek(start)
if block_size > end - start:
req_block_size = end - start
else:
req_block_size = block_size
print('block_size is', req_block_size)
buffer = real_file.read(req_block_size)
length = len(buffer)
while length > 0:
self.wfile.write(buffer)
start += len(buffer)
if req_block_size > end - start:
req_block_size = end - start
if req_block_size == 0:
break
print('block_size is', req_block_size)
buffer = real_file.read(req_block_size)
length = len(buffer)
def do_HEAD(self):
return self.__process()
def do_GET(self):
return self.__process()
|