︙ | | |
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
-
+
|
# _parts - list of stored parts of file
# _time - last time the file was checked
# everything else is just the headers
index = shelve.open(config['dir'] + os.sep + '.index')
desc_fields = const_desc_fields.copy()
ignore_fields = const_ignore_fields.copy()
if not config['noetag']:
if config['noetag'] == 'no':
desc_fields.add('ETag')
else:
ignore_fields.add('ETag')
proxy_ignored = set([
'Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language',
'Cache-Control', 'Connection', 'Content-Length', 'Cookie',
|
︙ | | |
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
|
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
|
+
+
-
+
-
|
# space map generally covers every bit of file we don't posess currently
if not my_path in index:
info += '\nThis one is new.'
reload = True
record = {}
else:
# forcibly checking file if no file present
record = index[my_path]
if os.access(file_name, os.R_OK):
info += '\nFull file found.'
file_stat = os.stat(file_name)
elif '_parts' in index[my_path] and os.access(temp_name, os.R_OK):
info += '\nPartial file found.'
file_stat = os.stat(temp_name)
recheck = True
else:
info += '\nFile not found or inaccessible.'
index[my_path]['_parts'] = None
record['_parts'] = None
reload = True
record = index[my_path]
if not '_parts' in record:
record['_parts'] = None
if record['_parts'] == None:
recheck = True
|
︙ | | |
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
|
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
|
-
+
-
+
|
my_headers = {}
for header in ('Cache-Control', 'Cookie', 'Referer', 'User-Agent'):
if header in self.headers:
my_headers[header] = self.headers[header]
needed = None
if '_parts' in record and record['_parts'] != None:
if config['noparts'] or requested_ranges == None:
if config['noparts'] != 'no' or requested_ranges == None or requested_ranges == spacemap.SpaceMap():
needed = record['_parts']
else:
needed = record['_parts'] | requested_ranges
elif not config['noparts']:
elif config['noparts'] =='no' and requested_ranges != None and requested_ranges != spacemap.SpaceMap():
needed = requested_ranges
ranges = ()
print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
if needed != None and len(needed) > 0:
needed.rewind()
while True:
range = needed.pop()
|
︙ | | |
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
|
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
|
-
+
+
|
if end - stream_last < block_size:
req_block_size = end - stream_last
buffer = source.read(req_block_size)
length = len(buffer)
# moving downloaded data to real file
temp_file.close()
print(new_record)
index[my_path] = new_record
index.sync()
except urllib.error.HTTPError as error:
# in case of error we don't need to do anything actually,
# if file download stalls or fails the file would not be moved to it's location
print(error)
print(index[my_path])
if not os.access(file_name, os.R_OK) and os.access(temp_name, os.R_OK) and '_parts' in index[my_path] and index[my_path]['_parts'] == spacemap.SpaceMap():
# just moving
# drop old dirs XXX
print('Moving temporary file to new destination.')
os.renames(temp_name, file_name)
|
︙ | | |