244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
|
if my_path in index:
index[my_path][header] = self.headers[header]
else:
print('Unknown header - ', header, ': ', self.headers[header], sep='')
return()
print(header, self.headers[header])
# creating empty placeholder in index
# if there's no space map and there's no file in real directory - we have no file
# if there's an empty space map - file is full
# space map generally covers every bit of file we don't posess currently
if not my_path in index:
info += '\nThis one is new.'
reload = True
record = {}
else:
record = index[my_path]
if not '_parts' in record:
record['_parts'] = None
# creating file name from my_path
file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
# partial file or unfinished download
temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)
# forcibly checking file if no file present
if os.access(file_name, os.R_OK):
file_stat = os.stat(file_name)
elif '_parts' in record and os.access(temp_name, os.R_OK):
file_stat = os.stat(temp_name)
elif not reload:
print(record)
info += '\nFile not found or inaccessible.'
record['_parts'] = None
reload = True
# forcibly checking file if file size doesn't match with index data
if not reload:
if '_parts' in record and record['_parts'] == spacemap.SpaceMap():
if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
reload = True
# forcibly checking file if index holds Pragma header
if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache':
info +='\nPragma on: recheck imminent.'
recheck = True
# skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0:
recheck = True
print(info)
if reload or recheck:
try:
request = 'http://' + config['root'] + my_path
needed = None
# XXX and if we specify full file we don't go partial?
if requested_ranges != None:
if '_parts' in record and record['_parts'] != None:
needed = record['_parts'] & requested_ranges
elif config['noparts']:
needed = record['_parts']
else:
needed = requested_ranges
ranges = ()
print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
if len(needed) > 0:
needed.rewind()
while True:
range = needed.pop()
if range[0] == None:
break
ranges += '{}-{}'.format(range[0], range[1] - 1),
request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
<
<
<
<
<
<
<
<
<
<
<
<
<
|
|
>
<
|
|
|
>
>
|
|
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
|
if my_path in index:
index[my_path][header] = self.headers[header]
else:
print('Unknown header - ', header, ': ', self.headers[header], sep='')
return()
print(header, self.headers[header])
# creating file name from my_path
file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
# partial file or unfinished download
temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)
# creating empty placeholder in index
# if there's no space map and there's no file in real directory - we have no file
# if there's an empty space map - file is full
# space map generally covers every bit of file we don't posess currently
if not my_path in index:
info += '\nThis one is new.'
reload = True
record = {}
else:
# forcibly checking file if no file present
if os.access(file_name, os.R_OK):
file_stat = os.stat(file_name)
elif '_parts' in index[my_path] and os.access(temp_name, os.R_OK):
file_stat = os.stat(temp_name)
else:
info += '\nFile not found or inaccessible.'
index[my_path]['_parts'] = None
reload = True
record = index[my_path]
print(record)
if not '_parts' in record:
record['_parts'] = None
if record['_parts'] == None:
recheck = True
# forcibly checking file if file size doesn't match with index data
if not reload:
if '_parts' in record and record['_parts'] == spacemap.SpaceMap():
if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
record['_parts'] = None
reload = True
# forcibly checking file if index holds Pragma header
if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache':
info +='\nPragma on: recheck imminent.'
recheck = True
# skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0:
recheck = True
print(info)
if reload or recheck:
try:
request = 'http://' + config['root'] + my_path
needed = None
# XXX and if we specify full file we don't go partial?
if requested_ranges != None:
if '_parts' in record and record['_parts'] != None:
if config['noparts']:
needed = record['_parts']
else:
needed = record['_parts'] | requested_ranges
elif not config['noparts']:
needed = requested_ranges
ranges = ()
print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
if needed != None and len(needed) > 0:
needed.rewind()
while True:
range = needed.pop()
if range[0] == None:
break
ranges += '{}-{}'.format(range[0], range[1] - 1),
request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
|
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
|
if reload:
print('Reloading.')
if os.access(temp_name, os.R_OK):
os.unlink(temp_name)
if os.access(file_name, os.R_OK):
os.unlink(file_name)
if new_record['_parts'] == None or reload:
new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
print(new_record)
# downloading file or segment
if 'Content-Length' in new_record:
if needed == None:
needed = new_record['_parts']
|
<
|
371
372
373
374
375
376
377
378
379
380
381
382
383
384
|
if reload:
print('Reloading.')
if os.access(temp_name, os.R_OK):
os.unlink(temp_name)
if os.access(file_name, os.R_OK):
os.unlink(file_name)
new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
print(new_record)
# downloading file or segment
if 'Content-Length' in new_record:
if needed == None:
needed = new_record['_parts']
|