Lines of
samesite.py
from check-in d0071bdbc7
that are changed by the sequence of edits moving toward
check-in fb10031536:
1: #!/usr/bin/env python3.1
2:
3: import datetime, http.cookiejar, optparse, os, sys, shelve, re, urllib.request
4:
5: from spacemap import SpaceMap
6:
7: parser = optparse.OptionParser()
8: parser.add_option('-v', '--verbose', action = 'store_true', dest = 'verbose', help = 'turns on verbose status notifications', metavar = 'bool', default = False)
9: parser.add_option('-d', '--dir', action = 'store', dest = 'dir', help = 'specify directory where the files should be stored', metavar = 'string', default = None)
10: parser.add_option('-r', '--root', action = 'store', dest = 'root', help = 'specify a site from which data should be mirrored', metavar = 'string', default = None)
11: parser.add_option('-l', '--log', action = 'store', dest = 'log', help = 'specify a log file to process', metavar = 'string', default = None)
12: parser.add_option('-e', '--skip-etag', action = 'store_true', dest = 'noetag', help = 'do not process etags', metavar = 'bool', default = False)
13: parser.add_option('-p', '--port', action = 'store', dest = 'port', help = 'listen on this port for incoming connections', metavar = 'integer', default = None)
14: parser.add_option('-n', '--no-update', action = 'store_true', dest = 'noupdate', help = 'do not update already downloaded files', metavar = 'bool', default = 'False')
15: (options, args) = parser.parse_args()
16:
17: assert options.dir, 'Directory not specified'
18: assert options.root, 'Server not specified'
19: assert options.log or options.port, 'Log file or port not specified'
20: assert options.port or os.access(options.log, os.R_OK), 'Log file unreadable'
21:
22: optionsDirWithSep = re.compile('^(.*?)/?$').match(options.dir)
23: if optionsDirWithSep:
24: options.dir = optionsDirWithSep.group(1)
25:
26: # this is file index - everything is stored in this file
27: # _parts - list of stored parts of file
28: # _time - last time the file was checked
29: # everything else is just the headers
30: index = shelve.open(options.dir + os.sep + '.index')
31: desc_fields = ('Content-Length', 'Pragma', 'Last-Modified')
32: ignore_fields = ('Accept-Ranges', 'Age', 'Cache-Control', 'Connection', 'Content-Type', 'Date', 'Expires', 'Server', 'Via', 'X-Cache', 'X-Cache-Lookup', 'X-Powered-By')
33:
34: if not options.noetag:
35: desc_fields += 'ETag',
36: else:
37: ignore_fields += 'ETag',
38:
39: block_size = 4096
40:
41: temp_file_name = options.dir + os.sep + '.tmp'
42:
43: '''
44: # later, kqueue would be good but later
45: class Connection:
46: __slots__ = frozenset(('__address', '__input', '__socket', '__status', 'error', 'method', 'url', 'http_version'))
47:
48: def __init__(self, socket, address):
49: self.__address = address
50: self.__input = b''
51: self.__socket = socket
52: self.__status = 0
53:
54: def read(self, kev):
55: buffer = self.__socket.recv(kev.data)
56: exhausted = False
57: if len(buffer) == 0:
58: eof = True
59: else:
60: self.__input += buffer
61: while not exhausted:
62: if self.__status == -1:
63: exhausted = True
64: elif self.__status == 0:
65: endstring = self.__input.find(b'\n')
66: if endstring > 0:
67: print('Processing request line.')
68: line = self.__input[:endstring].decode('ascii')
69: self.__input = self.__input[endstring + 1:]
70: isRequest = re.compile('(GET) ([^ ]+) HTTP/(1\.0)').match(line)
71: if not isRequest:
72: self.error = 'Not a HTTP connection.'
73: self.__status = -1
74: else:
75: self.method = isRequest.group(1)
76: self.url = isRequest.group(2)
77: self.http_version = isRequest.group(3)
78: self.__status = 1
79: else:
80: exhausted = True
81: elif self.__status == 1:
82: endstring = self.__input.find(b'\n')
83: if endstring > 0:
84: print('Processing header line.' + repr(self.__input))
85: line = self.__input[:endstring].decode('ascii')
86: self.__input = self.__input[endstring + 1:]
87: isHeader = re.compile('([^:]*): +(.*)').match(line)
88: if not isHeader:
89: self.error = 'Bad header.'
90: return(False)
91: # process header here
92: elif endstring == 0:
93: self.__status = 2
94: else:
95: exhausted = True
96:
97: def write(self, kev):
98: pass
99:
100: if options.port:
101: import select, socket
102:
103: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
104: try:
105: sock.bind(('127.0.0.1', int(options.port)))
106: sock.listen(-1)
107:
108: kq = select.kqueue()
109: assert kq.fileno() != -1, "Fatal error: can't initialise kqueue."
110:
111: kq.control([select.kevent(sock, select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0)
112: timeout = None
113:
114: connections = {sock.fileno(): None}
115:
116: while True:
117: kevs = kq.control(None, 1, timeout)
118:
119: for kev in kevs:
120: if type(connections[kev.ident]) == Connection:
121: print(kev.ident, kev.data, kev.filter, kev.flags)
122: assert kev.data != 0, 'No data available.'
123: if kev.filter == select.KQ_FILTER_READ:
124: connections[kev.ident].read(kev)
125: elif kev.filter == select.KQ_FILTER_WRITE:
126: connections[kev.ident].write(kev)
127: else:
128: assert kev.filter in (select.KQ_FILTER_READ, select.KQ_FILTER_WRITE), 'Do we support other filters?'
129: else:
130: (conn, addr) = sock.accept()
131: print('Connection from ' + repr(addr))
132: kq.control([select.kevent(conn, select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0)
133: connections[conn.fileno()] = Connection(conn, addr)
134:
135: if kev.flags >> 15 == 1:
136: kq.control([select.kevent(kev.ident, select.KQ_FILTER_READ, select.KQ_EV_DELETE)], 0)
137: kq.control([select.kevent(kev.ident, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)], 0)
138: del(connections[kev.ident])
139: finally:
140: sock.close()
141: '''
142:
143: if options.port:
144: import http.server
145:
146: class MyRequestHandler(http.server.BaseHTTPRequestHandler):
147: def __process(self):
148: # reload means file needs to be reloaded to serve request
149: reload = False
150: # recheck means file needs to be checked, this also means that if file hav been modified we can serve older copy
151: recheck = False
152: # file_stat means file definitely exists
153: file_stat = None
154: # requested_ranges holds data about any range requested
155: requested_ranges = None
156: # records holds data from index locally, should be written back upon successfull completion
157: record = None
158: info = 'Checking file: ' + self.path
159:
160: myPath = re.compile('^(.*?)(\?.*)$').match(self.path)
161: if myPath:
162: my_path = myPath.group(1)
163: else:
164: my_path = self.path
165:
166: proxy_ignored = ('Accept', 'Accept-Encoding',
167: 'Cache-Control', 'Connection',
168: 'Host',
169: 'User-Agent',
170: 'Via',
171: 'X-Forwarded-For',
172: )
173:
d0071bdbc7 2010-08-20 174: print('===============[ Request ]===')
d0071bdbc7 2010-08-20 175: print('Command:', self.command)
176:
177: for header in self.headers:
178: if header in proxy_ignored:
179: pass
180: elif header in ('Range'):
181: isRange = re.compile('bytes=(\d+)-(\d+)').match(self.headers[header])
182: if isRange:
183: requested_ranges = SpaceMap({int(isRange.group(1)): int(isRange.group(2)) + 1})
184: else:
185: return()
186: else:
187: print('Unknown header - ', header, ': ', self.headers[header], sep='')
188: return()
189: print(header, self.headers[header])
d0071bdbc7 2010-08-20 190: print(my_path)
191:
192: # creating empty placeholder in index
193: # if there's no space map and there's no file in real directory - we have no file
194: # if there's an empty space map - file is full
195: # space map generally covers every bit of file we don't posess currently
196: if not my_path in index:
197: info += '\nThis one is new.'
198: reload = True
199: record = {'_parts': None}
200: else:
201: record = index[my_path]
202: if '_parts' in index[my_path]:
d0071bdbc7 2010-08-20 203: print(record['_parts'])
204: if index[my_path]['_parts'] == {0: -1}:
205: index[my_path]['_parts'] = None
206:
207: # creating file name from my_path
208: file_name = options.dir + os.sep + re.compile('%20').sub(' ', my_path)
209: # partial file or unfinished download
210: temp_name = options.dir + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)
211:
212: # forcibly checking file if no file present
213: if os.access(file_name, os.R_OK):
214: file_stat = os.stat(file_name)
215: elif '_parts' in record and os.access(temp_name, os.R_OK):
216: file_stat = os.stat(temp_name)
217: elif not reload:
218: info += '\nFile not found or inaccessible.'
219: reload = True
220:
221: # forcibly checking file if file size doesn't match with index data
222: if not reload:
223: if '_parts' in record and record['_parts'] == SpaceMap():
224: if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
225: info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
226: reload = True
227:
228: # forcibly checking file if index holds Pragma header
229: if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache':
230: info +='\nPragma on: recheck imminent.'
231: recheck = True
232:
233: # skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
234: if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0:
235: recheck = True
236:
237: print(info)
238: if reload or recheck:
239:
240: try:
241: request = options.root + my_path
242: if requested_ranges != None:
243: if '_parts' in record and record['_parts'] != None:
244: needed = record['_parts'] & requested_ranges
245: else:
246: needed = requested_ranges
247: ranges = ()
d0071bdbc7 2010-08-20 248: print('Requesting ranges:', ranges)
d0071bdbc7 2010-08-20 249: print('Not stored ranges:', record['_parts'])
d0071bdbc7 2010-08-20 250: print('Requested ranges:', requested_ranges)
d0071bdbc7 2010-08-20 251: print('Needed ranges:', needed)
d0071bdbc7 2010-08-20 252: needed.rewind()
d0071bdbc7 2010-08-20 253: while True:
d0071bdbc7 2010-08-20 254: range = needed.pop()
d0071bdbc7 2010-08-20 255: if range[0] == None:
d0071bdbc7 2010-08-20 256: break
d0071bdbc7 2010-08-20 257: ranges += '{}-{}'.format(range[0], range[1] - 1),
d0071bdbc7 2010-08-20 258: request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)})
259:
260: with urllib.request.urlopen(request) as source:
261: new_record = {}
262: new_record['_parts'] = record['_parts']
263: headers = source.info()
264:
265: # stripping unneeded headers (XXX make this inplace?)
266: for header in headers:
267: if header in desc_fields:
268: #if header == 'Pragma' and headers[header] != 'no-cache':
d0071bdbc7 2010-08-20 269: print(header, headers[header])
270: if header == 'Content-Length':
271: if 'Content-Range' not in headers:
d0071bdbc7 2010-08-20 272: new_record[header] = headers[header]
273: else:
274: new_record[header] = headers[header]
275: elif header == 'Content-Range':
276: range = re.compile('^bytes (\d+)-(\d+)/(\d+)$').match(headers[header])
277: if range:
d0071bdbc7 2010-08-20 278: new_record['Content-Length'] = range.group(3)
279: else:
280: assert False, 'Content-Range unrecognized.'
281: elif not header in ignore_fields:
282: print('Undefined header "', header, '": ', headers[header], sep='')
d0071bdbc7 2010-08-20 283:
d0071bdbc7 2010-08-20 284: if new_record['_parts'] == None:
d0071bdbc7 2010-08-20 285: new_record['_parts'] = SpaceMap({0: int(new_record['Content-Length'])})
d0071bdbc7 2010-08-20 286: print(new_record)
287:
288: # comparing headers with data found in index
289: # if any header has changed (except Pragma) file is fully downloaded
290: # same if we get more or less headers
291: old_keys = set(record.keys())
292: old_keys.discard('_time')
293: old_keys.discard('Pragma')
294: more_keys = set(new_record.keys()) - old_keys
295: more_keys.discard('Pragma')
296: less_keys = old_keys - set(new_record.keys())
297: if len(more_keys) > 0:
298: if not len(old_keys) == 0:
299: print('More headers appear:', more_keys)
300: reload = True
301: elif len(less_keys) > 0:
302: print('Less headers appear:', less_keys)
303: else:
304: for key in record.keys():
305: if key[0] != '_' and key != 'Pragma' and not record[key] == new_record[key]:
306: print('Header "', key, '" changed from [', record[key], '] to [', new_record[key], ']', sep='')
307: reload = True
308:
309: if reload:
310: print('Reloading.')
311: if os.access(temp_name, os.R_OK):
312: os.unlink(temp_name)
313: if os.access(file_name, os.R_OK):
314: os.unlink(file_name)
315:
316: # downloading file or segment
317: if 'Content-Length' in new_record:
d0071bdbc7 2010-08-20 318: if requested_ranges == None:
d0071bdbc7 2010-08-20 319: requested_ranges = new_record['_parts']
320: else:
d0071bdbc7 2010-08-20 321: if len(requested_ranges) > 1:
322: print("Multipart requests currently not supported.")
323: assert False, 'Skip this one for now.'
324: else:
325: assert False, 'No Content-Length or Content-Range header.'
326:
d0071bdbc7 2010-08-20 327: if reload:
d0071bdbc7 2010-08-20 328: new_record['_time'] = datetime.datetime.now()
d0071bdbc7 2010-08-20 329: if self.command not in ('HEAD'):
d0071bdbc7 2010-08-20 330: # file is created at temporary location and moved in place only when download completes
d0071bdbc7 2010-08-20 331: if not os.access(temp_name, os.R_OK):
d0071bdbc7 2010-08-20 332: empty_name = options.dir + os.sep + '.tmp'
d0071bdbc7 2010-08-20 333: with open(empty_name, 'w+b') as some_file:
d0071bdbc7 2010-08-20 334: pass
d0071bdbc7 2010-08-20 335: os.renames(empty_name, temp_name)
d0071bdbc7 2010-08-20 336: temp_file = open(temp_name, 'r+b')
d0071bdbc7 2010-08-20 337: requested_ranges.rewind()
d0071bdbc7 2010-08-20 338: while True:
d0071bdbc7 2010-08-20 339: (start, end) = requested_ranges.pop()
d0071bdbc7 2010-08-20 340: if start == None:
d0071bdbc7 2010-08-20 341: break
d0071bdbc7 2010-08-20 342: stream_last = start
d0071bdbc7 2010-08-20 343: old_record = new_record
d0071bdbc7 2010-08-20 344: if end - start < block_size:
d0071bdbc7 2010-08-20 345: req_block_size = end - start
d0071bdbc7 2010-08-20 346: else:
d0071bdbc7 2010-08-20 347: req_block_size = block_size
d0071bdbc7 2010-08-20 348: buffer = source.read(req_block_size)
d0071bdbc7 2010-08-20 349: print(buffer)
d0071bdbc7 2010-08-20 350: length = len(buffer)
d0071bdbc7 2010-08-20 351: while length > 0 and stream_last < end:
d0071bdbc7 2010-08-20 352: stream_pos = stream_last + length
d0071bdbc7 2010-08-20 353: assert not stream_pos > end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
d0071bdbc7 2010-08-20 354: print('Writing', length, 'bytes to temp file at position', stream_last)
d0071bdbc7 2010-08-20 355: temp_file.seek(stream_last)
d0071bdbc7 2010-08-20 356: temp_file.write(buffer)
d0071bdbc7 2010-08-20 357: new_record['_parts'] = new_record['_parts'] - SpaceMap({stream_last: stream_pos})
d0071bdbc7 2010-08-20 358: print(new_record)
d0071bdbc7 2010-08-20 359: index[my_path] = old_record
d0071bdbc7 2010-08-20 360: index.sync()
d0071bdbc7 2010-08-20 361: old_record = new_record
d0071bdbc7 2010-08-20 362: stream_last = stream_pos
d0071bdbc7 2010-08-20 363: if end - stream_last < block_size:
d0071bdbc7 2010-08-20 364: req_block_size = end - stream_last
d0071bdbc7 2010-08-20 365: buffer = source.read(req_block_size)
d0071bdbc7 2010-08-20 366: print(buffer)
d0071bdbc7 2010-08-20 367: length = len(buffer)
d0071bdbc7 2010-08-20 368: print(new_record)
d0071bdbc7 2010-08-20 369: index[my_path] = new_record
d0071bdbc7 2010-08-20 370: index.sync()
d0071bdbc7 2010-08-20 371: temp_file.close()
d0071bdbc7 2010-08-20 372:
d0071bdbc7 2010-08-20 373: # moving downloaded data to real file
d0071bdbc7 2010-08-20 374: if new_record['_parts'] == SpaceMap():
d0071bdbc7 2010-08-20 375: # just moving
d0071bdbc7 2010-08-20 376: # drop old dirs XXX
d0071bdbc7 2010-08-20 377: print('Moving temporary file to new destination.')
d0071bdbc7 2010-08-20 378: os.renames(temp_name, file_name)
379:
380: except urllib.error.HTTPError as error:
381: # in case of error we don't need to do anything actually,
382: # if file download stalls or fails the file would not be moved to it's location
383: print(error)
384:
d0071bdbc7 2010-08-20 385: print('Sending response.')
386: if self.command == 'HEAD':
d0071bdbc7 2010-08-20 387: print('Sending HEAD response.')
388: self.send_response(200)
389: if 'Content-Length' in index[my_path]:
390: self.send_header('Content-Length', index[my_path]['Content-Length'])
391: self.send_header('Accept-Ranges', 'bytes')
392: self.send_header('Content-Type', 'application/octet-stream')
393: if 'Last-Modified' in index[my_path]:
394: self.send_header('Last-Modified', index[my_path]['Last-Modified'])
395: self.end_headers()
396: else:
397: if index[my_path]['_parts'] != SpaceMap():
398: file_name = temp_name
399:
400: with open(file_name, 'rb') as real_file:
401: file_stat = os.stat(file_name)
d0071bdbc7 2010-08-20 402: self.send_response(200)
d0071bdbc7 2010-08-20 403: self.send_header('Last-Modified', index[my_path]['Last-Modified'])
d0071bdbc7 2010-08-20 404: if requested_ranges != None:
405: ranges = ()
406: requested_ranges.rewind()
407: while True:
408: pair = requested_ranges.pop()
409: if pair[0] == None:
410: break
411: ranges += '{}-{}'.format(pair[0], str(pair[1] - 1)),
d0071bdbc7 2010-08-20 412: self.send_header('Content-Range', 'bytes ' + ','.join(ranges) + '/' + index[my_path]['Content-Length'])
413: else:
414: self.send_header('Content-Length', str(file_stat.st_size))
415: requested_ranges = SpaceMap({0: file_stat.st_size})
416: self.send_header('Content-Type', 'application/octet-stream')
417: self.end_headers()
418: if self.command in ('GET'):
d0071bdbc7 2010-08-20 419: requested_ranges.rewind()
d0071bdbc7 2010-08-20 420: (start, end) = requested_ranges.pop()
d0071bdbc7 2010-08-20 421: print('Seeking file to position', start)
422: real_file.seek(start)
423: if block_size > end - start:
424: req_block_size = end - start
425: else:
426: req_block_size = block_size
d0071bdbc7 2010-08-20 427: print('block_size is', req_block_size)
428: buffer = real_file.read(req_block_size)
429: length = len(buffer)
430: while length > 0:
431: self.wfile.write(buffer)
432: start += len(buffer)
433: if req_block_size > end - start:
434: req_block_size = end - start
435: if req_block_size == 0:
436: break
d0071bdbc7 2010-08-20 437: print('block_size is', req_block_size)
438: buffer = real_file.read(req_block_size)
439: length = len(buffer)
440:
441: def do_HEAD(self):
442: return self.__process()
443: def do_GET(self):
444: return self.__process()
445:
446: server = http.server.HTTPServer(('127.0.0.1', int(options.port)), MyRequestHandler)
447: server.serve_forever()
448:
449: else:
450: while True:
451: unchecked_files = set()
452: checked_files = 0
453:
454: # reading log and storing found urls for processing
455: # check file mtime XXX
456: with open(options.log, 'r') as log_file:
457: log_line = re.compile('^[^ ]+ - - \[.*] "(GET|HEAD) (.*?)(\?.*)? HTTP/1.1" (\d+) \d+ "(.*)" "(.*)"$')
458: for line in log_file:
459: this_line = log_line.match(line.strip())
460: if this_line:
461: unchecked_files.add(this_line.group(2))
462:
463: for url in unchecked_files:
464: reload = False
465: recheck = False
466: info = 'Checking file: ' + url
467:
468: # creating empty placeholder in index
469: if not url in index:
470: info += '\nThis one is new.'
471: index[url] = {}
472: reload = True
473:
474: # creating file name from url
475: file_name = options.dir + re.compile('%20').sub(' ', url)
476:
477: # forcibly checking file if no file present
478: if not reload and not os.access(file_name, os.R_OK):
479: info += '\nFile not found or inaccessible.'
480: reload = True
481:
482: # forcibly checking file if file size doesn't match with index data
483: elif not reload and 'Content-Length' in index[url] and os.stat(file_name).st_size != int(index[url]['Content-Length']):
484: info += '\nFile size is ' + os.stat(file_name).st_size + ' and stored file size is ' + index[url]['Content-Length'] + '.'
485: reload = True
486:
487: # forcibly checking file if index hods Pragma header
488: if not reload and 'Pragma' in index[url] and index[url]['Pragma'] == 'no-cache':
489: info +='\nPragma on: recheck imminent.'
490: recheck = True
491:
492: # skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
493: if not recheck and not reload and (options.noupdate or ('_time' in index[url] and (datetime.datetime.now() - datetime.timedelta(hours = 4) - index[url]['_time']).days < 0)):
494: if options.verbose:
495: print(info)
496: continue
497: else:
498: print(info)
499:
500: try:
501: with urllib.request.urlopen(options.root + url) as source:
502: new_headers = {}
503: headers = source.info()
504:
505: # stripping unneeded headers (XXX make this inplace?)
506: for header in headers:
507: if header in desc_fields:
508: if header == 'Pragma' and headers[header] != 'no-cache':
509: print('Pragma:', headers[header])
510: new_headers[header] = headers[header]
511: elif not header in ignore_fields:
512: print('Undefined header "', header, '": ', headers[header], sep='')
513:
514: # comparing headers with data found in index
515: # if any header has changed (except Pragma) file is fully downloaded
516: # same if we get more or less headers
517: old_keys = set(index[url].keys())
518: old_keys.discard('_time')
519: old_keys.discard('Pragma')
520: more_keys = set(new_headers.keys()) - old_keys
521: more_keys.discard('Pragma')
522: less_keys = old_keys - set(new_headers.keys())
523: if len(more_keys) > 0:
524: if not len(old_keys) == 0:
525: print('More headers appear:', more_keys)
526: reload = True
527: elif len(less_keys) > 0:
528: print('Less headers appear:', less_keys)
529: else:
530: for key in index[url].keys():
531: if key[0] != '_' and key != 'Pragma' and not index[url][key] == new_headers[key]:
532: print('Header "', key, '" changed from [', index[url][key], '] to [', new_headers[key], ']', sep='')
533: reload = True
534:
535: # downloading file
536: if reload:
537: if 'Content-Length' in headers:
538: print('Downloading', headers['Content-Length'], 'bytes [', end='')
539: else:
540: print('Downloading [', end='')
541: sys.stdout.flush()
542:
543: # file is created at temporary location and moved in place only when download completes
544: temp_file = open(options.dir + os.sep + '.tmp', 'wb')
545: buffer = source.read(block_size)
546: megablocks = 0
547: blocks = 0
548: megs = 0
549: while len(buffer) > 0:
550: temp_file.write(buffer)
551: buffer = source.read(block_size)
552: blocks += 1
553: if blocks > 102400/block_size:
554: megablocks += 1
555: if megablocks > 10:
556: megablocks = megablocks - 10
557: megs += 1
558: print('{}Mb'.format(megs), end='')
559: else:
560: print('.', end='')
561: blocks = blocks - 102400/block_size
562: sys.stdout.flush()
563: temp_file.close()
564: print(']')
565: os.renames(options.dir + os.sep + '.tmp', file_name)
566:
567: checked_files += 1
568:
569: # storing new time mark and storing new headers
570: new_headers['_time'] = datetime.datetime.now()
571: index[url] = new_headers
572: index.sync()
573:
574: except urllib.error.HTTPError as error:
575: # in case of error we don't need to do anything actually,
576: # if file download stalls or fails the file would not be moved to it's location
577: print(error)
578:
579: if options.verbose:
580: print('[', len(unchecked_files), '/', checked_files, ']')
581:
582: # checking if there were any files downloaded, if yes - restarting sequence
583: if checked_files == 0:
584: break