08ae38b6ce 2010-06-25 1: #!/usr/bin/env python3.1
08ae38b6ce 2010-06-25 2:
08ae38b6ce 2010-06-25 3: import datetime, http.cookiejar, optparse, os, sys, shelve, re, urllib.request
08ae38b6ce 2010-06-25 4:
08ae38b6ce 2010-06-25 5: parser = optparse.OptionParser()
08ae38b6ce 2010-06-25 6: parser.add_option('-v', '--verbose', action = 'store_true', dest = 'verbose', help = 'turns on verbose status notifications', metavar = 'bool', default = False)
08ae38b6ce 2010-06-25 7: parser.add_option('-d', '--dir', action = 'store', dest = 'dir', help = 'specify directory where the files should be stored', metavar = 'string', default = None)
08ae38b6ce 2010-06-25 8: parser.add_option('-r', '--root', action = 'store', dest = 'root', help = 'specify a site from which data should be mirrored', metavar = 'string', default = None)
08ae38b6ce 2010-06-25 9: parser.add_option('-l', '--log', action = 'store', dest = 'log', help = 'specify a log file to process', metavar = 'string', default = None)
38b25713eb 2010-07-26 10: parser.add_option('-e', '--skip-etag', action = 'store_true', dest = 'noetag', help = 'do not process etags', metavar = 'bool', default = False)
08ae38b6ce 2010-06-25 11: (options, args) = parser.parse_args()
08ae38b6ce 2010-06-25 12:
38b25713eb 2010-07-26 13: assert options.dir, 'Directory not specified'
38b25713eb 2010-07-26 14: assert options.root, 'Server not specified'
38b25713eb 2010-07-26 15: assert options.log, 'Log file not specified'
38b25713eb 2010-07-26 16: assert os.access(options.log, os.R_OK), 'Log file unreadable'
08ae38b6ce 2010-06-25 17:
08ae38b6ce 2010-06-25 18: # this is file index - everything is stored in this file
08ae38b6ce 2010-06-25 19: index = shelve.open(options.dir + '/.index')
38b25713eb 2010-07-26 20: desc_fields = ('Content-Length', 'Pragma', 'Last-Modified')
38b25713eb 2010-07-26 21: ignore_fields = ('Accept-Ranges', 'Age', 'Cache-Control', 'Connection', 'Content-Type', 'Date', 'Expires', 'Server', 'Via', 'X-Cache', 'X-Cache-Lookup', 'X-Powered-By')
38b25713eb 2010-07-26 22:
38b25713eb 2010-07-26 23: if not options.noetag:
38b25713eb 2010-07-26 24: desc_fields += 'ETag',
38b25713eb 2010-07-26 25: else:
38b25713eb 2010-07-26 26: ignore_fields += 'ETag',
7b27f1db02 2010-07-01 27:
7b27f1db02 2010-07-01 28: block_size = 32768
08ae38b6ce 2010-06-25 29:
08ae38b6ce 2010-06-25 30: while True:
08ae38b6ce 2010-06-25 31: unchecked_files = set()
08ae38b6ce 2010-06-25 32: checked_files = 0
08ae38b6ce 2010-06-25 33:
08ae38b6ce 2010-06-25 34: # reading log and storing found urls for processing
08ae38b6ce 2010-06-25 35: # check file mtime XXX
08ae38b6ce 2010-06-25 36: with open(options.log, 'r') as log_file:
7b27f1db02 2010-07-01 37: log_line = re.compile('^[^ ]+ - - \[.*] "(GET|HEAD) (.*?)(\?.*)? HTTP/1.1" (\d+) \d+ "(.*)" "(.*)"$')
08ae38b6ce 2010-06-25 38: for line in log_file:
08ae38b6ce 2010-06-25 39: this_line = log_line.match(line.strip())
08ae38b6ce 2010-06-25 40: if this_line:
7b27f1db02 2010-07-01 41: unchecked_files.add(this_line.group(2))
08ae38b6ce 2010-06-25 42:
08ae38b6ce 2010-06-25 43: for url in unchecked_files:
083ec707ea 2010-07-06 44: reload = False
083ec707ea 2010-07-06 45: recheck = False
083ec707ea 2010-07-06 46: print('Checking file:', url)
08ae38b6ce 2010-06-25 47:
08ae38b6ce 2010-06-25 48: # creating empty placeholder in index
08ae38b6ce 2010-06-25 49: if not url in index:
083ec707ea 2010-07-06 50: print('This one is new.')
08ae38b6ce 2010-06-25 51: index[url] = {}
083ec707ea 2010-07-06 52: reload = True
08ae38b6ce 2010-06-25 53:
08ae38b6ce 2010-06-25 54: # creating file name from url
08ae38b6ce 2010-06-25 55: file_name = options.dir + re.compile('%20').sub(' ', url)
08ae38b6ce 2010-06-25 56:
08ae38b6ce 2010-06-25 57: # forcibly checking file if no file present
083ec707ea 2010-07-06 58: if not reload and not os.access(file_name, os.R_OK):
53dcfdb8f7 2010-07-05 59: print('File not found or inaccessible.')
08ae38b6ce 2010-06-25 60: reload = True
08ae38b6ce 2010-06-25 61:
08ae38b6ce 2010-06-25 62: # forcibly checking file if file size doesn't match with index data
083ec707ea 2010-07-06 63: elif not reload and 'Content-Length' in index[url] and os.stat(file_name).st_size != int(index[url]['Content-Length']):
083ec707ea 2010-07-06 64: print('File size is ', os.stat(file_name).st_size, ' and stored file size is ', index[url]['Content-Length'], '.', sep='')
08ae38b6ce 2010-06-25 65: reload = True
08ae38b6ce 2010-06-25 66:
08ae38b6ce 2010-06-25 67: # forcibly checking file if index hods Pragma header
083ec707ea 2010-07-06 68: if not reload and 'Pragma' in index[url] and index[url]['Pragma'] == 'no-cache':
083ec707ea 2010-07-06 69: print('Pragma on: recheck imminent.')
083ec707ea 2010-07-06 70: recheck = True
08ae38b6ce 2010-06-25 71:
08ae38b6ce 2010-06-25 72: # skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
083ec707ea 2010-07-06 73: if not recheck and not reload and '__time__' in index[url] and (datetime.datetime.now() - datetime.timedelta(hours = 4) - index[url]['__time__']).days < 0:
08ae38b6ce 2010-06-25 74: continue
083ec707ea 2010-07-06 75:
08ae38b6ce 2010-06-25 76: try:
08ae38b6ce 2010-06-25 77: with urllib.request.urlopen(options.root + url) as source:
08ae38b6ce 2010-06-25 78: new_headers = {}
08ae38b6ce 2010-06-25 79: headers = source.info()
08ae38b6ce 2010-06-25 80:
08ae38b6ce 2010-06-25 81: # stripping unneeded headers (XXX make this inplace?)
08ae38b6ce 2010-06-25 82: for header in headers:
08ae38b6ce 2010-06-25 83: if header in desc_fields:
08ae38b6ce 2010-06-25 84: if header == 'Pragma' and headers[header] != 'no-cache':
08ae38b6ce 2010-06-25 85: print('Pragma:', headers[header])
08ae38b6ce 2010-06-25 86: new_headers[header] = headers[header]
08ae38b6ce 2010-06-25 87: elif not header in ignore_fields:
7b27f1db02 2010-07-01 88: print('Undefined header "', header, '": ', headers[header], sep='')
08ae38b6ce 2010-06-25 89:
08ae38b6ce 2010-06-25 90: # comparing headers with data found in index
08ae38b6ce 2010-06-25 91: # if any header has changed (except Pragma) file is fully downloaded
08ae38b6ce 2010-06-25 92: # same if we get more or less headers
08ae38b6ce 2010-06-25 93: old_keys = set(index[url].keys())
08ae38b6ce 2010-06-25 94: old_keys.discard('__time__')
08ae38b6ce 2010-06-25 95: old_keys.discard('Pragma')
08ae38b6ce 2010-06-25 96: more_keys = set(new_headers.keys()) - old_keys
08ae38b6ce 2010-06-25 97: more_keys.discard('Pragma')
08ae38b6ce 2010-06-25 98: less_keys = old_keys - set(new_headers.keys())
08ae38b6ce 2010-06-25 99: if len(more_keys) > 0:
083ec707ea 2010-07-06 100: if not len(old_keys) == 0:
53dcfdb8f7 2010-07-05 101: print('More headers appear:', more_keys)
08ae38b6ce 2010-06-25 102: reload = True
08ae38b6ce 2010-06-25 103: elif len(less_keys) > 0:
08ae38b6ce 2010-06-25 104: print('Less headers appear:', less_keys)
08ae38b6ce 2010-06-25 105: else:
08ae38b6ce 2010-06-25 106: for key in index[url].keys():
08ae38b6ce 2010-06-25 107: if key not in ('__time__', 'Pragma') and not index[url][key] == new_headers[key]:
7b27f1db02 2010-07-01 108: print('Header "', key, '" changed from [', index[url][key], '] to [', new_headers[key], ']', sep='')
08ae38b6ce 2010-06-25 109: reload = True
08ae38b6ce 2010-06-25 110:
08ae38b6ce 2010-06-25 111: # downloading file
08ae38b6ce 2010-06-25 112: if reload:
08ae38b6ce 2010-06-25 113: if 'Content-Length' in headers:
08ae38b6ce 2010-06-25 114: print('Downloading', headers['Content-Length'], 'bytes [', end='')
08ae38b6ce 2010-06-25 115: else:
08ae38b6ce 2010-06-25 116: print('Downloading [', end='')
08ae38b6ce 2010-06-25 117: sys.stdout.flush()
08ae38b6ce 2010-06-25 118:
08ae38b6ce 2010-06-25 119: # file is created at temporary location and moved in place only when download completes
08ae38b6ce 2010-06-25 120: temp_file = open(options.dir + '/.tmp', 'wb')
7b27f1db02 2010-07-01 121: buffer = source.read(block_size)
7b27f1db02 2010-07-01 122: blocks = 0
7b27f1db02 2010-07-01 123: megs = 0
08ae38b6ce 2010-06-25 124: while len(buffer) > 0:
08ae38b6ce 2010-06-25 125: temp_file.write(buffer)
08ae38b6ce 2010-06-25 126: print('.', end='')
08ae38b6ce 2010-06-25 127: sys.stdout.flush()
7b27f1db02 2010-07-01 128: buffer = source.read(block_size)
7b27f1db02 2010-07-01 129: blocks += 1
7b27f1db02 2010-07-01 130: if blocks > 1024*1024/block_size:
7b27f1db02 2010-07-01 131: blocks = blocks - 1024*1024/block_size
7b27f1db02 2010-07-01 132: megs += 1
7b27f1db02 2010-07-01 133: print('{}Mb'.format(megs), end='')
08ae38b6ce 2010-06-25 134: temp_file.close()
08ae38b6ce 2010-06-25 135: print(']')
08ae38b6ce 2010-06-25 136: os.renames(options.dir + '/.tmp', file_name)
08ae38b6ce 2010-06-25 137:
08ae38b6ce 2010-06-25 138: checked_files += 1
08ae38b6ce 2010-06-25 139:
08ae38b6ce 2010-06-25 140: # storing new time mark and storing new headers
08ae38b6ce 2010-06-25 141: new_headers['__time__'] = datetime.datetime.now()
08ae38b6ce 2010-06-25 142: index[url] = new_headers
7b27f1db02 2010-07-01 143: index.sync()
08ae38b6ce 2010-06-25 144:
08ae38b6ce 2010-06-25 145: except urllib.error.HTTPError as error:
08ae38b6ce 2010-06-25 146: # in case of error we don't need to do anything actually,
08ae38b6ce 2010-06-25 147: # if file download stalls or fails the file would not be moved to it's location
08ae38b6ce 2010-06-25 148: print(error)
08ae38b6ce 2010-06-25 149:
08ae38b6ce 2010-06-25 150: print('[', len(unchecked_files), '/', checked_files, ']')
08ae38b6ce 2010-06-25 151:
08ae38b6ce 2010-06-25 152: # checking if there were any files downloaded, if yes - restarting sequence
08ae38b6ce 2010-06-25 153: if checked_files == 0:
08ae38b6ce 2010-06-25 154: break