7ff9724ae4 2013-08-14 1: #!/usr/bin/env python3.3
d8731957ad 2012-08-07 2:
d8731957ad 2012-08-07 3: import argparse, os
d8731957ad 2012-08-07 4: parser = argparse.ArgumentParser()
d8731957ad 2012-08-07 5: parser.add_argument('-c', '--config', dest = 'config', help = 'config file location', metavar = 'FILE', default = 'samesite.conf')
d8731957ad 2012-08-07 6: args = parser.parse_args()
d8731957ad 2012-08-07 7: assert os.access(args.config, os.R_OK), "Fatal error: can't read {}".format(args.config)
d8731957ad 2012-08-07 8:
d8731957ad 2012-08-07 9: import configparser
d8731957ad 2012-08-07 10: config = configparser.ConfigParser({
d8731957ad 2012-08-07 11: 'port': '8008',
d8731957ad 2012-08-07 12: 'verbose': 'no',
d8731957ad 2012-08-07 13: 'noetag': 'no',
d8731957ad 2012-08-07 14: 'noparts': 'no',
d8731957ad 2012-08-07 15: 'strip': '',
d8731957ad 2012-08-07 16: 'sub': '',
d8731957ad 2012-08-07 17: 'proto': 'http',
d8731957ad 2012-08-07 18: })
d8731957ad 2012-08-07 19: config.read(args.config)
d8731957ad 2012-08-07 20:
d8731957ad 2012-08-07 21: cache_dir = os.path.realpath(os.path.dirname(args.config))
d8731957ad 2012-08-07 22:
d8731957ad 2012-08-07 23: import re
d8731957ad 2012-08-07 24: for section in config.sections():
d8731957ad 2012-08-07 25: if section != 'DEFAULT':
d8731957ad 2012-08-07 26: if 'dir' in config[section]:
d8731957ad 2012-08-07 27: if not re.compile('^/.*').match(config[section]['dir']):
d8731957ad 2012-08-07 28: config[section]['dir'] = cache_dir + os.sep + section
d8731957ad 2012-08-07 29: thisDir = re.compile('^(.*)/$').match(config[section]['dir'])
d8731957ad 2012-08-07 30: if thisDir:
d8731957ad 2012-08-07 31: config[section]['dir'] = thisDir.group(1)
d8731957ad 2012-08-07 32: if not re.compile('^/(.*)$').match(config[section]['dir']):
d8731957ad 2012-08-07 33: config[section]['dir'] = cache_dir + os.sep + config[section]['dir']
d8731957ad 2012-08-07 34: else:
d8731957ad 2012-08-07 35: config[section]['dir'] = cache_dir + os.sep + section
d8731957ad 2012-08-07 36: if not 'root' in config[section]:
d8731957ad 2012-08-07 37: config[section]['root'] = section
cab908195f 2010-09-06 38:
cab908195f 2010-09-06 39: #assert options.port or os.access(options.log, os.R_OK), 'Log file unreadable'
cab908195f 2010-09-06 40:
d8731957ad 2012-08-07 41: const_desc_fields = set(['Content-Length', 'Last-Modified', 'Pragma'])
cab908195f 2010-09-06 42: const_ignore_fields = set([
d8731957ad 2012-08-07 43: 'Accept-Ranges', 'Age',
d8731957ad 2012-08-07 44: 'Cache-Control', 'Connection', 'Content-Type',
d8731957ad 2012-08-07 45: 'Date',
d8731957ad 2012-08-07 46: 'Expires',
d8731957ad 2012-08-07 47: 'Referer',
d8731957ad 2012-08-07 48: 'Server',
d8731957ad 2012-08-07 49: 'Via',
f57e6e032b 2013-11-05 50: 'X-CCC', 'X-CID', 'X-Cache', 'X-Cache-Lookup', 'X-Livetool', 'X-Powered-By',
90160dbf50 2011-03-06 51: ])
90160dbf50 2011-03-06 52:
82969b1fc2 2012-01-25 53: block_size = 8192
82969b1fc2 2012-01-25 54:
d8731957ad 2012-08-07 55: import bsddb3.dbshelve, copy, datetime, http.server, spacemap, urllib.request, urllib.error
d8731957ad 2012-08-07 56:
d8731957ad 2012-08-07 57: class MyRequestHandler(http.server.BaseHTTPRequestHandler):
90160dbf50 2011-03-06 58: def __process(self):
90160dbf50 2011-03-06 59: # reload means file needs to be reloaded to serve request
90160dbf50 2011-03-06 60: reload = False
90160dbf50 2011-03-06 61: # recheck means file needs to be checked, this also means that if file hav been modified we can serve older copy
90160dbf50 2011-03-06 62: recheck = False
90160dbf50 2011-03-06 63: # file_stat means file definitely exists
90160dbf50 2011-03-06 64: file_stat = None
90160dbf50 2011-03-06 65: # requested_ranges holds data about any range requested
90160dbf50 2011-03-06 66: requested_ranges = None
90160dbf50 2011-03-06 67: # records holds data from index locally, should be written back upon successfull completion
90160dbf50 2011-03-06 68: record = None
90160dbf50 2011-03-06 69:
90160dbf50 2011-03-06 70: myPath = re.compile('^(.*?)(\?.*)$').match(self.path)
90160dbf50 2011-03-06 71: if myPath:
90160dbf50 2011-03-06 72: my_path = myPath.group(1)
90160dbf50 2011-03-06 73: else:
90160dbf50 2011-03-06 74: my_path = self.path
90160dbf50 2011-03-06 75:
996aa0149d 2013-03-13 76: if not config.has_section(self.headers['Host']):
996aa0149d 2013-03-13 77: config.add_section(self.headers['Host'])
996aa0149d 2013-03-13 78: config[self.headers['Host']]['root'] = self.headers['Host']
996aa0149d 2013-03-13 79: config[self.headers['Host']]['dir'] = cache_dir + os.sep + self.headers['Host']
d8731957ad 2012-08-07 80: config_host = config[self.headers['Host']]
90160dbf50 2011-03-06 81:
d8731957ad 2012-08-07 82: if config_host['sub'] != None and config_host['strip'] != None and len(config_host['strip']) > 0:
d8731957ad 2012-08-07 83: string = re.compile(config_host['strip']).sub(config_host['sub'], my_path)
90160dbf50 2011-03-06 84: my_path = string
90160dbf50 2011-03-06 85:
d8731957ad 2012-08-07 86: my_path_b = my_path.encode('utf-8')
90160dbf50 2011-03-06 87: info = 'Checking file: ' + my_path
90160dbf50 2011-03-06 88:
d8731957ad 2012-08-07 89: if not os.access(config_host['dir'], os.X_OK):
d8731957ad 2012-08-07 90: os.mkdir(config_host['dir'])
90160dbf50 2011-03-06 91: # this is file index - everything is stored in this file
90160dbf50 2011-03-06 92: # _parts - list of stored parts of file
90160dbf50 2011-03-06 93: # _time - last time the file was checked
90160dbf50 2011-03-06 94: # everything else is just the headers
d8731957ad 2012-08-07 95: index = bsddb3.dbshelve.open(config_host['dir'] + os.sep + '.index')
90160dbf50 2011-03-06 96:
90160dbf50 2011-03-06 97: desc_fields = const_desc_fields.copy()
90160dbf50 2011-03-06 98: ignore_fields = const_ignore_fields.copy()
d8731957ad 2012-08-07 99: if config_host['noetag'] == 'no':
b67d2538b2 2012-08-09 100: desc_fields.add('ETag')
90160dbf50 2011-03-06 101: else:
b67d2538b2 2012-08-09 102: ignore_fields.add('ETag')
90160dbf50 2011-03-06 103:
90160dbf50 2011-03-06 104: proxy_ignored = set([
d8731957ad 2012-08-07 105: 'Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language',
d8731957ad 2012-08-07 106: 'Cache-Control', 'Connection', 'Content-Length', 'Cookie',
4fece04acc 2014-01-13 107: 'DNT',
d8731957ad 2012-08-07 108: 'Host',
7ff9724ae4 2013-08-14 109: 'If-Modified-Since', 'If-None-Match', 'If-Unmodified-Since',
d8731957ad 2012-08-07 110: 'Referer',
a2857db2b5 2013-01-29 111: 'UA-CPU', 'User-Agent',
d8731957ad 2012-08-07 112: 'Via',
44aa59eb58 2012-08-09 113: 'X-Forwarded-For', 'X-Last-HR', 'X-Last-HTTP-Status-Code', 'X-Old-UID', 'X-Removed', 'X-Real-IP', 'X-Retry-Count',
90160dbf50 2011-03-06 114: ])
90160dbf50 2011-03-06 115:
90160dbf50 2011-03-06 116: print('===============[ {} request ]==='.format(self.command))
90160dbf50 2011-03-06 117:
90160dbf50 2011-03-06 118: for header in self.headers:
90160dbf50 2011-03-06 119: if header in proxy_ignored:
90160dbf50 2011-03-06 120: pass
d8731957ad 2012-08-07 121: elif header in ('Range'):
90160dbf50 2011-03-06 122: isRange = re.compile('bytes=(\d+)-(\d+)').match(self.headers[header])
90160dbf50 2011-03-06 123: if isRange:
90160dbf50 2011-03-06 124: requested_ranges = spacemap.SpaceMap({int(isRange.group(1)): int(isRange.group(2)) + 1})
90160dbf50 2011-03-06 125: else:
90160dbf50 2011-03-06 126: return()
d8731957ad 2012-08-07 127: elif header in ('Pragma'):
d8731957ad 2012-08-07 128: if my_path_b in index:
d8731957ad 2012-08-07 129: index[my_path_b][header] = self.headers[header]
90160dbf50 2011-03-06 130: else:
90160dbf50 2011-03-06 131: print('Unknown header - ', header, ': ', self.headers[header], sep='')
90160dbf50 2011-03-06 132: return()
90160dbf50 2011-03-06 133: print(header, self.headers[header])
90160dbf50 2011-03-06 134:
90160dbf50 2011-03-06 135: # creating file name from my_path
d8731957ad 2012-08-07 136: file_name = config_host['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
90160dbf50 2011-03-06 137: # partial file or unfinished download
d8731957ad 2012-08-07 138: temp_name = config_host['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)
90160dbf50 2011-03-06 139:
90160dbf50 2011-03-06 140: # creating empty placeholder in index
90160dbf50 2011-03-06 141: # if there's no space map and there's no file in real directory - we have no file
90160dbf50 2011-03-06 142: # if there's an empty space map - file is full
90160dbf50 2011-03-06 143: # space map generally covers every bit of file we don't posess currently
d8731957ad 2012-08-07 144: if not my_path_b in index:
90160dbf50 2011-03-06 145: info += '\nThis one is new.'
90160dbf50 2011-03-06 146: reload = True
90160dbf50 2011-03-06 147: record = {}
90160dbf50 2011-03-06 148: else:
90160dbf50 2011-03-06 149: # forcibly checking file if no file present
d8731957ad 2012-08-07 150: record = index[my_path_b]
90160dbf50 2011-03-06 151: if os.access(file_name, os.R_OK):
90160dbf50 2011-03-06 152: info += '\nFull file found.'
90160dbf50 2011-03-06 153: file_stat = os.stat(file_name)
d8731957ad 2012-08-07 154: elif '_parts' in index[my_path_b] and os.access(temp_name, os.R_OK):
90160dbf50 2011-03-06 155: info += '\nPartial file found.'
90160dbf50 2011-03-06 156: file_stat = os.stat(temp_name)
d1fa9d0737 2012-01-16 157: recheck = True
90160dbf50 2011-03-06 158: else:
90160dbf50 2011-03-06 159: info += '\nFile not found or inaccessible.'
90160dbf50 2011-03-06 160: record['_parts'] = None
90160dbf50 2011-03-06 161: reload = True
90160dbf50 2011-03-06 162:
90160dbf50 2011-03-06 163: if not '_parts' in record:
90160dbf50 2011-03-06 164: record['_parts'] = None
90160dbf50 2011-03-06 165:
90160dbf50 2011-03-06 166: if record['_parts'] == None:
90160dbf50 2011-03-06 167: recheck = True
90160dbf50 2011-03-06 168:
90160dbf50 2011-03-06 169: # forcibly checking file if file size doesn't match with index data
90160dbf50 2011-03-06 170: if not reload:
90160dbf50 2011-03-06 171: if '_parts' in record and record['_parts'] == spacemap.SpaceMap():
601ec56da6 2011-12-19 172: if 'content-length' in record and file_stat and file_stat.st_size != int(record['content-length']):
601ec56da6 2011-12-19 173: info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['content-length'])
90160dbf50 2011-03-06 174: record['_parts'] = None
90160dbf50 2011-03-06 175: reload = True
90160dbf50 2011-03-06 176:
90160dbf50 2011-03-06 177: # forcibly checking file if index holds Pragma header
601ec56da6 2011-12-19 178: if not reload and 'pragma' in record and record['pragma'] == 'no-cache':
90160dbf50 2011-03-06 179: info +='\nPragma on: recheck imminent.'
90160dbf50 2011-03-06 180: recheck = True
90160dbf50 2011-03-06 181:
90160dbf50 2011-03-06 182: # skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
8425e2e393 2011-12-14 183: if not recheck and not reload and '_time' in record and (record['_time'] - datetime.datetime.now() + datetime.timedelta(hours = 4)).days < 0:
8425e2e393 2011-12-14 184: info += '\nFile is old - rechecking.'
90160dbf50 2011-03-06 185: recheck = True
90160dbf50 2011-03-06 186:
90160dbf50 2011-03-06 187: print(info)
90160dbf50 2011-03-06 188: if reload or recheck:
90160dbf50 2011-03-06 189:
90160dbf50 2011-03-06 190: try:
d8731957ad 2012-08-07 191: request = config_host['proto'] + '://' + config_host['root'] + self.path
90160dbf50 2011-03-06 192: my_headers = {}
44aa59eb58 2012-08-09 193: for header in ('Accept', 'Cache-Control', 'Cookie', 'Referer', 'User-Agent'):
90160dbf50 2011-03-06 194: if header in self.headers:
90160dbf50 2011-03-06 195: my_headers[header] = self.headers[header]
90160dbf50 2011-03-06 196:
90160dbf50 2011-03-06 197: needed = None
b5c328f916 2012-01-04 198: if self.command not in ('HEAD'):
b5c328f916 2012-01-04 199: if '_parts' in record and record['_parts'] != None:
d8731957ad 2012-08-07 200: if config_host['noparts'] != 'no' or requested_ranges == None or requested_ranges == spacemap.SpaceMap():
b5c328f916 2012-01-04 201: needed = record['_parts']
b5c328f916 2012-01-04 202: else:
b5c328f916 2012-01-04 203: needed = record['_parts'] & requested_ranges
d8731957ad 2012-08-07 204: elif config_host['noparts'] =='no' and requested_ranges != None and requested_ranges != spacemap.SpaceMap():
b5c328f916 2012-01-04 205: needed = requested_ranges
b5c328f916 2012-01-04 206: ranges = ()
b5c328f916 2012-01-04 207: print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
b5c328f916 2012-01-04 208: if needed != None and len(needed) > 0:
b5c328f916 2012-01-04 209: needed.rewind()
b5c328f916 2012-01-04 210: while True:
b5c328f916 2012-01-04 211: range = needed.pop()
b5c328f916 2012-01-04 212: if range[0] == None:
b5c328f916 2012-01-04 213: break
b5c328f916 2012-01-04 214: ranges += '{}-{}'.format(range[0], range[1] - 1),
d8731957ad 2012-08-07 215: my_headers['Range'] = 'bytes=' + ','.join(ranges)
b5c328f916 2012-01-04 216:
a2857db2b5 2013-01-29 217: #my_headers['Accept-Encoding'] = 'gzip, compress, deflate, identity; q=0'
d8731957ad 2012-08-07 218: request = urllib.request.Request(request, headers = my_headers)
b5c328f916 2012-01-04 219:
d8731957ad 2012-08-07 220: source = urllib.request.urlopen(request, timeout = 60)
601ec56da6 2011-12-19 221: new_record = {}
601ec56da6 2011-12-19 222: new_record['_parts'] = record['_parts']
601ec56da6 2011-12-19 223: headers = source.info()
62e6d8a7ab 2012-01-16 224:
d8731957ad 2012-08-07 225: if 'Content-Encoding' in headers and headers['Content-Encoding'] == 'gzip':
a81f1a70fb 2012-01-16 226: import gzip
a81f1a70fb 2012-01-16 227: source = gzip.GzipFile(fileobj=source)
601ec56da6 2011-12-19 228:
601ec56da6 2011-12-19 229: # stripping unneeded headers (XXX make this inplace?)
601ec56da6 2011-12-19 230: for header in headers:
601ec56da6 2011-12-19 231: if header in desc_fields:
601ec56da6 2011-12-19 232: #if header == 'Pragma' and headers[header] != 'no-cache':
d8731957ad 2012-08-07 233: if header == 'Content-Length':
d8731957ad 2012-08-07 234: if 'Content-Range' not in headers:
601ec56da6 2011-12-19 235: new_record[header] = int(headers[header])
601ec56da6 2011-12-19 236: else:
601ec56da6 2011-12-19 237: new_record[header] = headers[header]
d8731957ad 2012-08-07 238: elif header == 'Content-Range':
601ec56da6 2011-12-19 239: range = re.compile('^bytes (\d+)-(\d+)/(\d+)$').match(headers[header])
601ec56da6 2011-12-19 240: if range:
d8731957ad 2012-08-07 241: new_record['Content-Length'] = int(range.group(3))
601ec56da6 2011-12-19 242: else:
601ec56da6 2011-12-19 243: assert False, 'Content-Range unrecognized.'
601ec56da6 2011-12-19 244: elif not header in ignore_fields:
601ec56da6 2011-12-19 245: print('Undefined header "', header, '": ', headers[header], sep='')
601ec56da6 2011-12-19 246:
601ec56da6 2011-12-19 247: # comparing headers with data found in index
601ec56da6 2011-12-19 248: # if any header has changed (except Pragma) file is fully downloaded
601ec56da6 2011-12-19 249: # same if we get more or less headers
601ec56da6 2011-12-19 250: old_keys = set(record.keys())
601ec56da6 2011-12-19 251: old_keys.discard('_time')
d8731957ad 2012-08-07 252: old_keys.discard('Pragma')
601ec56da6 2011-12-19 253: more_keys = set(new_record.keys()) - old_keys
d8731957ad 2012-08-07 254: more_keys.discard('Pragma')
601ec56da6 2011-12-19 255: less_keys = old_keys - set(new_record.keys())
601ec56da6 2011-12-19 256: if len(more_keys) > 0:
601ec56da6 2011-12-19 257: if len(old_keys) != 0:
601ec56da6 2011-12-19 258: print('More headers appear:', more_keys)
601ec56da6 2011-12-19 259: reload = True
601ec56da6 2011-12-19 260: elif len(less_keys) > 0:
601ec56da6 2011-12-19 261: print('Less headers appear:', less_keys)
601ec56da6 2011-12-19 262: else:
601ec56da6 2011-12-19 263: for key in record.keys():
d8731957ad 2012-08-07 264: if key[0] != '_' and key != 'Pragma' and record[key] != new_record[key]:
601ec56da6 2011-12-19 265: print('Header "', key, '" changed from [', record[key], '] to [', new_record[key], ']', sep='')
601ec56da6 2011-12-19 266: print(type(record[key]), type(new_record[key]))
601ec56da6 2011-12-19 267: reload = True
601ec56da6 2011-12-19 268:
601ec56da6 2011-12-19 269: if reload:
601ec56da6 2011-12-19 270: print('Reloading.')
601ec56da6 2011-12-19 271: if os.access(temp_name, os.R_OK):
601ec56da6 2011-12-19 272: os.unlink(temp_name)
601ec56da6 2011-12-19 273: if os.access(file_name, os.R_OK):
601ec56da6 2011-12-19 274: os.unlink(file_name)
d8731957ad 2012-08-07 275: if 'Content-Length' in new_record:
d8731957ad 2012-08-07 276: new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
601ec56da6 2011-12-19 277: if not new_record['_parts']:
601ec56da6 2011-12-19 278: new_record['_parts'] = spacemap.SpaceMap()
601ec56da6 2011-12-19 279: print(new_record)
601ec56da6 2011-12-19 280:
601ec56da6 2011-12-19 281: # downloading file or segment
d8731957ad 2012-08-07 282: if 'Content-Length' in new_record:
601ec56da6 2011-12-19 283: if needed == None:
601ec56da6 2011-12-19 284: needed = new_record['_parts']
601ec56da6 2011-12-19 285: else:
601ec56da6 2011-12-19 286: if len(needed) > 1:
601ec56da6 2011-12-19 287: print("Multipart requests currently not supported.")
601ec56da6 2011-12-19 288: assert False, 'Skip this one for now.'
601ec56da6 2011-12-19 289: #else:
601ec56da6 2011-12-19 290: #assert False, 'No content-length or Content-Range header.'
601ec56da6 2011-12-19 291:
601ec56da6 2011-12-19 292: new_record['_time'] = datetime.datetime.now()
601ec56da6 2011-12-19 293: if self.command not in ('HEAD'):
601ec56da6 2011-12-19 294: # file is created at temporary location and moved in place only when download completes
601ec56da6 2011-12-19 295: if not os.access(temp_name, os.R_OK):
d8731957ad 2012-08-07 296: empty_name = config_host['dir'] + os.sep + '.tmp'
601ec56da6 2011-12-19 297: with open(empty_name, 'w+b') as some_file:
601ec56da6 2011-12-19 298: pass
601ec56da6 2011-12-19 299: os.renames(empty_name, temp_name)
601ec56da6 2011-12-19 300: temp_file = open(temp_name, 'r+b')
601ec56da6 2011-12-19 301: if requested_ranges == None and needed == None:
601ec56da6 2011-12-19 302: needed = new_record['_parts']
601ec56da6 2011-12-19 303: needed.rewind()
f57e6e032b 2013-11-05 304: countdown = 16
601ec56da6 2011-12-19 305: while True:
62e6d8a7ab 2012-01-16 306: # XXX can make this implicit - one request per range
601ec56da6 2011-12-19 307: (start, end) = needed.pop()
601ec56da6 2011-12-19 308: if start == None:
601ec56da6 2011-12-19 309: break
601ec56da6 2011-12-19 310: stream_last = start
601ec56da6 2011-12-19 311: old_record = copy.copy(new_record)
601ec56da6 2011-12-19 312: if end - start < block_size:
601ec56da6 2011-12-19 313: req_block_size = end - start
601ec56da6 2011-12-19 314: else:
601ec56da6 2011-12-19 315: req_block_size = block_size
601ec56da6 2011-12-19 316: buffer = source.read(req_block_size)
601ec56da6 2011-12-19 317: length = len(buffer)
601ec56da6 2011-12-19 318: while length > 0 and stream_last < end:
601ec56da6 2011-12-19 319: stream_pos = stream_last + length
601ec56da6 2011-12-19 320: assert stream_pos <= end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
601ec56da6 2011-12-19 321: temp_file.seek(stream_last)
601ec56da6 2011-12-19 322: temp_file.write(buffer)
601ec56da6 2011-12-19 323: x = new_record['_parts'] - spacemap.SpaceMap({stream_last: stream_pos})
601ec56da6 2011-12-19 324: new_record['_parts'] = new_record['_parts'] - spacemap.SpaceMap({stream_last: stream_pos})
f57e6e032b 2013-11-05 325: countdown -= 1
f57e6e032b 2013-11-05 326: if countdown == 0:
f57e6e032b 2013-11-05 327: index[my_path_b] = old_record
f57e6e032b 2013-11-05 328: index.sync()
f57e6e032b 2013-11-05 329: countdown = 16
601ec56da6 2011-12-19 330: old_record = copy.copy(new_record)
601ec56da6 2011-12-19 331: stream_last = stream_pos
601ec56da6 2011-12-19 332: if end - stream_last < block_size:
601ec56da6 2011-12-19 333: req_block_size = end - stream_last
601ec56da6 2011-12-19 334: buffer = source.read(req_block_size)
601ec56da6 2011-12-19 335: length = len(buffer)
601ec56da6 2011-12-19 336: # moving downloaded data to real file
601ec56da6 2011-12-19 337: temp_file.close()
601ec56da6 2011-12-19 338:
d8731957ad 2012-08-07 339: index[my_path_b] = new_record
601ec56da6 2011-12-19 340: index.sync()
90160dbf50 2011-03-06 341:
d8731957ad 2012-08-07 342: except urllib.error.HTTPError as error:
90160dbf50 2011-03-06 343: # in case of error we don't need to do anything actually,
90160dbf50 2011-03-06 344: # if file download stalls or fails the file would not be moved to it's location
6cf3431e69 2013-08-23 345: self.send_response(error.code)
6cf3431e69 2013-08-23 346: self.end_headers()
44aa59eb58 2012-08-09 347: print(error, repr(my_headers))
6cf3431e69 2013-08-23 348: return
90160dbf50 2011-03-06 349:
f57e6e032b 2013-11-05 350: #print(index[my_path_b])
90160dbf50 2011-03-06 351:
d8731957ad 2012-08-07 352: if not os.access(file_name, os.R_OK) and os.access(temp_name, os.R_OK) and '_parts' in index[my_path_b] and index[my_path_b]['_parts'] == spacemap.SpaceMap():
90160dbf50 2011-03-06 353: # just moving
90160dbf50 2011-03-06 354: # drop old dirs XXX
90160dbf50 2011-03-06 355: print('Moving temporary file to new destination.')
90160dbf50 2011-03-06 356: os.renames(temp_name, file_name)
90160dbf50 2011-03-06 357:
d8731957ad 2012-08-07 358: if not my_path_b in index:
90160dbf50 2011-03-06 359: self.send_response(502)
90160dbf50 2011-03-06 360: self.end_headers()
90160dbf50 2011-03-06 361: return
90160dbf50 2011-03-06 362:
90160dbf50 2011-03-06 363: if self.command == 'HEAD':
90160dbf50 2011-03-06 364: self.send_response(200)
d8731957ad 2012-08-07 365: if 'Content-Length' in index[my_path_b]:
d8731957ad 2012-08-07 366: self.send_header('Content-Length', index[my_path_b]['Content-Length'])
d8731957ad 2012-08-07 367: self.send_header('Accept-Ranges', 'bytes')
d8731957ad 2012-08-07 368: self.send_header('Content-Type', 'application/octet-stream')
d8731957ad 2012-08-07 369: if 'Last-Modified' in index[my_path_b]:
d8731957ad 2012-08-07 370: self.send_header('Last-Modified', index[my_path_b]['Last-Modified'])
90160dbf50 2011-03-06 371: self.end_headers()
90160dbf50 2011-03-06 372: else:
d8731957ad 2012-08-07 373: if ('_parts' in index[my_path_b] and index[my_path_b]['_parts'] != spacemap.SpaceMap()) or not os.access(file_name, os.R_OK):
90160dbf50 2011-03-06 374: file_name = temp_name
90160dbf50 2011-03-06 375:
90160dbf50 2011-03-06 376: with open(file_name, 'rb') as real_file:
90160dbf50 2011-03-06 377: file_stat = os.stat(file_name)
d8731957ad 2012-08-07 378: if 'Range' in self.headers:
90160dbf50 2011-03-06 379: self.send_response(206)
90160dbf50 2011-03-06 380: ranges = ()
90160dbf50 2011-03-06 381: requested_ranges.rewind()
90160dbf50 2011-03-06 382: while True:
90160dbf50 2011-03-06 383: pair = requested_ranges.pop()
90160dbf50 2011-03-06 384: if pair[0] == None:
90160dbf50 2011-03-06 385: break
90160dbf50 2011-03-06 386: ranges += '{}-{}'.format(pair[0], str(pair[1] - 1)),
d8731957ad 2012-08-07 387: self.send_header('Content-Range', 'bytes {}/{}'.format(','.join(ranges), index[my_path_b]['Content-Length']))
90160dbf50 2011-03-06 388: else:
90160dbf50 2011-03-06 389: self.send_response(200)
d8731957ad 2012-08-07 390: self.send_header('Content-Length', str(file_stat.st_size))
90160dbf50 2011-03-06 391: requested_ranges = spacemap.SpaceMap({0: file_stat.st_size})
d8731957ad 2012-08-07 392: if 'Last-Modified' in index[my_path_b]:
d8731957ad 2012-08-07 393: self.send_header('Last-Modified', index[my_path_b]['Last-Modified'])
d8731957ad 2012-08-07 394: self.send_header('Content-Type', 'application/octet-stream')
90160dbf50 2011-03-06 395: self.end_headers()
90160dbf50 2011-03-06 396: if self.command in ('GET'):
90160dbf50 2011-03-06 397: if len(requested_ranges) > 0:
90160dbf50 2011-03-06 398: requested_ranges.rewind()
90160dbf50 2011-03-06 399: (start, end) = requested_ranges.pop()
90160dbf50 2011-03-06 400: else:
90160dbf50 2011-03-06 401: start = 0
9a8a46bcf0 2011-09-06 402: # XXX ugly hack
d8731957ad 2012-08-07 403: if 'Content-Length' in index[my_path_b]:
d8731957ad 2012-08-07 404: end = index[my_path_b]['Content-Length']
9a8a46bcf0 2011-09-06 405: else:
9a8a46bcf0 2011-09-06 406: end = 0
90160dbf50 2011-03-06 407: real_file.seek(start)
90160dbf50 2011-03-06 408: if block_size > end - start:
90160dbf50 2011-03-06 409: req_block_size = end - start
90160dbf50 2011-03-06 410: else:
90160dbf50 2011-03-06 411: req_block_size = block_size
90160dbf50 2011-03-06 412: buffer = real_file.read(req_block_size)
90160dbf50 2011-03-06 413: length = len(buffer)
90160dbf50 2011-03-06 414: while length > 0:
90160dbf50 2011-03-06 415: self.wfile.write(buffer)
90160dbf50 2011-03-06 416: start += len(buffer)
90160dbf50 2011-03-06 417: if req_block_size > end - start:
90160dbf50 2011-03-06 418: req_block_size = end - start
90160dbf50 2011-03-06 419: if req_block_size == 0:
90160dbf50 2011-03-06 420: break
90160dbf50 2011-03-06 421: buffer = real_file.read(req_block_size)
90160dbf50 2011-03-06 422: length = len(buffer)
90160dbf50 2011-03-06 423:
90160dbf50 2011-03-06 424: def do_HEAD(self):
90160dbf50 2011-03-06 425: return self.__process()
90160dbf50 2011-03-06 426: def do_GET(self):
90160dbf50 2011-03-06 427: return self.__process()
90160dbf50 2011-03-06 428:
d8731957ad 2012-08-07 429: server = http.server.HTTPServer(('127.0.0.1', int(config['DEFAULT']['port'])), MyRequestHandler)
90160dbf50 2011-03-06 430: server.serve_forever()
82969b1fc2 2012-01-25 431:
82969b1fc2 2012-01-25 432: #gevent.joinall()