9a8a46bcf0 2011-09-06 c.kworr@b84a3: #!/usr/bin/env python3.2
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: import datetime, http.cookiejar, os, sys, shelve, spacemap, re, urllib.request
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: class Config:
e7b837a681 2010-08-25 c.kworr@b84a3: __slots__ = frozenset(['_config', '_default', '_section', 'options', 'root'])
e7b837a681 2010-08-25 c.kworr@b84a3: _default = {
e7b837a681 2010-08-25 c.kworr@b84a3: 'general': {
e7b837a681 2010-08-25 c.kworr@b84a3: 'port': '8008',
e7b837a681 2010-08-25 c.kworr@b84a3: },
e7b837a681 2010-08-25 c.kworr@b84a3: '_other': {
e7b837a681 2010-08-25 c.kworr@b84a3: 'verbose': 'no',
e7b837a681 2010-08-25 c.kworr@b84a3: 'noetag': 'no',
e7b837a681 2010-08-25 c.kworr@b84a3: 'noparts': 'no',
cab908195f 2010-09-06 c.kworr@b84a3: 'strip': '',
cab908195f 2010-09-06 c.kworr@b84a3: 'sub': '',
e7b837a681 2010-08-25 c.kworr@b84a3: },}
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: # function to read in config file
e7b837a681 2010-08-25 c.kworr@b84a3: def __init__(self):
e7b837a681 2010-08-25 c.kworr@b84a3: import configparser, optparse
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: parser = optparse.OptionParser()
e7b837a681 2010-08-25 c.kworr@b84a3: parser.add_option('-c', '--config', dest = 'config', help = 'config file location', metavar = 'FILE', default = 'samesite.conf')
e7b837a681 2010-08-25 c.kworr@b84a3: (self.options, args) = parser.parse_args()
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: assert os.access(self.options.config, os.R_OK), "Fatal error: can't read {}".format(self.options.config)
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: configDir = re.compile('^(.*)/[^/]+$').match(self.options.config)
e7b837a681 2010-08-25 c.kworr@b84a3: if configDir:
e7b837a681 2010-08-25 c.kworr@b84a3: self.root = configDir.group(1)
e7b837a681 2010-08-25 c.kworr@b84a3: else:
e7b837a681 2010-08-25 c.kworr@b84a3: self.root = os.getcwd()
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: self._config = configparser.ConfigParser()
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.readfp(open(self.options.config))
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: for section in self._config.sections():
e7b837a681 2010-08-25 c.kworr@b84a3: if section != 'general':
e7b837a681 2010-08-25 c.kworr@b84a3: if self._config.has_option(section, 'dir'):
e7b837a681 2010-08-25 c.kworr@b84a3: if re.compile('^/$').match(self._config.get(section, 'dir')):
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(section, 'dir', self.root + os.sep + section)
e7b837a681 2010-08-25 c.kworr@b84a3: thisDir = re.compile('^(.*)/$').match(self._config.get(section, 'dir'))
e7b837a681 2010-08-25 c.kworr@b84a3: if thisDir:
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(section, 'dir', thisDir.group(1))
e7b837a681 2010-08-25 c.kworr@b84a3: if not re.compile('^/(.*)$').match(self._config.get(section, 'dir')):
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(section, 'dir', self.root + os.sep + self._config.get(section, 'dir'))
e7b837a681 2010-08-25 c.kworr@b84a3: else:
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(section, 'dir', self.root + os.sep + section)
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: if not self._config.has_option(section, 'root'):
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(section, 'root', section)
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: # function to select config file section or create one
e7b837a681 2010-08-25 c.kworr@b84a3: def section(self, section):
e7b837a681 2010-08-25 c.kworr@b84a3: if not self._config.has_section(section):
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.add_section(section)
e7b837a681 2010-08-25 c.kworr@b84a3: self._section = section
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: # function to get config parameter, if parameter doesn't exists the default
e7b837a681 2010-08-25 c.kworr@b84a3: # value or None is substituted
e7b837a681 2010-08-25 c.kworr@b84a3: def __getitem__(self, name):
e7b837a681 2010-08-25 c.kworr@b84a3: if not self._config.has_option(self._section, name):
e7b837a681 2010-08-25 c.kworr@b84a3: if self._section in self._default:
e7b837a681 2010-08-25 c.kworr@b84a3: if name in self._default[self._section]:
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(self._section, name, self._default[self._section][name])
e7b837a681 2010-08-25 c.kworr@b84a3: else:
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(self._section, name, None)
e7b837a681 2010-08-25 c.kworr@b84a3: elif name in self._default['_other']:
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(self._section, name, self._default['_other'][name])
e7b837a681 2010-08-25 c.kworr@b84a3: else:
e7b837a681 2010-08-25 c.kworr@b84a3: self._config.set(self._section, name, None)
e7b837a681 2010-08-25 c.kworr@b84a3: return(self._config.get(self._section, name))
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: config = Config()
e7b837a681 2010-08-25 c.kworr@b84a3:
e7b837a681 2010-08-25 c.kworr@b84a3: #assert options.port or os.access(options.log, os.R_OK), 'Log file unreadable'
e7b837a681 2010-08-25 c.kworr@b84a3:
cab908195f 2010-09-06 c.kworr@b84a3: const_desc_fields = set(['Content-Length', 'Last-Modified', 'Pragma'])
cab908195f 2010-09-06 c.kworr@b84a3: const_ignore_fields = set([
cab908195f 2010-09-06 c.kworr@b84a3: 'Accept-Ranges', 'Age',
cab908195f 2010-09-06 c.kworr@b84a3: 'Cache-Control', 'Connection', 'Content-Type',
cab908195f 2010-09-06 c.kworr@b84a3: 'Date',
cab908195f 2010-09-06 c.kworr@b84a3: 'Expires',
439e1753a4 2010-09-07 c.kworr@b84a3: 'Referer',
439e1753a4 2010-09-07 c.kworr@b84a3: 'Server',
439e1753a4 2010-09-07 c.kworr@b84a3: 'Via',
9a8a46bcf0 2011-09-06 c.kworr@b84a3: 'X-Cache', 'X-Cache-Lookup', 'X-Powered-By',
90160dbf50 2011-03-06 c.kworr@b84a3: ])
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: block_size = 4096
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: import http.server
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: class MyRequestHandler(http.server.BaseHTTPRequestHandler):
90160dbf50 2011-03-06 c.kworr@b84a3: def __process(self):
90160dbf50 2011-03-06 c.kworr@b84a3: # reload means file needs to be reloaded to serve request
90160dbf50 2011-03-06 c.kworr@b84a3: reload = False
90160dbf50 2011-03-06 c.kworr@b84a3: # recheck means file needs to be checked, this also means that if file hav been modified we can serve older copy
90160dbf50 2011-03-06 c.kworr@b84a3: recheck = False
90160dbf50 2011-03-06 c.kworr@b84a3: # file_stat means file definitely exists
90160dbf50 2011-03-06 c.kworr@b84a3: file_stat = None
90160dbf50 2011-03-06 c.kworr@b84a3: # requested_ranges holds data about any range requested
90160dbf50 2011-03-06 c.kworr@b84a3: requested_ranges = None
90160dbf50 2011-03-06 c.kworr@b84a3: # records holds data from index locally, should be written back upon successfull completion
90160dbf50 2011-03-06 c.kworr@b84a3: record = None
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: myPath = re.compile('^(.*?)(\?.*)$').match(self.path)
90160dbf50 2011-03-06 c.kworr@b84a3: if myPath:
90160dbf50 2011-03-06 c.kworr@b84a3: my_path = myPath.group(1)
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: my_path = self.path
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: config.section(self.headers['Host'])
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: if config['sub'] != None and config['strip'] != None and len(config['strip']) > 0:
90160dbf50 2011-03-06 c.kworr@b84a3: string = re.compile(config['strip']).sub(config['sub'], my_path)
90160dbf50 2011-03-06 c.kworr@b84a3: my_path = string
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: info = 'Checking file: ' + my_path
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: if not os.access(config['dir'], os.X_OK):
90160dbf50 2011-03-06 c.kworr@b84a3: os.mkdir(config['dir'])
90160dbf50 2011-03-06 c.kworr@b84a3: # this is file index - everything is stored in this file
90160dbf50 2011-03-06 c.kworr@b84a3: # _parts - list of stored parts of file
90160dbf50 2011-03-06 c.kworr@b84a3: # _time - last time the file was checked
90160dbf50 2011-03-06 c.kworr@b84a3: # everything else is just the headers
90160dbf50 2011-03-06 c.kworr@b84a3: index = shelve.open(config['dir'] + os.sep + '.index')
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: desc_fields = const_desc_fields.copy()
90160dbf50 2011-03-06 c.kworr@b84a3: ignore_fields = const_ignore_fields.copy()
90160dbf50 2011-03-06 c.kworr@b84a3: if config['noetag'] == 'no':
90160dbf50 2011-03-06 c.kworr@b84a3: desc_fields.add('ETag')
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: ignore_fields.add('ETag')
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: proxy_ignored = set([
90160dbf50 2011-03-06 c.kworr@b84a3: 'Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language',
90160dbf50 2011-03-06 c.kworr@b84a3: 'Cache-Control', 'Connection', 'Content-Length', 'Cookie',
90160dbf50 2011-03-06 c.kworr@b84a3: 'Host',
90160dbf50 2011-03-06 c.kworr@b84a3: 'If-Modified-Since', 'If-Unmodified-Since',
90160dbf50 2011-03-06 c.kworr@b84a3: 'Referer',
90160dbf50 2011-03-06 c.kworr@b84a3: 'User-Agent',
90160dbf50 2011-03-06 c.kworr@b84a3: 'Via',
9a8a46bcf0 2011-09-06 c.kworr@b84a3: 'X-Forwarded-For', 'X-Last-HR', 'X-Last-HTTP-Status-Code', 'X-REMOVED', 'X-Real-IP', 'X-Retry-Count',
90160dbf50 2011-03-06 c.kworr@b84a3: ])
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: print('===============[ {} request ]==='.format(self.command))
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: for header in self.headers:
90160dbf50 2011-03-06 c.kworr@b84a3: if header in proxy_ignored:
90160dbf50 2011-03-06 c.kworr@b84a3: pass
90160dbf50 2011-03-06 c.kworr@b84a3: elif header in ('Range'):
90160dbf50 2011-03-06 c.kworr@b84a3: isRange = re.compile('bytes=(\d+)-(\d+)').match(self.headers[header])
90160dbf50 2011-03-06 c.kworr@b84a3: if isRange:
90160dbf50 2011-03-06 c.kworr@b84a3: requested_ranges = spacemap.SpaceMap({int(isRange.group(1)): int(isRange.group(2)) + 1})
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: return()
90160dbf50 2011-03-06 c.kworr@b84a3: elif header in ('Pragma'):
90160dbf50 2011-03-06 c.kworr@b84a3: if my_path in index:
90160dbf50 2011-03-06 c.kworr@b84a3: index[my_path][header] = self.headers[header]
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: print('Unknown header - ', header, ': ', self.headers[header], sep='')
90160dbf50 2011-03-06 c.kworr@b84a3: return()
90160dbf50 2011-03-06 c.kworr@b84a3: print(header, self.headers[header])
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: # creating file name from my_path
90160dbf50 2011-03-06 c.kworr@b84a3: file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path)
90160dbf50 2011-03-06 c.kworr@b84a3: # partial file or unfinished download
90160dbf50 2011-03-06 c.kworr@b84a3: temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path)
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: # creating empty placeholder in index
90160dbf50 2011-03-06 c.kworr@b84a3: # if there's no space map and there's no file in real directory - we have no file
90160dbf50 2011-03-06 c.kworr@b84a3: # if there's an empty space map - file is full
90160dbf50 2011-03-06 c.kworr@b84a3: # space map generally covers every bit of file we don't posess currently
90160dbf50 2011-03-06 c.kworr@b84a3: if not my_path in index:
90160dbf50 2011-03-06 c.kworr@b84a3: info += '\nThis one is new.'
90160dbf50 2011-03-06 c.kworr@b84a3: reload = True
90160dbf50 2011-03-06 c.kworr@b84a3: record = {}
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: # forcibly checking file if no file present
90160dbf50 2011-03-06 c.kworr@b84a3: record = index[my_path]
90160dbf50 2011-03-06 c.kworr@b84a3: if os.access(file_name, os.R_OK):
90160dbf50 2011-03-06 c.kworr@b84a3: info += '\nFull file found.'
90160dbf50 2011-03-06 c.kworr@b84a3: file_stat = os.stat(file_name)
90160dbf50 2011-03-06 c.kworr@b84a3: elif '_parts' in index[my_path] and os.access(temp_name, os.R_OK):
90160dbf50 2011-03-06 c.kworr@b84a3: info += '\nPartial file found.'
90160dbf50 2011-03-06 c.kworr@b84a3: file_stat = os.stat(temp_name)
90160dbf50 2011-03-06 c.kworr@b84a3: recheck = True
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: info += '\nFile not found or inaccessible.'
90160dbf50 2011-03-06 c.kworr@b84a3: record['_parts'] = None
90160dbf50 2011-03-06 c.kworr@b84a3: reload = True
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: if not '_parts' in record:
90160dbf50 2011-03-06 c.kworr@b84a3: record['_parts'] = None
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: if record['_parts'] == None:
90160dbf50 2011-03-06 c.kworr@b84a3: recheck = True
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: # forcibly checking file if file size doesn't match with index data
90160dbf50 2011-03-06 c.kworr@b84a3: if not reload:
90160dbf50 2011-03-06 c.kworr@b84a3: if '_parts' in record and record['_parts'] == spacemap.SpaceMap():
90160dbf50 2011-03-06 c.kworr@b84a3: if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']):
90160dbf50 2011-03-06 c.kworr@b84a3: info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length'])
90160dbf50 2011-03-06 c.kworr@b84a3: record['_parts'] = None
90160dbf50 2011-03-06 c.kworr@b84a3: reload = True
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: # forcibly checking file if index holds Pragma header
90160dbf50 2011-03-06 c.kworr@b84a3: if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache':
90160dbf50 2011-03-06 c.kworr@b84a3: info +='\nPragma on: recheck imminent.'
90160dbf50 2011-03-06 c.kworr@b84a3: recheck = True
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: # skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago
90160dbf50 2011-03-06 c.kworr@b84a3: if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0:
90160dbf50 2011-03-06 c.kworr@b84a3: recheck = True
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: print(info)
90160dbf50 2011-03-06 c.kworr@b84a3: if reload or recheck:
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: try:
90160dbf50 2011-03-06 c.kworr@b84a3: request = 'http://' + config['root'] + self.path
90160dbf50 2011-03-06 c.kworr@b84a3: my_headers = {}
90160dbf50 2011-03-06 c.kworr@b84a3: for header in ('Cache-Control', 'Cookie', 'Referer', 'User-Agent'):
90160dbf50 2011-03-06 c.kworr@b84a3: if header in self.headers:
90160dbf50 2011-03-06 c.kworr@b84a3: my_headers[header] = self.headers[header]
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: needed = None
90160dbf50 2011-03-06 c.kworr@b84a3: if '_parts' in record and record['_parts'] != None:
90160dbf50 2011-03-06 c.kworr@b84a3: if config['noparts'] != 'no' or requested_ranges == None or requested_ranges == spacemap.SpaceMap():
90160dbf50 2011-03-06 c.kworr@b84a3: needed = record['_parts']
90160dbf50 2011-03-06 c.kworr@b84a3: else:
9a8a46bcf0 2011-09-06 c.kworr@b84a3: needed = record['_parts'] & requested_ranges
90160dbf50 2011-03-06 c.kworr@b84a3: elif config['noparts'] =='no' and requested_ranges != None and requested_ranges != spacemap.SpaceMap():
90160dbf50 2011-03-06 c.kworr@b84a3: needed = requested_ranges
90160dbf50 2011-03-06 c.kworr@b84a3: ranges = ()
90160dbf50 2011-03-06 c.kworr@b84a3: print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed))
90160dbf50 2011-03-06 c.kworr@b84a3: if needed != None and len(needed) > 0:
90160dbf50 2011-03-06 c.kworr@b84a3: needed.rewind()
90160dbf50 2011-03-06 c.kworr@b84a3: while True:
90160dbf50 2011-03-06 c.kworr@b84a3: range = needed.pop()
90160dbf50 2011-03-06 c.kworr@b84a3: if range[0] == None:
90160dbf50 2011-03-06 c.kworr@b84a3: break
90160dbf50 2011-03-06 c.kworr@b84a3: ranges += '{}-{}'.format(range[0], range[1] - 1),
90160dbf50 2011-03-06 c.kworr@b84a3: my_headers['Range'] = 'bytes=' + ','.join(ranges)
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: request = urllib.request.Request(request, headers = my_headers)
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: with urllib.request.urlopen(request) as source:
90160dbf50 2011-03-06 c.kworr@b84a3: new_record = {}
90160dbf50 2011-03-06 c.kworr@b84a3: new_record['_parts'] = record['_parts']
90160dbf50 2011-03-06 c.kworr@b84a3: headers = source.info()
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: # stripping unneeded headers (XXX make this inplace?)
90160dbf50 2011-03-06 c.kworr@b84a3: for header in headers:
90160dbf50 2011-03-06 c.kworr@b84a3: if header in desc_fields:
90160dbf50 2011-03-06 c.kworr@b84a3: #if header == 'Pragma' and headers[header] != 'no-cache':
90160dbf50 2011-03-06 c.kworr@b84a3: if header == 'Content-Length':
90160dbf50 2011-03-06 c.kworr@b84a3: if 'Content-Range' not in headers:
90160dbf50 2011-03-06 c.kworr@b84a3: new_record[header] = int(headers[header])
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: new_record[header] = headers[header]
90160dbf50 2011-03-06 c.kworr@b84a3: elif header == 'Content-Range':
90160dbf50 2011-03-06 c.kworr@b84a3: range = re.compile('^bytes (\d+)-(\d+)/(\d+)$').match(headers[header])
90160dbf50 2011-03-06 c.kworr@b84a3: if range:
90160dbf50 2011-03-06 c.kworr@b84a3: new_record['Content-Length'] = int(range.group(3))
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: assert False, 'Content-Range unrecognized.'
80f8e3804a 2010-08-20 c.kworr@b84a3: elif not header in ignore_fields:
80f8e3804a 2010-08-20 c.kworr@b84a3: print('Undefined header "', header, '": ', headers[header], sep='')
80f8e3804a 2010-08-20 c.kworr@b84a3:
80f8e3804a 2010-08-20 c.kworr@b84a3: # comparing headers with data found in index
80f8e3804a 2010-08-20 c.kworr@b84a3: # if any header has changed (except Pragma) file is fully downloaded
80f8e3804a 2010-08-20 c.kworr@b84a3: # same if we get more or less headers
90160dbf50 2011-03-06 c.kworr@b84a3: old_keys = set(record.keys())
80f8e3804a 2010-08-20 c.kworr@b84a3: old_keys.discard('_time')
80f8e3804a 2010-08-20 c.kworr@b84a3: old_keys.discard('Pragma')
90160dbf50 2011-03-06 c.kworr@b84a3: more_keys = set(new_record.keys()) - old_keys
80f8e3804a 2010-08-20 c.kworr@b84a3: more_keys.discard('Pragma')
90160dbf50 2011-03-06 c.kworr@b84a3: less_keys = old_keys - set(new_record.keys())
80f8e3804a 2010-08-20 c.kworr@b84a3: if len(more_keys) > 0:
80f8e3804a 2010-08-20 c.kworr@b84a3: if not len(old_keys) == 0:
80f8e3804a 2010-08-20 c.kworr@b84a3: print('More headers appear:', more_keys)
80f8e3804a 2010-08-20 c.kworr@b84a3: reload = True
80f8e3804a 2010-08-20 c.kworr@b84a3: elif len(less_keys) > 0:
80f8e3804a 2010-08-20 c.kworr@b84a3: print('Less headers appear:', less_keys)
80f8e3804a 2010-08-20 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: for key in record.keys():
90160dbf50 2011-03-06 c.kworr@b84a3: if key[0] != '_' and key != 'Pragma' and not record[key] == new_record[key]:
90160dbf50 2011-03-06 c.kworr@b84a3: print('Header "', key, '" changed from [', record[key], '] to [', new_record[key], ']', sep='')
90160dbf50 2011-03-06 c.kworr@b84a3: print(type(record[key]), type(new_record[key]))
80f8e3804a 2010-08-20 c.kworr@b84a3: reload = True
80f8e3804a 2010-08-20 c.kworr@b84a3:
80f8e3804a 2010-08-20 c.kworr@b84a3: if reload:
90160dbf50 2011-03-06 c.kworr@b84a3: print('Reloading.')
90160dbf50 2011-03-06 c.kworr@b84a3: if os.access(temp_name, os.R_OK):
90160dbf50 2011-03-06 c.kworr@b84a3: os.unlink(temp_name)
90160dbf50 2011-03-06 c.kworr@b84a3: if os.access(file_name, os.R_OK):
90160dbf50 2011-03-06 c.kworr@b84a3: os.unlink(file_name)
9a8a46bcf0 2011-09-06 c.kworr@b84a3: if 'Content-Length' in new_record:
9a8a46bcf0 2011-09-06 c.kworr@b84a3: new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])})
9a8a46bcf0 2011-09-06 c.kworr@b84a3: else:
9a8a46bcf0 2011-09-06 c.kworr@b84a3: new_record['_parts'] = spacemap.SpaceMap()
90160dbf50 2011-03-06 c.kworr@b84a3: print(new_record)
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: # downloading file or segment
90160dbf50 2011-03-06 c.kworr@b84a3: if 'Content-Length' in new_record:
90160dbf50 2011-03-06 c.kworr@b84a3: if needed == None:
90160dbf50 2011-03-06 c.kworr@b84a3: needed = new_record['_parts']
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: if len(needed) > 1:
90160dbf50 2011-03-06 c.kworr@b84a3: print("Multipart requests currently not supported.")
90160dbf50 2011-03-06 c.kworr@b84a3: assert False, 'Skip this one for now.'
9a8a46bcf0 2011-09-06 c.kworr@b84a3: #else:
9a8a46bcf0 2011-09-06 c.kworr@b84a3: #assert False, 'No Content-Length or Content-Range header.'
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: new_record['_time'] = datetime.datetime.now()
90160dbf50 2011-03-06 c.kworr@b84a3: if self.command not in ('HEAD'):
90160dbf50 2011-03-06 c.kworr@b84a3: # file is created at temporary location and moved in place only when download completes
90160dbf50 2011-03-06 c.kworr@b84a3: if not os.access(temp_name, os.R_OK):
90160dbf50 2011-03-06 c.kworr@b84a3: empty_name = config['dir'] + os.sep + '.tmp'
90160dbf50 2011-03-06 c.kworr@b84a3: with open(empty_name, 'w+b') as some_file:
90160dbf50 2011-03-06 c.kworr@b84a3: pass
90160dbf50 2011-03-06 c.kworr@b84a3: os.renames(empty_name, temp_name)
90160dbf50 2011-03-06 c.kworr@b84a3: temp_file = open(temp_name, 'r+b')
90160dbf50 2011-03-06 c.kworr@b84a3: if requested_ranges == None and needed == None:
90160dbf50 2011-03-06 c.kworr@b84a3: needed = new_record['_parts']
90160dbf50 2011-03-06 c.kworr@b84a3: needed.rewind()
90160dbf50 2011-03-06 c.kworr@b84a3: while True:
90160dbf50 2011-03-06 c.kworr@b84a3: (start, end) = needed.pop()
90160dbf50 2011-03-06 c.kworr@b84a3: if start == None:
90160dbf50 2011-03-06 c.kworr@b84a3: break
90160dbf50 2011-03-06 c.kworr@b84a3: stream_last = start
90160dbf50 2011-03-06 c.kworr@b84a3: old_record = new_record
90160dbf50 2011-03-06 c.kworr@b84a3: if end - start < block_size:
90160dbf50 2011-03-06 c.kworr@b84a3: req_block_size = end - start
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: req_block_size = block_size
90160dbf50 2011-03-06 c.kworr@b84a3: buffer = source.read(req_block_size)
90160dbf50 2011-03-06 c.kworr@b84a3: length = len(buffer)
90160dbf50 2011-03-06 c.kworr@b84a3: while length > 0 and stream_last < end:
90160dbf50 2011-03-06 c.kworr@b84a3: stream_pos = stream_last + length
90160dbf50 2011-03-06 c.kworr@b84a3: assert not stream_pos > end, 'Received more data then requested: pos:{} start:{} end:{}.'.format(stream_pos, start, end)
90160dbf50 2011-03-06 c.kworr@b84a3: temp_file.seek(stream_last)
90160dbf50 2011-03-06 c.kworr@b84a3: temp_file.write(buffer)
90160dbf50 2011-03-06 c.kworr@b84a3: new_record['_parts'] = new_record['_parts'] - spacemap.SpaceMap({stream_last: stream_pos})
90160dbf50 2011-03-06 c.kworr@b84a3: index[my_path] = old_record
90160dbf50 2011-03-06 c.kworr@b84a3: index.sync()
90160dbf50 2011-03-06 c.kworr@b84a3: old_record = new_record
90160dbf50 2011-03-06 c.kworr@b84a3: stream_last = stream_pos
90160dbf50 2011-03-06 c.kworr@b84a3: if end - stream_last < block_size:
90160dbf50 2011-03-06 c.kworr@b84a3: req_block_size = end - stream_last
90160dbf50 2011-03-06 c.kworr@b84a3: buffer = source.read(req_block_size)
90160dbf50 2011-03-06 c.kworr@b84a3: length = len(buffer)
90160dbf50 2011-03-06 c.kworr@b84a3: # moving downloaded data to real file
90160dbf50 2011-03-06 c.kworr@b84a3: temp_file.close()
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: index[my_path] = new_record
80f8e3804a 2010-08-20 c.kworr@b84a3: index.sync()
80f8e3804a 2010-08-20 c.kworr@b84a3:
80f8e3804a 2010-08-20 c.kworr@b84a3: except urllib.error.HTTPError as error:
80f8e3804a 2010-08-20 c.kworr@b84a3: # in case of error we don't need to do anything actually,
80f8e3804a 2010-08-20 c.kworr@b84a3: # if file download stalls or fails the file would not be moved to it's location
80f8e3804a 2010-08-20 c.kworr@b84a3: print(error)
80f8e3804a 2010-08-20 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: print(index[my_path])
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: if not os.access(file_name, os.R_OK) and os.access(temp_name, os.R_OK) and '_parts' in index[my_path] and index[my_path]['_parts'] == spacemap.SpaceMap():
90160dbf50 2011-03-06 c.kworr@b84a3: # just moving
90160dbf50 2011-03-06 c.kworr@b84a3: # drop old dirs XXX
90160dbf50 2011-03-06 c.kworr@b84a3: print('Moving temporary file to new destination.')
90160dbf50 2011-03-06 c.kworr@b84a3: os.renames(temp_name, file_name)
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: if not my_path in index:
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_response(502)
90160dbf50 2011-03-06 c.kworr@b84a3: self.end_headers()
90160dbf50 2011-03-06 c.kworr@b84a3: return
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: if self.command == 'HEAD':
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_response(200)
90160dbf50 2011-03-06 c.kworr@b84a3: if 'Content-Length' in index[my_path]:
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Content-Length', index[my_path]['Content-Length'])
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Accept-Ranges', 'bytes')
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Content-Type', 'application/octet-stream')
90160dbf50 2011-03-06 c.kworr@b84a3: if 'Last-Modified' in index[my_path]:
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Last-Modified', index[my_path]['Last-Modified'])
90160dbf50 2011-03-06 c.kworr@b84a3: self.end_headers()
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: if ('_parts' in index[my_path] and index[my_path]['_parts'] != spacemap.SpaceMap()) or not os.access(file_name, os.R_OK):
90160dbf50 2011-03-06 c.kworr@b84a3: file_name = temp_name
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: with open(file_name, 'rb') as real_file:
90160dbf50 2011-03-06 c.kworr@b84a3: file_stat = os.stat(file_name)
90160dbf50 2011-03-06 c.kworr@b84a3: if 'Range' in self.headers:
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_response(206)
90160dbf50 2011-03-06 c.kworr@b84a3: ranges = ()
90160dbf50 2011-03-06 c.kworr@b84a3: requested_ranges.rewind()
90160dbf50 2011-03-06 c.kworr@b84a3: while True:
90160dbf50 2011-03-06 c.kworr@b84a3: pair = requested_ranges.pop()
90160dbf50 2011-03-06 c.kworr@b84a3: if pair[0] == None:
90160dbf50 2011-03-06 c.kworr@b84a3: break
90160dbf50 2011-03-06 c.kworr@b84a3: ranges += '{}-{}'.format(pair[0], str(pair[1] - 1)),
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Content-Range', 'bytes {}/{}'.format(','.join(ranges), index[my_path]['Content-Length']))
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_response(200)
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Content-Length', str(file_stat.st_size))
90160dbf50 2011-03-06 c.kworr@b84a3: requested_ranges = spacemap.SpaceMap({0: file_stat.st_size})
90160dbf50 2011-03-06 c.kworr@b84a3: if 'Last-Modified' in index[my_path]:
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Last-Modified', index[my_path]['Last-Modified'])
90160dbf50 2011-03-06 c.kworr@b84a3: self.send_header('Content-Type', 'application/octet-stream')
90160dbf50 2011-03-06 c.kworr@b84a3: self.end_headers()
90160dbf50 2011-03-06 c.kworr@b84a3: if self.command in ('GET'):
90160dbf50 2011-03-06 c.kworr@b84a3: if len(requested_ranges) > 0:
90160dbf50 2011-03-06 c.kworr@b84a3: requested_ranges.rewind()
90160dbf50 2011-03-06 c.kworr@b84a3: (start, end) = requested_ranges.pop()
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: start = 0
9a8a46bcf0 2011-09-06 c.kworr@b84a3: # XXX ugly hack
9a8a46bcf0 2011-09-06 c.kworr@b84a3: if 'Content-Length' in index[my_path]:
9a8a46bcf0 2011-09-06 c.kworr@b84a3: end = index[my_path]['Content-Length']
9a8a46bcf0 2011-09-06 c.kworr@b84a3: else:
9a8a46bcf0 2011-09-06 c.kworr@b84a3: end = 0
90160dbf50 2011-03-06 c.kworr@b84a3: real_file.seek(start)
90160dbf50 2011-03-06 c.kworr@b84a3: if block_size > end - start:
90160dbf50 2011-03-06 c.kworr@b84a3: req_block_size = end - start
90160dbf50 2011-03-06 c.kworr@b84a3: else:
90160dbf50 2011-03-06 c.kworr@b84a3: req_block_size = block_size
90160dbf50 2011-03-06 c.kworr@b84a3: buffer = real_file.read(req_block_size)
90160dbf50 2011-03-06 c.kworr@b84a3: length = len(buffer)
90160dbf50 2011-03-06 c.kworr@b84a3: while length > 0:
90160dbf50 2011-03-06 c.kworr@b84a3: self.wfile.write(buffer)
90160dbf50 2011-03-06 c.kworr@b84a3: start += len(buffer)
90160dbf50 2011-03-06 c.kworr@b84a3: if req_block_size > end - start:
90160dbf50 2011-03-06 c.kworr@b84a3: req_block_size = end - start
90160dbf50 2011-03-06 c.kworr@b84a3: if req_block_size == 0:
90160dbf50 2011-03-06 c.kworr@b84a3: break
90160dbf50 2011-03-06 c.kworr@b84a3: buffer = real_file.read(req_block_size)
90160dbf50 2011-03-06 c.kworr@b84a3: length = len(buffer)
90160dbf50 2011-03-06 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: def do_HEAD(self):
90160dbf50 2011-03-06 c.kworr@b84a3: return self.__process()
90160dbf50 2011-03-06 c.kworr@b84a3: def do_GET(self):
90160dbf50 2011-03-06 c.kworr@b84a3: return self.__process()
80f8e3804a 2010-08-20 c.kworr@b84a3:
90160dbf50 2011-03-06 c.kworr@b84a3: config.section('general')
90160dbf50 2011-03-06 c.kworr@b84a3: server = http.server.HTTPServer(('127.0.0.1', int(config['port'])), MyRequestHandler)
90160dbf50 2011-03-06 c.kworr@b84a3: server.serve_forever()