Overview
Comment: | optimized remains detection |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | master | trunk |
Files: | files | file ages | folders |
SHA3-256: |
b0975a28fbc7c10f43d18cace9f764b8 |
User & Date: | c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 on 2010-08-26 15:06:19.000 |
Other Links: | branch diff | manifest | tags |
Context
2010-09-06
| ||
20:53 | More features: * path substitution * more fields ignored in requests; * request now uses original path - not modified by stripping or substituting; * some headers are proxied to request; * when there is no file to send 502 error is returned; * recheck file before moving it to full dir; * Last-Modified only sent when present. check-in: cab908195f user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk | |
2010-08-26
| ||
15:06 | optimized remains detection check-in: b0975a28fb user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk | |
2010-08-25
| ||
15:16 | multiconfiguration - working with many sites at ones fullfile - avoiding partial transfers Pragma support in requests files are checked both in full dir and part dir file is moved to full dir even if it's left untouched check-in: e7b837a681 user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk | |
Changes
Modified samesite.py
from [f1c4262f2e]
to [ed1a5254d3].
︙ | ︙ | |||
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 | if my_path in index: index[my_path][header] = self.headers[header] else: print('Unknown header - ', header, ': ', self.headers[header], sep='') return() print(header, self.headers[header]) # creating empty placeholder in index # if there's no space map and there's no file in real directory - we have no file # if there's an empty space map - file is full # space map generally covers every bit of file we don't posess currently if not my_path in index: info += '\nThis one is new.' reload = True record = {} else: record = index[my_path] if not '_parts' in record: record['_parts'] = None | > > > > > > > > > > > > > > > > < < < < < < < < < < < < < | | > < | | | > > | | 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 | if my_path in index: index[my_path][header] = self.headers[header] else: print('Unknown header - ', header, ': ', self.headers[header], sep='') return() print(header, self.headers[header]) # creating file name from my_path file_name = config['dir'] + os.sep + re.compile('%20').sub(' ', my_path) # partial file or unfinished download temp_name = config['dir'] + os.sep + '.parts' + re.compile('%20').sub(' ', my_path) # creating empty placeholder in index # if there's no space map and there's no file in real directory - we have no file # if there's an empty space map - file is full # space map generally covers every bit of file we don't posess currently if not my_path in index: info += '\nThis one is new.' reload = True record = {} else: # forcibly checking file if no file present if os.access(file_name, os.R_OK): file_stat = os.stat(file_name) elif '_parts' in index[my_path] and os.access(temp_name, os.R_OK): file_stat = os.stat(temp_name) else: info += '\nFile not found or inaccessible.' index[my_path]['_parts'] = None reload = True record = index[my_path] print(record) if not '_parts' in record: record['_parts'] = None if record['_parts'] == None: recheck = True # forcibly checking file if file size doesn't match with index data if not reload: if '_parts' in record and record['_parts'] == spacemap.SpaceMap(): if 'Content-Length' in record and file_stat and file_stat.st_size != int(record['Content-Length']): info += '\nFile size is {} and stored file size is {}.'.format(file_stat.st_size, record['Content-Length']) record['_parts'] = None reload = True # forcibly checking file if index holds Pragma header if not reload and 'Pragma' in record and record['Pragma'] == 'no-cache': info +='\nPragma on: recheck imminent.' recheck = True # skipping file processing if there's no need to recheck it and we have checked it at least 4 hours ago if not recheck and not reload and '_time' in record and (datetime.datetime.now() - datetime.timedelta(hours = 4) - record['_time']).days < 0: recheck = True print(info) if reload or recheck: try: request = 'http://' + config['root'] + my_path needed = None # XXX and if we specify full file we don't go partial? if requested_ranges != None: if '_parts' in record and record['_parts'] != None: if config['noparts']: needed = record['_parts'] else: needed = record['_parts'] | requested_ranges elif not config['noparts']: needed = requested_ranges ranges = () print('Missing ranges: {}, requested ranges: {}, needed ranges: {}.'.format(record['_parts'], requested_ranges, needed)) if needed != None and len(needed) > 0: needed.rewind() while True: range = needed.pop() if range[0] == None: break ranges += '{}-{}'.format(range[0], range[1] - 1), request = urllib.request.Request(request, headers = {'Range': 'bytes=' + ','.join(ranges)}) |
︙ | ︙ | |||
366 367 368 369 370 371 372 | if reload: print('Reloading.') if os.access(temp_name, os.R_OK): os.unlink(temp_name) if os.access(file_name, os.R_OK): os.unlink(file_name) | < | 371 372 373 374 375 376 377 378 379 380 381 382 383 384 | if reload: print('Reloading.') if os.access(temp_name, os.R_OK): os.unlink(temp_name) if os.access(file_name, os.R_OK): os.unlink(file_name) new_record['_parts'] = spacemap.SpaceMap({0: int(new_record['Content-Length'])}) print(new_record) # downloading file or segment if 'Content-Length' in new_record: if needed == None: needed = new_record['_parts'] |
︙ | ︙ |