Overview
| Comment: | option to skip ETag processing, X-Powered-By ignored |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | master | trunk |
| Files: | files | file ages | folders |
| SHA3-256: |
38b25713ebf3d32fc39180982dd63b1d |
| User & Date: | c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 on 2010-07-26 09:38:15.000 |
| Other Links: | branch diff | manifest | tags |
Context
|
2010-08-04
| ||
| 07:55 | adding verbose support, i.e. silencing most of stuff check-in: 827033dd7e user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk | |
|
2010-07-26
| ||
| 09:38 | option to skip ETag processing, X-Powered-By ignored check-in: 38b25713eb user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk | |
|
2010-07-06
| ||
| 15:02 | changed logic and some log messages check-in: 083ec707ea user: c.kworr@b84a3442-36b4-a7b2-c7ad-07429f13c525 tags: master, trunk | |
Changes
Modified samesite.py
from [16e005263b]
to [81ccd8c170].
1 2 3 4 5 6 7 8 9 10 11 |
#!/usr/bin/env python3.1
import datetime, http.cookiejar, optparse, os, sys, shelve, re, urllib.request
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action = 'store_true', dest = 'verbose', help = 'turns on verbose status notifications', metavar = 'bool', default = False)
parser.add_option('-d', '--dir', action = 'store', dest = 'dir', help = 'specify directory where the files should be stored', metavar = 'string', default = None)
parser.add_option('-r', '--root', action = 'store', dest = 'root', help = 'specify a site from which data should be mirrored', metavar = 'string', default = None)
parser.add_option('-l', '--log', action = 'store', dest = 'log', help = 'specify a log file to process', metavar = 'string', default = None)
(options, args) = parser.parse_args()
| > < | < < < | < < < | < < | < < | | > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
#!/usr/bin/env python3.1
import datetime, http.cookiejar, optparse, os, sys, shelve, re, urllib.request
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action = 'store_true', dest = 'verbose', help = 'turns on verbose status notifications', metavar = 'bool', default = False)
parser.add_option('-d', '--dir', action = 'store', dest = 'dir', help = 'specify directory where the files should be stored', metavar = 'string', default = None)
parser.add_option('-r', '--root', action = 'store', dest = 'root', help = 'specify a site from which data should be mirrored', metavar = 'string', default = None)
parser.add_option('-l', '--log', action = 'store', dest = 'log', help = 'specify a log file to process', metavar = 'string', default = None)
parser.add_option('-e', '--skip-etag', action = 'store_true', dest = 'noetag', help = 'do not process etags', metavar = 'bool', default = False)
(options, args) = parser.parse_args()
assert options.dir, 'Directory not specified'
assert options.root, 'Server not specified'
assert options.log, 'Log file not specified'
assert os.access(options.log, os.R_OK), 'Log file unreadable'
# this is file index - everything is stored in this file
index = shelve.open(options.dir + '/.index')
desc_fields = ('Content-Length', 'Pragma', 'Last-Modified')
ignore_fields = ('Accept-Ranges', 'Age', 'Cache-Control', 'Connection', 'Content-Type', 'Date', 'Expires', 'Server', 'Via', 'X-Cache', 'X-Cache-Lookup', 'X-Powered-By')
if not options.noetag:
desc_fields += 'ETag',
else:
ignore_fields += 'ETag',
block_size = 32768
while True:
unchecked_files = set()
checked_files = 0
|
| ︙ | ︙ | |||
103 104 105 106 107 108 109 |
less_keys = old_keys - set(new_headers.keys())
if len(more_keys) > 0:
if not len(old_keys) == 0:
print('More headers appear:', more_keys)
reload = True
elif len(less_keys) > 0:
print('Less headers appear:', less_keys)
| < | 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
less_keys = old_keys - set(new_headers.keys())
if len(more_keys) > 0:
if not len(old_keys) == 0:
print('More headers appear:', more_keys)
reload = True
elif len(less_keys) > 0:
print('Less headers appear:', less_keys)
else:
for key in index[url].keys():
if key not in ('__time__', 'Pragma') and not index[url][key] == new_headers[key]:
print('Header "', key, '" changed from [', index[url][key], '] to [', new_headers[key], ']', sep='')
reload = True
# downloading file
|
| ︙ | ︙ |