Samesite - proxy that can cache partial transfers

Check-in [996aa0149d]
anonymous

Check-in [996aa0149d]

Overview
Comment:simpler way to configure hosts, more examples for Windows Update, support for unconfigured hosts
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: 996aa0149d0e89c93c10abb70a33692f820e6d833ee0ca238c50e1107f3ec130
User & Date: arcade@b1t.name on 2013-03-13 17:21:38.000
Other Links: branch diff | manifest | tags
Context
2013-08-14
08:54
one more header, python3.3 check-in: 7ff9724ae4 user: arcade@b1t.name tags: master, trunk
2013-03-13
17:21
simpler way to configure hosts, more examples for Windows Update, support for unconfigured hosts check-in: 996aa0149d user: arcade@b1t.name tags: master, trunk
2013-01-29
14:33
capitilize header correctly check-in: a2857db2b5 user: arcade@b1t.name tags: master, trunk
Changes
Unified Diff Ignore Whitespace Patch
Modified nginx.conf from [2a30458003] to [b3d4f05e0b]. 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# sample nginx config to serve the local site

server {
	listen *:80;
	listen [fec0::1]:80;
	server_name site.to.hijack;
	access_log /var/log/nginx/site_name.access.log;
	error_log /var/log/nginx/site_name.error.log debug;
	expires -1;
	try_files $uri @break;
	error_page 404 = @break;
	log_not_found off;
	# autoindex on;

	location / {
		if ($request_method != HEAD) {
			root /path/to/www/site_name/;
		}
	}

	location @break {
		# pointing out where our server listens
		proxy_pass http://localhost:8008;
		# saying that we downloading from real site
		proxy_set_header Host real.site.name;
		proxy_read_timeout 600;
	}
}






|










|







|




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# sample nginx config to serve the local site

server {
	listen *:80;
	listen [fec0::1]:80;
	server_name ^(?<domain>.+).hijack;
	access_log /var/log/nginx/site_name.access.log;
	error_log /var/log/nginx/site_name.error.log debug;
	expires -1;
	try_files $uri @break;
	error_page 404 = @break;
	log_not_found off;
	# autoindex on;

	location / {
		if ($request_method != HEAD) {
			root /path/to/www/$domain;
		}
	}

	location @break {
		# pointing out where our server listens
		proxy_pass http://localhost:8008;
		# saying that we downloading from real site
		proxy_set_header Host $domain;
		proxy_read_timeout 600;
	}
}
Modified samesite.conf from [11abb2a4b4] to [5f0eac2d62]. 
1
2
3
4
5
6



7
8
9
10









11






12

# basic list of options for each site:
# - noetag: ignore etags
# - noparts: request full file even for partial request
# - proto: protocol, defaults to http

# Examples:




[download.windowsupdate.com]
noetag: yes










[download.microsoft.com]






noetag: yes







>
>
>




>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

# basic list of options for each site:
# - noetag: ignore etags
# - noparts: request full file even for partial request
# - proto: protocol, defaults to http

# Examples:

[download.microsoft.com]
noetag: yes

[download.windowsupdate.com]
noetag: yes

[au.download.windowsupdate.com]
noetag: yes

[ds.download.windowsupdate.com]
noetag: yes

[www.download.windowsupdate.com]
noetag: yes

[update.microsoft.com]
noetag: yes

[fe1.update.microsoft.com]
noetag: yes

[www.update.microsoft.com]
noetag: yes
Modified samesite.py from [83281b8c59] to [c17785c61d]. 
1
2
3
4
5
6
7
8
9
10
11
12
13
14

#!/usr/bin/env python3.2

from __future__ import unicode_literals, print_function

#import gevent.monkey
#gevent.monkey.patch_all()

import argparse, os
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', dest = 'config', help = 'config file location', metavar = 'FILE', default = 'samesite.conf')
args = parser.parse_args()
assert os.access(args.config, os.R_OK), "Fatal error: can't read {}".format(args.config)

import configparser



<
<
<
<
<








1
2





3
4
5
6
7
8
9

#!/usr/bin/env python3.2






import argparse, os
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', dest = 'config', help = 'config file location', metavar = 'FILE', default = 'samesite.conf')
args = parser.parse_args()
assert os.access(args.config, os.R_OK), "Fatal error: can't read {}".format(args.config)

import configparser
︙︙
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

			thisDir = re.compile('^(.*)/$').match(config[section]['dir'])
			if thisDir:
				config[section]['dir'] = thisDir.group(1)
			if not re.compile('^/(.*)$').match(config[section]['dir']):
				config[section]['dir'] = cache_dir + os.sep + config[section]['dir']
		else:
			config[section]['dir'] = cache_dir + os.sep + section

		if not 'root' in config[section]:
			config[section]['root'] = section

#assert options.port or os.access(options.log, os.R_OK), 'Log file unreadable'

const_desc_fields = set(['Content-Length', 'Last-Modified', 'Pragma'])
const_ignore_fields = set([








<








29
30
31
32
33
34
35

36
37
38
39
40
41
42

			thisDir = re.compile('^(.*)/$').match(config[section]['dir'])
			if thisDir:
				config[section]['dir'] = thisDir.group(1)
			if not re.compile('^/(.*)$').match(config[section]['dir']):
				config[section]['dir'] = cache_dir + os.sep + config[section]['dir']
		else:
			config[section]['dir'] = cache_dir + os.sep + section

		if not 'root' in config[section]:
			config[section]['root'] = section

#assert options.port or os.access(options.log, os.R_OK), 'Log file unreadable'

const_desc_fields = set(['Content-Length', 'Last-Modified', 'Pragma'])
const_ignore_fields = set([
︙︙
75
76
77
78
79
80
81




82
83
84
85
86
87
88


		myPath = re.compile('^(.*?)(\?.*)$').match(self.path)
		if myPath:
			my_path = myPath.group(1)
		else:
			my_path = self.path





		config_host = config[self.headers['Host']]

		if config_host['sub'] != None and config_host['strip'] != None and len(config_host['strip']) > 0:
			string = re.compile(config_host['strip']).sub(config_host['sub'], my_path)
			my_path = string

		my_path_b = my_path.encode('utf-8')








>
>
>
>








69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86


		myPath = re.compile('^(.*?)(\?.*)$').match(self.path)
		if myPath:
			my_path = myPath.group(1)
		else:
			my_path = self.path

		if not config.has_section(self.headers['Host']):
			config.add_section(self.headers['Host'])
			config[self.headers['Host']]['root'] = self.headers['Host']
			config[self.headers['Host']]['dir'] = cache_dir + os.sep + self.headers['Host']
		config_host = config[self.headers['Host']]

		if config_host['sub'] != None and config_host['strip'] != None and len(config_host['strip']) > 0:
			string = re.compile(config_host['strip']).sub(config_host['sub'], my_path)
			my_path = string

		my_path_b = my_path.encode('utf-8')
︙︙