35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
|
config['user'],
config['password'],
config['host'],
config['database'],
) )
def load(self, csv_data):
insert = self._db.prepare("select mark($1, array[$2], $3)")
with self._db.xact():
config.section('loader')
if config['drop_database']:
self._db.execute('delete from urls; delete from site;');
#print('dropped', config['drop_database'])
for row in csv_data:
insert(row[0], row[1], row[2])
self._db.execute('vacuum analyze site;')
self._db.execute('vacuum analyze urls;')
# this classes processes config file and substitutes default values
class Config:
|
|
|
|
>
|
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
config['user'],
config['password'],
config['host'],
config['database'],
) )
def load(self, csv_data):
insert = self._db.prepare("select set($1, $2, $3)")
with self._db.xact():
config.section('loader')
if config['drop_database']:
self._db.execute('delete from urls;')
if config['drop_site']:
self._db.execute('delete from site;');
for row in csv_data:
insert(row[0], row[1], row[2])
self._db.execute('vacuum analyze site;')
self._db.execute('vacuum analyze urls;')
# this classes processes config file and substitutes default values
class Config:
|
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
'user': 'squidTag',
'password': 'password',
'host': 'localhost',
'database': 'squidTag',
},
'loader': {
'drop_database': False,
},}
# function to read in config file
def __init__(self):
parser = optparse.OptionParser()
parser.add_option('-c', '--config', dest = 'config',
help = 'config file location', metavar = 'FILE',
default = '/usr/local/etc/squid-tagger.conf')
parser.add_option('-d', '--drop-database', dest = 'drop_database',
help = 'signals loader to drop previous database',
action = 'store_true')
(options, args) = parser.parse_args()
if options.drop_database:
self._default['loader']['drop_database'] = True
if not os.access(options.config, os.R_OK):
print("Can't read {}: exitting".format(options.config))
sys.exit(2)
self._config = configparser.ConfigParser()
self._config.readfp(open(options.config))
|
>
>
>
>
>
>
>
|
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
'user': 'squidTag',
'password': 'password',
'host': 'localhost',
'database': 'squidTag',
},
'loader': {
'drop_database': False,
'drop_site': False,
},}
# function to read in config file
def __init__(self):
parser = optparse.OptionParser()
parser.add_option('-c', '--config', dest = 'config',
help = 'config file location', metavar = 'FILE',
default = '/usr/local/etc/squid-tagger.conf')
parser.add_option('-d', '--drop-database', dest = 'drop_database',
help = 'signals loader to drop previous database',
action = 'store_true')
parser.add_option('-D', '--drop-site', dest = 'drop_site',
help = 'signals loader to drop not only url definitions but site index too',
action = 'store_true')
(options, args) = parser.parse_args()
if options.drop_database:
self._default['loader']['drop_database'] = True
if options.drop_site:
self._default['loader']['drop_site'] = True
if not os.access(options.config, os.R_OK):
print("Can't read {}: exitting".format(options.config))
sys.exit(2)
self._config = configparser.ConfigParser()
self._config.readfp(open(options.config))
|