35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
config['user'],
config['password'],
config['host'],
config['database'],
) )
def load(self, csv_data):
insert = self._db.prepare("select set($1, $2, $3)")
insert(row[0], row[1], row[2])
self._db.execute('vacuum analyze site;')
self._db.execute('vacuum analyze urls;')
# this classes processes config file and substitutes default values
class Config:
__slots__ = frozenset(['_config', '_default', '_section'])
_default = {
|
<
>
>
>
>
>
|
|
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
config['user'],
config['password'],
config['host'],
config['database'],
) )
def load(self, csv_data):
with self._db.xact():
config.section('loader')
if config['drop_database']:
self._db.execute('delete from urls;')
if config['drop_site']:
self._db.execute('delete from site;');
insertreg = self._db.prepare("select set($1, $2, $3)")
insert = self._db.prepare("select set($1, $2)")
for row in csv_data:
if len(row[2]) > 0:
insertreg(row[0], row[1], row[2])
else:
insert(row[0], row[1])
self._db.execute('vacuum analyze site;')
self._db.execute('vacuum analyze urls;')
# this classes processes config file and substitutes default values
class Config:
__slots__ = frozenset(['_config', '_default', '_section'])
_default = {
|