Index: database.sql ================================================================== --- database.sql +++ database.sql @@ -134,22 +134,40 @@ LANGUAGE sql immutable STRICT AS $$ select mark(get_site($1), $2) as result; $$; --- this function adds tag to domain with regexp -CREATE or replace FUNCTION mark(domain text, tags text[], regexp text) RETURNS integer - LANGUAGE sql immutable STRICT +-- this function sets tags for site/regexp pair +CREATE or replace FUNCTION set(my_id_site integer, my_id_tag integer) RETURNS integer + LANGUAGE sql STRICT + AS $$ +delete from urls where $1 = id_site and regexp is null; +insert into urls (id_site, id_tag) values ($1, $2); +select $1; +$$; + +-- this function sets tags for site/regexp pair +CREATE or replace FUNCTION set(my_id_site integer, my_id_tag integer, my_regexp text) RETURNS integer + LANGUAGE sql STRICT AS $$ -select mark(get_site($1), get_tag($2), $3) as result; +delete from urls where $1 = id_site and $3 = regexp; +insert into urls (id_site, id_tag, regexp) values ($1, $2, $3); +select $1; +$$; + +-- this function stores new data for site/regexp pair +create or replace function set(domain text, tags text, regexp text) returns integer + language sql immutable strict + as $$ +select set(get_site($1), get_tag($2::text[]), $3); $$; --- this function adds tag to domain with regexp -CREATE or replace FUNCTION mark(domain text, tags text[]) RETURNS integer - LANGUAGE sql immutable STRICT - AS $$ -select mark(get_site($1), get_tag($2), NULL) as result; +-- this function stores new data for site/regexp pair +create or replace function set(domain text, tags text) returns integer + language sql immutable strict + as $$ +select set(get_site($1), get_tag($2::text[])); $$; -- this function returns id of tag array create or replace function get_tag(my_tag text[]) returns integer language plpgsql strict Index: st-load.py ================================================================== --- st-load.py +++ st-load.py @@ -37,16 +37,17 @@ config['host'], config['database'], ) ) def load(self, csv_data): - insert = self._db.prepare("select mark($1, array[$2], $3)") + insert = self._db.prepare("select set($1, $2, $3)") with self._db.xact(): config.section('loader') if config['drop_database']: - self._db.execute('delete from urls; delete from site;'); - #print('dropped', config['drop_database']) + self._db.execute('delete from urls;') + if config['drop_site']: + self._db.execute('delete from site;'); for row in csv_data: insert(row[0], row[1], row[2]) self._db.execute('vacuum analyze site;') self._db.execute('vacuum analyze urls;') @@ -66,10 +67,11 @@ 'host': 'localhost', 'database': 'squidTag', }, 'loader': { 'drop_database': False, + 'drop_site': False, },} # function to read in config file def __init__(self): parser = optparse.OptionParser() @@ -77,16 +79,22 @@ help = 'config file location', metavar = 'FILE', default = '/usr/local/etc/squid-tagger.conf') parser.add_option('-d', '--drop-database', dest = 'drop_database', help = 'signals loader to drop previous database', action = 'store_true') + parser.add_option('-D', '--drop-site', dest = 'drop_site', + help = 'signals loader to drop not only url definitions but site index too', + action = 'store_true') (options, args) = parser.parse_args() if options.drop_database: self._default['loader']['drop_database'] = True + if options.drop_site: + self._default['loader']['drop_site'] = True + if not os.access(options.config, os.R_OK): print("Can't read {}: exitting".format(options.config)) sys.exit(2) self._config = configparser.ConfigParser()