Overview
Comment: | update script, multiple schemas, rust offloader |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | master | trunk |
Files: | files | file ages | folders |
SHA3-256: |
b33171b487379c32e85109920e2b1d72 |
User & Date: | arcade@b1t.name on 2018-06-01 10:35:23.000 |
Other Links: | branch diff | manifest | tags |
Context
2018-10-28
| ||
13:32 | update regexp to include uppercase letters Leaf check-in: f721596292 user: arcade tags: master, trunk | |
2018-06-01
| ||
10:35 | update script, multiple schemas, rust offloader check-in: b33171b487 user: arcade@b1t.name tags: master, trunk | |
2014-07-03
| ||
14:54 | initial commit check-in: 38ec946cba user: arcade@b1t.name tags: master, trunk | |
Changes
Modified counter.body.conf
from [b630972664]
to [5f43526cce].
1 2 3 4 5 6 | location = /postgres { internal; postgres_query $echo_request_body; postgres_pass counter; } | < < < < < | > < | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | location = /postgres { internal; postgres_query $echo_request_body; postgres_pass counter; } location = /counter { default_type application/json; userid on; userid_name uid; userid_expires 365d; userid_p3p 'policyref="/w3c/p3p.xml", CP="NOI CUR ADM PSA OUR STP STA"'; content_by_lua_file '/home/arcade/work/admin/nginx/counter.content.lua'; #log_by_lua_file '/home/arcade/work/admin/nginx/counter.log.lua'; expires epoch; } |
Added counter.content.lua version [d2cc80b908].
Added counter.log.lua version [08105b55ed].
Modified counter.lua
from [3b4097dd02]
to [636d67d4ad].
1 2 | -- module counter - counts some junk | < < | < < < > > | | < | | | | | > | | < > | < | | > | > | | | | > > < < > | | | | | | | | | | > | < | | | < | | < < | > > > > | | | | > | | > > | > > > > > > > > > > > | > | > > > | > > > | > | | | | | | | | | | | < | | < | | | | | | | > | | | | < | | | | | | < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | -- module counter - counts some junk local _counter = {} local redis = require 'resty.redis' local cjson = require 'cjson' local closed = false function _counter.exit() if not closed then local ok, err = _counter.red:set_keepalive(60000, 10) if not ok then ngx.log(ngx.ERR, "redis keepalive: " .. err) end closed = true end end function _counter.decode_bulk(reply) -- decodes values from array into dict local data = {} for j=1, #reply, 2 do data[reply[j] ] = reply[j+1] end return data end function _counter.connect() -- connect to redis _counter.red = redis:new() local ok, err = _counter.red:connect('127.0.0.1', 6379) if not ok then ngx.log(ngx.ERR, 'redis connection failed: ', err) ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end closed = false end function _counter.check_schema() if ngx.var.site_schema == nil then ngx.log(ngx.ERR, "No 'site_schema' specified"); ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end end function _counter.update() _counter.check_schema() local today = os.date('*t') local timestamp = os.time{year = today['year'], month = today['month'], day = today['day']} local referer = '-'; if ngx.var.http_referer ~= nil then _, _, referer = string.find(ngx.var.http_referer, '^https?://([^?&]+)') --ngx.log(ngx.ERR, "Referer: " .. referer) end local key = ngx.var.site_schema .. '_counter_' .. timestamp .. '_' .. referer -- ngx.log(ngx.ERR, "Using key: " .. key) local res, err _counter.red:init_pipeline(4) _counter.red:multi() _counter.red:hincrby(key, 'today', 1) _counter.red:hgetall(key) _counter.red:exec() res, err = _counter.red:commit_pipeline() if not res then ngx.log(ngx.ERR, 'redis pipeline failed: ', err) ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end local data = _counter.red:array_to_hash(res[4][2]) -- ngx.log(ngx.ERR, "Got data: " .. cjson.encode(data)) if tonumber(data.today) == 1 then ngx.log(ngx.ERR, "Reading postgres") -- postgres fallback result = ngx.location.capture('/postgres', { method = ngx.HTTP_PUT, body = "select * from " .. ngx.var.site_schema .. ".get_stats('" .. referer .. "') as (today int, lastday int, week bigint, whole bigint);" }) if result.status ~= 200 or not result.body then ngx.log(ngx.ERR, 'postgres access failed') ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) else local unrds = require "rds.parser" local res, err = unrds.parse(result.body) if res == nil then ngx.log(ngx.ERR, 'failed to obtain data: ' .. err) ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) else data = res.resultset[1] data.today = data.today + 1 ngx.log(ngx.ERR, "Got data: " .. cjson.encode(data)) _counter.red:init_pipeline(4) _counter.red:multi() _counter.red:hmset(key, data) _counter.red:expire(key, 129600) _counter.red:exec() res, err = _counter.red:commit_pipeline() if not res then ngx.log(ngx.ERR, 'redis pipeline failed: ', err) ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end end end end ngx.say(cjson.encode(data)) ngx.eof() local uid = '' if ngx.var.uid_got ~= nil and string.find(ngx.var.uid_got, 'uid=') == 1 then uid = string.sub(ngx.var.uid_got, 5) elseif ngx.var.uid_set ~= nil and string.find(ngx.var.uid_set, 'uid=') == 1 then uid = string.sub(ngx.var.uid_set, 5) end local hit_key = uid .. '_' .. referer .. '_' .. ngx.var.remote_addr local key_pending = ngx.var.site_schema .. '_counter_pending' _counter.red:init_pipeline(4) _counter.red:multi() _counter.red:hincrby(key_pending, hit_key, 1) _counter.red:expire(key_pending, 604800) _counter.red:exec() res, err = _counter.red:commit_pipeline() if not res then ngx.log(ngx.ERR, 'redis transaction failed: ', err) ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end end return _counter |
Modified counter.sql
from [9e36f017b8]
to [f3002f17d4].
|
| < < < | | | | | | | | | | | | | | | | < > | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | CREATE TABLE if not exists counter ( uid text NOT NULL, referer text DEFAULT '-' NOT NULL, ip inet NOT NULL, day date DEFAULT now() NOT NULL, count integer DEFAULT 0 NOT NULL ); CREATE TABLE if not exists counter_sum ( referer text DEFAULT '-'::text NOT NULL, day date DEFAULT now() NOT NULL, count integer DEFAULT 0 NOT NULL ); CREATE or replace FUNCTION get_stats(target text) RETURNS record LANGUAGE sql AS $_$ select coalesce((select count from limbo.counter_sum where day = current_date and referer = $1), 0) as today, coalesce((select count from limbo.counter_sum where day = (current_date - interval '1 day')::date and referer = $1), 0) as lastday, coalesce((select sum(count) from limbo.counter_sum where date_trunc('week', current_date) = date_trunc('week', day) and day <> current_date and referer = $1 group by referer), 0) as week, coalesce((select sum(count) from limbo.counter_sum where day <> current_date and referer = $1 group by referer), 0) as whole $_$; CREATE or replace FUNCTION counter_sum__change_trigger() RETURNS trigger LANGUAGE plpgsql AS $$ declare delta_count smallint; begin if (TG_OP = 'UPDATE') then if (old.referer <> new.referer or old.day <> new.day) then raise exception 'Changing referer or day is prohibited'; end if; delta_count = new.count - old.count; elsif (TG_OP = 'INSERT') then delta_count = new.count; elsif (TG_OP = 'DELETE') then raise exception 'Deleting counters prohibited'; end if; <<insert_update>> loop update limbo.counter_sum set count = count + delta_count where referer = new.referer and day = new.day; exit insert_update when found; begin insert into limbo.counter_sum (referer, day, count) values (new.referer, new.day, delta_count); exit insert_update; exception when unique_violation then end; end loop insert_update; return new; end; $$; CREATE or replace FUNCTION merge_counter(merge_uid text, merge_referer text, merge_ip inet, merge_count smallint) RETURNS boolean LANGUAGE plpgsql AS $$ BEGIN <<insert_update>> LOOP UPDATE limbo.counter SET count = count + merge_count WHERE uid = merge_uid and ip = merge_ip and referer = merge_referer and day = current_date; exit insert_update when found; BEGIN INSERT INTO limbo.counter (uid, referer, ip, count) VALUES (merge_uid, merge_referer, merge_ip, merge_count); EXIT insert_update; EXCEPTION WHEN unique_violation THEN END; END LOOP insert_update; return true; END; $$; CREATE INDEX if not exists counter__date ON counter USING btree (day); CREATE UNIQUE INDEX if not exists counter__uid_nreferer_ip_date ON counter (uid, ip, day) WHERE (referer IS NULL); CREATE UNIQUE INDEX if not exists counter__uid_referer_ip_date ON counter (uid, referer, ip, day); create unique index if not exists counter_sum__referer_day ON counter_sum (referer, day); drop trigger if exists counter__change on counter; CREATE TRIGGER counter__change AFTER INSERT OR DELETE OR UPDATE ON counter FOR EACH ROW EXECUTE PROCEDURE counter_sum__change_trigger(); |
Deleted counter.upstream.conf version [bda2d9f130].
Added counter.upstream.conf.sample version [804ec274ae].
Added offload/Cargo.toml version [4558953d19].
Added offload/offload.toml.sample version [9440cc0f00].
Added offload/src/main.rs version [1e0198fa61].