1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
-- module counter - counts some junk
local today = os.date('*t')
local timestamp = os.time{year = today['year'], month = today['month'], day = today['day']}
local referer = '-';
if ngx.var.http_referer ~= nil then
_, _, referer = string.find(ngx.var.http_referer, '^https?://([%w%.]+)')
end
local key = 'stat_counter_' .. timestamp .. '_' .. referer
-- connect to redis
local redis = require 'resty.redis'
local red = redis:new()
local ok, err = red:connect('127.0.0.1', 6379)
if not ok then
ngx.log(ngx.ERR, 'redis connection failed: ', err)
ngx.exit(500)
end
local cjson = require 'cjson'
-- decodes values from array into dict
function decode_bulk(reply)
local data = {}
for j=1, #reply, 2 do
data[reply[j] ] = reply[j+1]
end
return(data)
end
local data, empty
empty, err = red:hsetnx(key, 'today', 0)
if not empty then
ngx.log(ngx.ERR, 'redis query failed: ', err)
ngx.exit(500)
end
if empty == 0 then
data, err = red:hgetall(key)
if not data then
ngx.log(ngx.ERR, 'redis query failed: ', err)
ngx.exit(500)
end
end
if empty == 1 or data.whole == nil then
-- postgres fallback
result = ngx.location.capture('/postgres', {
method = ngx.HTTP_PUT,
body = "select * from get_stats('" .. referer .. "') as (today int, lastday int, week bigint, whole bigint);"
})
if result.status ~= 200 or not result.body then
ngx.log(ngx.ERR, 'postgres access failed')
ngx.exit(500)
else
local unrds = require "rds.parser"
local res, err = unrds.parse(result.body)
if res == nil then
ngx.log(ngx.ERR, 'failed to obtain data: ' .. err)
ngx.exit(500)
else
data = res.resultset[1]
local req = {key}
for name, value in pairs(data) do
if value == unrds.null then
value = 0
end
if name ~= 'today' then
table.insert(req, name)
table.insert(req, value)
end
end
red:init_pipeline(3)
red:hmset{req}
red:expire(key, 129600)
red:hincrby(key, 'today', data.today)
res, err = red:commit_pipeline()
if not res then
ngx.log(ngx.ERR, 'redis pipeline failed: ', err)
ngx.exit(500)
end
end
end
end
data.today = data.today + 1
ngx.say(cjson.encode(data))
ngx.eof()
res, err = red:hincrby(key, 'today', 1)
local uid = ''
if ngx.var.uid_got ~= nil and string.find(ngx.var.uid_got, 'uid=') == 1 then
uid = string.sub(ngx.var.uid_got, 5)
elseif ngx.var.uid_set ~= nil and string.find(ngx.var.uid_set, 'uid=') == 1 then
uid = string.sub(ngx.var.uid_set, 5)
end
local hit_key = uid .. '_' .. referer .. '_' .. ngx.var.remote_addr
red:init_pipeline(5)
red:multi()
red:hincrby('stat_counter_pending', hit_key, 1)
red:expire('stat_counter_pending', 60)
red:renamenx('stat_counter_pending', 'stat_counter_pending_tmp')
red:exec()
renamed, err = red:commit_pipeline()
if not renamed then
ngx.log(ngx.ERR, 'redis multi failed: ', err)
ngx.exit(500)
end
local continue = false
if tonumber(renamed[5][3]) == 1 then
continue = true
end
while continue do
ngx.location.capture('/sleep')
local data
data, err = red:hgetall('stat_counter_pending_tmp')
if not data then
ngx.log(ngx.ERR, 'redis request failed: ', err)
ngx.exit(500)
end
for name, value in pairs(decode_bulk(data)) do
local fields = {}
name:gsub("[^_]+", function(c) fields[#fields + 1] = c end)
fields[#fields + 1] = value
local result = ngx.location.capture('/postgres', {
method = ngx.HTTP_PUT,
-- prepare select? XXX
body = "select merge_counter('" .. fields[1] .. "'::text, '" .. fields[2] .. "'::text, '" .. fields[3] .. "'::inet, '" .. fields[4] .. "'::smallint);"
})
if result.status ~= 200 or not result.body then
ngx.log(ngx.ERR, 'postgres access failed')
ngx.exit(500)
end
end
red:init_pipeline(5)
red:multi()
red:del('stat_counter_pending_tmp')
red:exists('stat_counter_pending')
red:renamenx('stat_counter_pending', 'stat_counter_pending_tmp')
red:exec()
renamed, err = red:commit_pipeline()
if not renamed then
ngx.log(ngx.ERR, 'redis multi failed: ', err)
ngx.exit(500)
end
if tonumber(renamed[5][1] == 0) or tonumber(renamed[5][2]) == 0 then
continue = false
end
end
|
<
<
|
<
<
<
>
>
|
|
<
|
|
|
|
|
>
|
|
<
>
|
<
|
|
>
|
>
|
|
|
|
>
>
<
<
>
|
|
|
|
|
|
|
|
|
|
>
|
<
|
|
|
<
|
|
<
<
|
>
>
>
>
|
|
|
|
>
|
|
>
>
|
>
>
>
>
>
>
>
>
>
>
>
|
>
|
>
>
>
|
>
>
>
|
>
|
|
|
|
|
|
|
|
|
|
|
<
|
|
<
|
|
|
|
|
|
|
>
|
|
|
|
<
|
|
|
|
|
|
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
-- module counter - counts some junk
local _counter = {}
local redis = require 'resty.redis'
local cjson = require 'cjson'
local closed = false
function _counter.exit()
if not closed then
local ok, err = _counter.red:set_keepalive(60000, 10)
if not ok then
ngx.log(ngx.ERR, "redis keepalive: " .. err)
end
closed = true
end
end
function _counter.decode_bulk(reply)
-- decodes values from array into dict
local data = {}
for j=1, #reply, 2 do
data[reply[j] ] = reply[j+1]
end
return data
end
function _counter.connect()
-- connect to redis
_counter.red = redis:new()
local ok, err = _counter.red:connect('127.0.0.1', 6379)
if not ok then
ngx.log(ngx.ERR, 'redis connection failed: ', err)
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
end
closed = false
end
function _counter.check_schema()
if ngx.var.site_schema == nil then
ngx.log(ngx.ERR, "No 'site_schema' specified");
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
end
end
function _counter.update()
_counter.check_schema()
local today = os.date('*t')
local timestamp = os.time{year = today['year'], month = today['month'], day = today['day']}
local referer = '-';
if ngx.var.http_referer ~= nil then
_, _, referer = string.find(ngx.var.http_referer, '^https?://([^?&]+)')
--ngx.log(ngx.ERR, "Referer: " .. referer)
end
local key = ngx.var.site_schema .. '_counter_' .. timestamp .. '_' .. referer
-- ngx.log(ngx.ERR, "Using key: " .. key)
local res, err
_counter.red:init_pipeline(4)
_counter.red:multi()
_counter.red:hincrby(key, 'today', 1)
_counter.red:hgetall(key)
_counter.red:exec()
res, err = _counter.red:commit_pipeline()
if not res then
ngx.log(ngx.ERR, 'redis pipeline failed: ', err)
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
end
local data = _counter.red:array_to_hash(res[4][2])
-- ngx.log(ngx.ERR, "Got data: " .. cjson.encode(data))
if tonumber(data.today) == 1 then
ngx.log(ngx.ERR, "Reading postgres")
-- postgres fallback
result = ngx.location.capture('/postgres', {
method = ngx.HTTP_PUT,
body = "select * from " .. ngx.var.site_schema .. ".get_stats('" .. referer .. "') as (today int, lastday int, week bigint, whole bigint);"
})
if result.status ~= 200 or not result.body then
ngx.log(ngx.ERR, 'postgres access failed')
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
else
local unrds = require "rds.parser"
local res, err = unrds.parse(result.body)
if res == nil then
ngx.log(ngx.ERR, 'failed to obtain data: ' .. err)
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
else
data = res.resultset[1]
data.today = data.today + 1
ngx.log(ngx.ERR, "Got data: " .. cjson.encode(data))
_counter.red:init_pipeline(4)
_counter.red:multi()
_counter.red:hmset(key, data)
_counter.red:expire(key, 129600)
_counter.red:exec()
res, err = _counter.red:commit_pipeline()
if not res then
ngx.log(ngx.ERR, 'redis pipeline failed: ', err)
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
end
end
end
end
ngx.say(cjson.encode(data))
ngx.eof()
local uid = ''
if ngx.var.uid_got ~= nil and string.find(ngx.var.uid_got, 'uid=') == 1 then
uid = string.sub(ngx.var.uid_got, 5)
elseif ngx.var.uid_set ~= nil and string.find(ngx.var.uid_set, 'uid=') == 1 then
uid = string.sub(ngx.var.uid_set, 5)
end
local hit_key = uid .. '_' .. referer .. '_' .. ngx.var.remote_addr
local key_pending = ngx.var.site_schema .. '_counter_pending'
_counter.red:init_pipeline(4)
_counter.red:multi()
_counter.red:hincrby(key_pending, hit_key, 1)
_counter.red:expire(key_pending, 604800)
_counter.red:exec()
res, err = _counter.red:commit_pipeline()
if not res then
ngx.log(ngx.ERR, 'redis transaction failed: ', err)
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
end
end
return _counter
|