lua-counter example

Diff
anonymous

Diff

Differences From Artifact [3b4097dd02]:

To Artifact [636d67d4ad]:


1
2
3
4
5
6
7
8
9


10
11
12
13
14
15
16
17
18
19

20
21
22
23

24
25
26
27
28
29
30
31
32
33

34

35
36
37
38


39
40
41

42
43
44
45
46
47
48
49
50
51
52
53
54

55
56
57
58
59
60
61
62
63
64
65




66
67
68
69

70
71


72











73

74



75



76
77

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
-- module counter - counts some junk

local today = os.date('*t')
local timestamp = os.time{year = today['year'], month = today['month'], day = today['day']}
local referer = '-';

if ngx.var.http_referer ~= nil then
	_, _, referer = string.find(ngx.var.http_referer, '^https?://([%w%.]+)')
end



local key = 'stat_counter_' .. timestamp .. '_' .. referer

-- connect to redis
local redis = require 'resty.redis'
local red = redis:new()
local ok, err = red:connect('127.0.0.1', 6379)
if not ok then
	ngx.log(ngx.ERR, 'redis connection failed: ', err)
	ngx.exit(500)

end

local cjson = require 'cjson'


-- decodes values from array into dict
function decode_bulk(reply)
	local data = {}
	for j=1, #reply, 2 do
		data[reply[j] ] = reply[j+1]
	end
	return(data)
end

local data, empty

empty, err = red:hsetnx(key, 'today', 0)

if not empty then
	ngx.log(ngx.ERR, 'redis query failed: ', err)
	ngx.exit(500)
end



if empty == 0 then
	data, err = red:hgetall(key)

	if not data then
		ngx.log(ngx.ERR, 'redis query failed: ', err)
		ngx.exit(500)
	end
end

if empty == 1 or data.whole == nil then
	-- postgres fallback
	result = ngx.location.capture('/postgres', {
		method = ngx.HTTP_PUT,
		body = "select * from get_stats('" .. referer .. "') as (today int, lastday int, week bigint, whole bigint);"
	})
	if result.status ~= 200 or not result.body then

		ngx.log(ngx.ERR, 'postgres access failed')
		ngx.exit(500)
	else
		local unrds = require "rds.parser"
		local res, err = unrds.parse(result.body)
		if res == nil then
			ngx.log(ngx.ERR, 'failed to obtain data: ' .. err)
			ngx.exit(500)
		else
			data = res.resultset[1]
			local req = {key}





			for name, value in pairs(data) do
				if value == unrds.null then
					value = 0

				end
				if name ~= 'today' then


					table.insert(req, name)











					table.insert(req, value)

				end



			end




			red:init_pipeline(3)

			red:hmset{req}
			red:expire(key, 129600)
			red:hincrby(key, 'today', data.today)
			res, err = red:commit_pipeline()
			if not res then
				ngx.log(ngx.ERR, 'redis pipeline failed: ', err)
				ngx.exit(500)
			end
		end
	end
end

data.today = data.today + 1
ngx.say(cjson.encode(data))
ngx.eof()

res, err = red:hincrby(key, 'today', 1)
local uid = ''

if ngx.var.uid_got ~= nil and string.find(ngx.var.uid_got, 'uid=') == 1 then
	uid = string.sub(ngx.var.uid_got, 5)
elseif ngx.var.uid_set ~= nil and string.find(ngx.var.uid_set, 'uid=') == 1 then
	uid = string.sub(ngx.var.uid_set, 5)
end

local hit_key = uid .. '_' .. referer .. '_' .. ngx.var.remote_addr


red:init_pipeline(5)
red:multi()
red:hincrby('stat_counter_pending', hit_key, 1)
red:expire('stat_counter_pending', 60)
red:renamenx('stat_counter_pending', 'stat_counter_pending_tmp')
red:exec()
renamed, err = red:commit_pipeline()
if not renamed then
	ngx.log(ngx.ERR, 'redis multi failed: ', err)
	ngx.exit(500)
end

local continue = false
if tonumber(renamed[5][3]) == 1 then
	continue = true
end

while continue do
	ngx.location.capture('/sleep')

	local data
	data, err = red:hgetall('stat_counter_pending_tmp')
	if not data then
		ngx.log(ngx.ERR, 'redis request failed: ', err)
		ngx.exit(500)
	end

	for name, value in pairs(decode_bulk(data)) do
		local fields = {}
		name:gsub("[^_]+", function(c) fields[#fields + 1] = c end)
		fields[#fields + 1] = value
		local result = ngx.location.capture('/postgres', {
			method = ngx.HTTP_PUT,
			-- prepare select? XXX
			body = "select merge_counter('" .. fields[1] .. "'::text, '" .. fields[2] .. "'::text, '" .. fields[3] .. "'::inet, '" .. fields[4] .. "'::smallint);"
		})
		if result.status ~= 200 or not result.body then
			ngx.log(ngx.ERR, 'postgres access failed')
			ngx.exit(500)
		end
	end

	red:init_pipeline(5)
	red:multi()
	red:del('stat_counter_pending_tmp')
	red:exists('stat_counter_pending')
	red:renamenx('stat_counter_pending', 'stat_counter_pending_tmp')
	red:exec()
	renamed, err = red:commit_pipeline()
	if not renamed then
		ngx.log(ngx.ERR, 'redis multi failed: ', err)
		ngx.exit(500)
	end

	if tonumber(renamed[5][1] == 0) or tonumber(renamed[5][2]) == 0 then
		continue = false
	end
end


<
<
|

<
<
<
>
>

|

|
<
|
|
|
|
|
>
|
|
<

>
|
<




|


|
>
|
>
|
|
|
|
>
>

<
<
>
|
|
|



|
|
|
|
|
|
|
>
|
<
|
|
|
<
|
|
<
<
|
>
>
>
>
|
|
|
|
>
|
|
>
>
|
>
>
>
>
>
>
>
>
>
>
>
|
>
|
>
>
>
|
>
>
>

|
>
|
|
|
|
|
|
|
|
|
|
|

<
|
|

<
|

|
|
|
|
|

|
>

|
|
|
|
<
|
|
|
|
|
|
<
<
<
<


<
<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
1
2


3
4



5
6
7
8
9
10

11
12
13
14
15
16
17
18

19
20
21

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39


40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

56
57
58

59
60


61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

112
113
114

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

130
131
132
133
134
135




136
137


138






































-- module counter - counts some junk



local _counter = {}




local redis = require 'resty.redis'
local cjson = require 'cjson'

local closed = false

function _counter.exit()

	if not closed then
		local ok, err = _counter.red:set_keepalive(60000, 10)
		if not ok then
			ngx.log(ngx.ERR, "redis keepalive: " .. err)
		end
		closed = true
	end
end


function _counter.decode_bulk(reply)
	-- decodes values from array into dict

	local data = {}
	for j=1, #reply, 2 do
		data[reply[j] ] = reply[j+1]
	end
	return data
end

function _counter.connect()
	-- connect to redis
	_counter.red = redis:new()
	local ok, err = _counter.red:connect('127.0.0.1', 6379)
	if not ok then
		ngx.log(ngx.ERR, 'redis connection failed: ', err)
		ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
	end
	closed = false
end



function _counter.check_schema()
	if ngx.var.site_schema == nil then
		ngx.log(ngx.ERR, "No 'site_schema' specified");
		ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
	end
end

function _counter.update()
	_counter.check_schema()
	local today = os.date('*t')
	local timestamp = os.time{year = today['year'], month = today['month'], day = today['day']}
	local referer = '-';

	if ngx.var.http_referer ~= nil then
		_, _, referer = string.find(ngx.var.http_referer, '^https?://([^?&]+)')
		--ngx.log(ngx.ERR, "Referer: " .. referer)

	end

	local key = ngx.var.site_schema .. '_counter_' .. timestamp .. '_' .. referer

	-- ngx.log(ngx.ERR, "Using key: " .. key)



	local res, err
	_counter.red:init_pipeline(4)
	_counter.red:multi()
	_counter.red:hincrby(key, 'today', 1)
	_counter.red:hgetall(key)
	_counter.red:exec()
	res, err = _counter.red:commit_pipeline()
	if not res then
		ngx.log(ngx.ERR, 'redis pipeline failed: ', err)
		ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
	end

	local data = _counter.red:array_to_hash(res[4][2])
	-- ngx.log(ngx.ERR, "Got data: " .. cjson.encode(data))

	if tonumber(data.today) == 1 then
		ngx.log(ngx.ERR, "Reading postgres")
		-- postgres fallback
		result = ngx.location.capture('/postgres', {
			method = ngx.HTTP_PUT,
			body = "select * from " .. ngx.var.site_schema .. ".get_stats('" .. referer .. "') as (today int, lastday int, week bigint, whole bigint);"
		})
		if result.status ~= 200 or not result.body then
			ngx.log(ngx.ERR, 'postgres access failed')
			ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
		else
			local unrds = require "rds.parser"
			local res, err = unrds.parse(result.body)

			if res == nil then
				ngx.log(ngx.ERR, 'failed to obtain data: ' .. err)
				ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
			else
				data = res.resultset[1]
				data.today = data.today + 1
				ngx.log(ngx.ERR, "Got data: " .. cjson.encode(data))

				_counter.red:init_pipeline(4)
				_counter.red:multi()
				_counter.red:hmset(key, data)
				_counter.red:expire(key, 129600)
				_counter.red:exec()
				res, err = _counter.red:commit_pipeline()
				if not res then
					ngx.log(ngx.ERR, 'redis pipeline failed: ', err)
					ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
				end
			end
		end
	end


	ngx.say(cjson.encode(data))
	ngx.eof()


	local uid = ''

	if ngx.var.uid_got ~= nil and string.find(ngx.var.uid_got, 'uid=') == 1 then
		uid = string.sub(ngx.var.uid_got, 5)
	elseif ngx.var.uid_set ~= nil and string.find(ngx.var.uid_set, 'uid=') == 1 then
		uid = string.sub(ngx.var.uid_set, 5)
	end

	local hit_key = uid .. '_' .. referer .. '_' .. ngx.var.remote_addr
	local key_pending = ngx.var.site_schema .. '_counter_pending'

	_counter.red:init_pipeline(4)
	_counter.red:multi()
	_counter.red:hincrby(key_pending, hit_key, 1)
	_counter.red:expire(key_pending, 604800)

	_counter.red:exec()
	res, err = _counter.red:commit_pipeline()
	if not res then
		ngx.log(ngx.ERR, 'redis transaction failed: ', err)
		ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
	end




end



return _counter