nginx 支持lua及lua脚本格式化日志

iyuytre

1.安装LuaJIT-2.0.4
链接：
http://luajit.org/download.html

1 2 3 4

tar -xf LuaJIT-2.0.4.tar.gz cd LuaJIT-2.0.4 make PREFIX=/usr/local/luajit make install PREFIX=/usr/local/luajit

2.修改环境变量

1 2 3 4

vim /etc/profile export LUAJIT_LIB=/usr/local/luajit/lib export LUAJIT_INC=/usr/local/luajit/include/luajit-2.0 source /etc/profile

3.下载nginx lua模块
ngx_devel_kit 链接：https://github.com/simpl/ngx_devel_kit/tags
lua-nginx-module 链接：https://github.com/openresty/lua-nginx-module/tags

4.编译nginx 增加支持模块

1 2 3

cd /tmp/soft/ tar -xf ngx_devel_kit-0.3.0.tar.gz tar -xf lua-nginx-module-0.10.5.tar.gz

4.1.编译nginx

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36

tar -xf nginx-1.9.14.tar.gz cd nginx-1.9.14 ./configure \ --prefix=/app/local/nginx \ --pid-path=/app/local/nginx \ --user=nginx \ --group=nginx \ --with-threads \ --with-file-aio \ --with-http_ssl_module \ --with-http_v2_module \ --with-http_addition_module \ --with-http_sub_module \ --with-http_dav_module \ --with-http_flv_module \ --with-http_mp4_module \ --with-http_perl_module \ --with-mail \ --with-http_gzip_static_module \ --with-http_auth_request_module \ --with-http_random_index_module \ --with-http_secure_link_module \ --with-http_degradation_module \ --with-http_slice_module \ --with-http_stub_status_module \ --with-http_perl_module \ --with-zlib=/tmp/soft/zlib-1.2.8 \ --with-stream \ --with-stream_ssl_module \ --with-pcre=/tmp/soft/pcre-8.37 \ --with-openssl=/tmp/soft/openssl-1.0.2 \ --with-libatomic \ --add-module=/tmp/soft/ngx_log_if-master \ --add-module=/tmp/soft/ngx_devel_kit-0.3.0 \ --add-module=/tmp/soft/lua-nginx-module-0.10.5 make

5.备份原有的程序及增加链接

1 2 3 4

mv /app/local/nginx/sbin/nginx /app/local/nginx/sbin/nginx.0729 cp ./objs/nginx /app/local/nginx/sbin/ ln -s /usr/local/lib/libpcre.so.1 /lib64/libpcre.so.1 ln -s /usr/local/luajit/lib/libluajit-5.1.so.2 /lib64/libluajit-5.1.so.2

6.查看模块

1 2 3 4 5 6

# /app/local/nginx/sbin/nginx -V nginx version: nginx/1.9.14 built by gcc 4.4.7 20120313 (Red Hat 4.4.7-17) (GCC) built with OpenSSL 1.0.2 22 Jan 2015 TLS SNI support enabled configure arguments: --prefix=/app/local/nginx --pid-path=/app/local/nginx --user=nginx --group=nginx --with-threads --with-file-aio --with-http_ssl_module --with-http_v2_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_perl_module --with-mail --with-http_gzip_static_module --with-http_auth_request_module --with-http_random_index_module --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module --with-http_stub_status_module --with-http_perl_module --with-zlib=/tmp/soft/zlib-1.2.8 --with-stream --with-stream_ssl_module --with-pcre=/tmp/soft/pcre-8.37 --with-openssl=/tmp/soft/openssl-1.0.2 --with-libatomic --add-module=/tmp/soft/ngx_log_if-master --add-module=/tmp/soft/ngx_devel_kit-0.3.0 --add-module=/tmp/soft/lua-nginx-module-0.10.5

7.nginx 修改为json格式
修改很简单，如下:

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16

log_format ng_json '{'                          '"http_cdn_src_ip":"$http_cdn_src_ip",'                          '"time_local": "$time_local",'                          '"request":"$request",'                          '"status":"$status",'                          '"body_bytes_sent":"$body_bytes_sent",'                          '"request_body":"$request_body",'                          '"content_length":"$content_length",'                          '"http_referer":"$http_referer",'                          '"http_user_agent":"$http_user_agent",'                          '"http_x_forwarded_for":"$http_x_forwarded_for",'                          '"remote_addr":"$remote_addr",'                          '"upstream_response_time":"$upstream_response_time",'                          '"request_time":"$request_time",'                          '"http_x_trace_code":"$http_x_trace_code"} }';

查看生成的json：

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16

{     "http_cdn_src_ip": "-",     "time_local": "29/Jul/2016:03:01:02 +0800",     "request": "GET /q.gif?platform=pc&category=player&action=bufferEmpty&t=1469732461170&loc=/star/3459038 HTTP/1.1",     "status": "200",     "body_bytes_sent": "43",     "request_body": "-",     "content_length": "-",     "http_referer": "http://www.ckl.com/ckl/3459038",     "http_user_agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 BIDUBrowser/7.5 Safari/537.36",     "http_x_forwarded_for": "-",     "remote_addr": "117.174.247.112",     "upstream_response_time": "-",     "request_time": "0.000",     "http_x_trace_code": "-" }

发现，request 字段依然不是json格式，如何修改，nginx测试无法实现（自己无法实现）所以修改了，增加lua脚本，不通过nginx记录日志
而是直接使用lua来记录，lua脚本如下：

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42

#cd /app/local/nginx/lua/ #vim parse.lua local args = {} args = ngx.req.get_uri_args() local v_prev = "" local sp = "\"" for key,val in pairs(args) do     if key == nil or val == nil then     else         v_prev = v_prev .. sp ..  key .. sp .. ":" .. sp .. val .. sp .. ","     end end function isnil(value)     if value == nil then         value = "-"     end     return value end local logContent = "" if v_prev ~= nil then     local http_cdn_src_ip = isnil(ngx.var.http_cdn_src_ip)     local time_local = isnil(ngx.var.time_local)     local status = isnil(ngx.var.status)     local body_bytes_sent = isnil(ngx.var.body_bytes_sent)     local request_body = isnil(ngx.var.request_body)     local content_length = isnil(ngx.var.content_length)     local http_referer = isnil(ngx.var.http_referer)     local http_user_agent = isnil(ngx.var.http_user_agent)     local http_x_forwarded_for = isnil(ngx.var.http_x_forwarded_for)     local remote_addr = isnil(ngx.var.remote_addr)     local upstream_response_time = isnil(ngx.var.upstream_response_time)     local request_time = isnil(ngx.var.request_time)     local http_x_trace_code = isnil(ngx.var.http_x_trace_code)     logContent = "{\"http_cdn_src_ip\":" .. sp .. http_cdn_src_ip .. sp .. ",\"time_local\":" .. sp .. time_local .. sp  .. ",\"request\":" ..  "{" .. string.sub(v_prev,1,#v_prev-1) ..  "}" .. ",\"status\":" .. sp .. status .. sp .. ",\"body_bytes_sent\":" .. sp .. body_bytes_sent .. sp .. ",\"request_body\":" .. sp .. request_body .. sp .. ",\"content_length\":" .. sp .. content_length .. sp .. ",\"http_referer\":" .. sp .. http_referer .. sp .. ",\"http_user_agent\":" .. sp .. http_user_agent .. sp .. ",\"http_x_forwarded_for\":" .. sp .. http_x_forwarded_for .. sp .. ",\"remote_addr\":" .. sp .. remote_addr .. sp .. ",\"upstream_response_time\":" .. sp .. upstream_response_time .. sp .. ",\"request_time\":" .. sp .. request_time .. sp .. ",\"http_x_trace_code\":" .. sp .. http_x_trace_code .. sp .. "}"     local file = io.open("/app/data/log/nginx/ckl_access.log","a")     local hc = "\n"     file:write(logContent)     file:write(hc)     file:close() end ngx.status = 200 ngx.exit(0)

修改nginx配置：
增加如下：

1 2 3 4 5

location ~ /ckl.gif { set $logContent ""; default_type text/plain; content_by_lua_file "/app/local/nginx/lua/parse.lua"; }

重启nginx 生效，这个必须重启

1	/etc/init.d/nginx restart

查看新的json

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22

{     "http_cdn_src_ip": "-",     "time_local": "01/Aug/2016:02:01:02 +0800",     "request": {         "t": "1469988090580",         "category": "player",         "action": "bufferEmpty",         "platform": "pc",         "loc": "/v/2955653"     },     "status": "000",     "body_bytes_sent": "0",     "request_body": "-",     "content_length": "-",     "http_referer": "http://www.ckl.com/v/2955653",     "http_user_agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",     "http_x_forwarded_for": "-",     "remote_addr": "221.226.105.101",     "upstream_response_time": "-",     "request_time": "0.000",     "http_x_trace_code": "-" }