|
ELK收集中断,定位问题到redis发现redis内存不足,所以迁移到新的redis上
elasticsearch 删除索引语句:
1
| curl -XDELETE http://localhost:9200/logstash-nginx.access-2016.05.31
|
错误1:Failed to send event to Redis
1
| Failed to send event to Redis {:event=>#<LogStash::Event:0x4062cce3 @metadata_accessors=#<LogStash::Util::Accessors:0x7cbdd35c @store={"path"=>"/app/local/log/nginx/ckl_access.log"}, @lut={"[path]"=>[{"path"=>"/app/local/log/nginx/ckl_access.log"}, "path"]}>, @cancelled=false, @data={"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, @metadata={"path"=>"/app/local/log/nginx/ckl_access.log"}, @accessors=#<LogStash::Util::Accessors:0x64f18bac @store={"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, @lut={"path"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "path"], "host"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "host"], "type"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "type"], "host_name"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "host_name"], "[type]"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "type"]}>>, :identity=>"default", :exception=>#<Redis::CommandError: OOM command not allowed when used memory > 'maxmemory'.>, :backtrace=>["/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis/client.rb:121:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:1070:in `rpush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:58:in `synchronize'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/monitor.rb:211:in `mon_synchronize'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:58:in `synchronize'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:1069:in `rpush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-redis-2.0.5/lib/logstash/outputs/redis.rb:246:in `send_to_redis'", "org/jruby/RubyProc.java:281:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-codec-json-2.1.4/lib/logstash/codecs/json.rb:42:in `encode'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-redis-2.0.5/lib/logstash/outputs/redis.rb:152:in `receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/outputs/base.rb:83:in `multi_receive'", "org/jruby/RubyArray.java:1613:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/outputs/base.rb:83:in `multi_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/output_delegator.rb:130:in `worker_multi_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/output_delegator.rb:114:in `multi_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:301:in `output_batch'", "org/jruby/RubyHash.java:1342:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:301:in `output_batch'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:232:in `worker_loop'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in `start_workers'"], :level=>:warn}
|
发现如下:
1
| commanderror oom command not allowed when used memory
|
定位为redis问题
修改redis 配置文件,去掉max-memeory选项即可
错误2:elasticsearch` is obsolete and is no longer available
1
| {:timestamp=>"2016-08-03T17:21:26.451000+0800", :message=>"Pipeline aborted due to error", :exception=>#<LogStash::ConfigurationError: The setting `host` in plugin `elasticsearch` is obsolete and is no longer available. Please use the 'hosts' setting instead. You can specify multiple entries separated by comma in 'host:port' format. If you have any questions about this, you are invited to visit https://discuss.elastic.co/c/logstash and ask.>, :backtrace=>["/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/config/mixin.rb:87:in `config_init'", "org/jruby/RubyHash.java:1342:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/config/mixin.rb:71:in `config_init'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/outputs/base.rb:63:in `initialize'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/output_delegator.rb:74:in `register'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/pipeline.rb:181:in `start_workers'", "org/jruby/RubyArray.java:1613:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/pipeline.rb:181:in `start_workers'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/pipeline.rb:136:in `run'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/agent.rb:465:in `start_pipeline'"], :level=>:error}
|
原配置文件:
1
2
3
4
5
6
7
8
9
10
| elasticsearch {
hosts => ["10.11.11.12"]
protocol => "http"
index => "logstash-%{type}-%{+YYYY.MM.dd}"
document_type => "%{type}"
workers => 5
flush_size => 3840
idle_flush_time => 10
template_overwrite => true
}
|
解决:
新的配置文件:
1
2
3
4
5
6
7
8
9
| elasticsearch {
hosts => "10.11.11.12"
index => "logstash-%{type}-%{+YYYY.MM.dd}"
document_type => "%{type}"
workers => 5
flush_size => 3840
idle_flush_time => 10
template_overwrite => true
}
|
错误3: Permission denied
1
2
3
4
5
| Errno::EACCES: Permission denied - /tmp/logstash-log/log-2016.08.03.log
initialize at org/jruby/RubyFile.java:370
new at org/jruby/RubyIO.java:853
open at /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-file-2.2.5/lib/logstash/outputs/file.rb:264
write_event at /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-file-2.2.5/lib/logstash/outputs/file.rb:162
|
logstash 日志目录修改:
1
| chown -R logstash.logstash logstash-log/
|
|
|