Reputation: 187
Logstash stopping randomly after few hours
I'm getting a random error when I run logstash:
16:30:26.240 [[main]>worker0] ERROR logstash.pipeline - Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash. {"exception"=>#, "backtrace"=>["org/jruby/RubyString.java:3101:in
gsub'", "org/jruby/RubyString.java:3069:ingsub'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:317:ingsub_dynamic_fields'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:308:ingsub'", "org/jruby/RubyArray.java:1613:ineach'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:290:ingsub'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:207:infilter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:145:indo_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:164:inmulti_filter'", "org/jruby/RubyArray.java:1613:ineach'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:161:inmulti_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filter_delegator.rb:41:inmulti_filter'", "(eval):4135:ininitialize'", "org/jruby/RubyArray.java:1613:ineach'", "(eval):4131:ininitialize'", "org/jruby/RubyProc.java:281:incall'", "(eval):997:infilter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:295:infilter_batch'", "org/jruby/RubyProc.java:281:incall'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_synchronous_queue.rb:192:ineach'", "org/jruby/RubyHash.java:1342:ineach'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_synchronous_queue.rb:191:ineach'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:294:infilter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:282:inworker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:258:instart_workers'"]} 16:30:26.542 [LogStash::Runner] FATAL logstash.runner - An unexpected error occurred! {:error=>#<InterruptedRegexpError: Regexp Interrupted>, :backtrace=>["org/jruby/RubyString.java:3101:ingsub'", "org/jruby/RubyString.java:3069:ingsub'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:317:ingsub_dynamic_fields'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:308:ingsub'", "org/jruby/RubyArray.java:1613:ineach'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:290:ingsub'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:207:infilter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:145:indo_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:164:inmulti_filter'", "org/jruby/RubyArray.java:1613:ineach'", "/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:161:inmulti_filter'", "/usr/share/logstash/logstash-core/lib/logstash/filter_delegator.rb:41:inmulti_filter'", "(eval):4135:ininitialize'", "org/jruby/RubyArray.java:1613:ineach'", "(eval):4131:ininitialize'", "org/jruby/RubyProc.java:281:incall'", "(eval):997:infilter_func'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:295:infilter_batch'", "org/jruby/RubyProc.java:281:incall'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_synchronous_queue.rb:192:ineach'", "org/jruby/RubyHash.java:1342:ineach'", "/usr/share/logstash/logstash-core/lib/logstash/util/wrapped_synchronous_queue.rb:191:ineach'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:294:infilter_batch'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:282:inworker_loop'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:258:instart_workers'"]}
My logstash config file is:
input {
file {
type => "SystemError"
path => "/app/systemerr/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^\s"
what => "previous"
}
}
file {
type => "SystemOut"
path => "/app/systemout/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
file {
type => "Errorlog"
path => "/app/error/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^FATAL"
negate => true
what => "previous"
}
}
file {
type => "Messagelog"
path => "/app/message/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^ERROR"
negate => true
what => "previous"
}
}
}
filter {
if [type] == "SystemError" {
grok {
match => { "message" => "\[%{DATA:timestamp}] %{BASE16NUM:threadID} (?<shortname>\b[A-Za-z0-9\$]{2,}\b)%{SPACE}%{WORD:loglevel}%{SPACE} %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
mutate {
gsub => ["timestamp", " GMT\+05\:30", ""]
}
date {
match => ["timestamp", "M/dd/yy HH:mm:ss:SSS"]
}
if ([message] =~ "^\tat") {
drop {}
}
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
if [type] == "SystemOut" {
grok {
match => { "message" => "\[%{DATA:timestamp}] %{BASE16NUM:threadID} (?<shortname>\b[A-Za-z0-9\$]{2,}\b)%{SPACE}%{WORD:loglevel}%{SPACE} %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
mutate {
gsub => ["timestamp", " GMT\+05\:30", ""]
}
date {
match => ["timestamp", "M/dd/yy HH:mm:ss:SSS"]
}
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
if [type] == "Errorlog" {
grok {
match => { "message" => "%{LOGLEVEL:loglevel} \| %{TIMESTAMP_ISO8601:timestamp} \| %{DATA:string} \: %{DATA:WebContainer} \| %{DATA:code} \| %{DATA:country} \| %{DATA:user} \| %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
date {
match => ["timestamp", "yyyy-M-dd HH:mm:ss,SSS"]
}
mutate { remove_field => [ "string" ] }
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
if [type] == "Messagelog" {
grok {
match => { "message" => "%{LOGLEVEL:loglevel} \| %{TIMESTAMP_ISO8601:timestamp} \| %{DATA:string} \: %{DATA:WebContainer} \| %{DATA:code} \| %{DATA:country} \| %{DATA:user} \| %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
date {
match => ["timestamp", "yyyy-M-dd HH:mm:ss,SSS"]
}
mutate {
remove_field => [ "string" ]
}
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
}
Is there anything wrong in the config file? Please help.
Upvotes: 0
Views: 681
Answers (1)
Reputation: 17155
you are likely getting a _grokparsefailure and so the timestamp field isn't set. You can surround the mutate/date with an if block like this:
if "_grokparsefailure" not in [tags] {
mutate {
gsub => ["timestamp", " GMT\+05\:30", ""]
}
date {
match => ["timestamp", "M/dd/yy HH:mm:ss:SSS"]
}
}
you may also want to add an else { drop {} }, but you should probably figure out what isn't matching first.
Upvotes: 2
Related Questions
- Logstash got exception in running
- Logstash shuts down during clustering
- Logstash stopped processing because of an error: (SystemExit) exit
- How can i resolve a problem with Logstash?
- Logstash - java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit Error in Logstash
- logstash http output infinite retries
- Logstash worker dies with no reason
- Logstash not running
- Logstash stops receiving after some time in ELK stack?
- Logstash keep runing without crashing or output (windows as well as linux)