CODEWITHSUNDEEP
logstash
Denyo
Denyo

Reputation: 117

truncate/prune logstash config to be easy maintainable

I would like to truncate/prune my logstash configuration.

As u can see it is very long and always repeating the same. I was not able to minimize/eliminate code/config doublication.

see below: for every tomcat instance there are application-, access-, standardout- and standerderror-logs which were shipped by filebeat to the elkserver:9200.

There are round about 15 instances, which ends up to 60 "if" and "else" instructions.

Does anyone have a hint, how to truncate/prune the output section?

input {
  beats {
    port => 5044
    ssl => true
    ssl_certificate_authorities => ["/etc/logstash/root-ca.pem"]
    ssl_certificate => "/etc/logstash/elkserver.pem"
    ssl_key => "/etc/logstash/elkserver.key"
    ssl_verify_mode => "force_peer"
  }
}

filter {
  grok { match => ["message",'^%{TIMESTAMP_ISO8601:TIMESTAMP} %{LOGLEVEL:LEVEL} *\[(?<CLASS>[A-Za-z0-9$]+).%{NOTSPACE:METHOD}:%{NONNEGINT:LINE}:%{NOTSPACE:THREAD}\] %{GREEDYDATA:MESSAGE}$'] }
  if "beats_input_codec_plain_applied" in [tags] { mutate { remove_tag => ["beats_input_codec_plain_applied"] } }
}

output {
  if "jt09_02_access" in [tags] {
    elasticsearch {
      hosts => ["elkserver:9200"]
      user => user
      password => password
      ssl => true
      ssl_certificate_verification => true
      cacert => "/etc/logstash/master.pem"
      index => "jt09_02_access"
    }
  } else if "jt07_02_access" in [tags] {
      elasticsearch {
        hosts => ["elkserver:9200"]
        user => user
        password => password
        ssl => true
        ssl_certificate_verification => true
        cacert => "/etc/logstash/master.pem"
        index => "jt07_02_access"
      }
  } else if "jt07_04_access" in [tags] {
      elasticsearch {
        hosts => ["elkserver:9200"]
        user => user
        password => password
        ssl => true
        ssl_certificate_verification => true
        cacert => "/etc/logstash/master.pem"
        index => "jt07_04_access"
      }
  } else if "jt07_01_access" in [tags] {
      elasticsearch {
        hosts => ["elkserver:9200"]
        user => user
        password => password
        ssl => true
        ssl_certificate_verification => true
        cacert => "/etc/logstash/master.pem"
        index => "jt07_01_access"
      }
  }  else if "jt07_09_sdterr" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         ssl_certificate_verification => true
         cacert => "/etc/logstash/master.pem"
         index => "jt07_09_sdterr"
       }
  }  else if "jt07_09_sdtout" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         ssl_certificate_verification => true
         cacert => "/etc/logstash/master.pem"
         index => "jt07_09_sdtout"
       }
  }  else if "jt07_09_custom_pattern" in [tags] {
        elasticsearch {
          hosts => ["elkserver:9200"]
          user => user
          password => password
          ssl => true
          ssl_certificate_verification => true
          cacert => "/etc/logstash/master.pem"
          index => "jt07_09_custom_pattern"
        }
  }  else if "jt09_01_access" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_01_access"
       }
  }  else if "jt09_03_access" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_03_access"
       }
  }  else if "jt09_01_sdterr" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_01_sdterr"
       }
  }  else if "jt09_01_sdtout" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_01_sdterr"
       }
  }  else if "jt09_02_sdterr" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_02_sdterr"
       }
  }  else if "jt09_02_sdtout" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_02_sdtout"
       }
  }  else if "jt09_03_sdterr" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_03_sdterr"
       }
  }  else if "jt09_03_sdtout" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt09_03_sdtout"
       }
  }  else if "jt08_03_access" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt08_03_access"
       }
  }  else if "jt08_03_sdterr" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt08_03_sdterr"
       }
  }  else if "jt08_03_sdtout" in [tags] {
       elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         cacert => "/etc/logstash/master.pem"
         index => "jt08_03_sdtout"
       }
  }
}

Thanks Michael Dz, your answer helped me to solve my issue in the following way:

Filebeat adds already the required tag while shipping the data

- type: log
  paths: pathto\log_custompattern.log
  tags: ["jt07_09_custom_pattern"]
  close_older: 24h

- type: log
  paths: pathto\tomcat-stdout.??????????.log
  tags: ["jt07_09_sdtout"]
  scan_frequency: 30s
  close_inactive: 12h

- type: log
  paths: pathto\tomcat-stderr.??????????.log
  tags: ["jt07_09_sdterr"]
  scan_frequency: 30s
  close_inactive: 12h
  ...

and Logstash now does the output in the following way

output {
  elasticsearch {
    hosts => ["ma-javainfra02.konzern.mvvcorp.de:9200"]
    user => admin
    password => logfileselkadmin
    ssl => true
    ssl_certificate_verification => true
    cacert => "/etc/logstash/master.pem"
    index => "%{tags[0]}"
  }
}

Upvotes: 1

Views: 171

Answers (1)

Michael Dz
Michael Dz

Reputation: 3834

You can add a tag as a new field and then based on the new field name create an index, I don't know how you make tags or how many of them are stored in the array but I will assume that you are interested in the first one.

filter {
  mutate {
      add_field => { "[@metadata][indexname]", "tags[0]" }
  }
}
output {
   elasticsearch {
         hosts => ["elkserver:9200"]
         user => user
         password => password
         ssl => true
         ssl_certificate_verification => true
         cacert => "/etc/logstash/master.pem"
         index => "%{[@metadata][indexname]}"
   }
}

Upvotes: 1

Related Questions