CODEWITHSUNDEEP
kubernetesfluent-bit
S Kr
S Kr

Reputation: 1840

FluentBit doesn't work with multiple tail options for Kubernetes filter

This is one that works when using single tail input

inputs: |
    [INPUT]
        Name    tail
        Tag     kube.*
        Path    /var/log/containers/*.log
        Parser  docker

  filters: |
    [FILTER]
        Name             kubernetes
        Match            *
        Kube_URL         https://kubernetes.default.svc:443
        Kube_CA_File     /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File  /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix  kube.var.log.containers.
        Merge_Log        On
        Merge_Log_Key    log_processed
    [FILTER]
        Name    Lua
        Match   kube.*
        code    function dummy_filter(a,b,c)local n=c;n["dummy"]="dummy";return 2,b,n end
        call    dummy_filter
    [FILTER]
        Name             parser
        Match            kube.*
        Key_Name         log
        Parser           tomcat_parser
        Preserve_Key     On
        Reserve_Data     On
    [FILTER]
        Name    Lua
        Match   kube.*
        code    function dummy_filter1(a,b,c)local n=c;n["dummy1"]="dummy1";return 2,b,n end
        call    dummy_filter1

  customParsers: |
    [PARSER]
       Format regex
       Name   tomcat_parser
       Regex  ^(?<apptime>[0-9-a-zA-Z]+\s[0-9:\.]+)\s+(?<level>[a-zA-Z]+)\s+\[(?<thread>[a-zA-Z]+)\]\s+(?<applog>.*$)

  outputs: |
    [OUTPUT]
        Name              cloudwatch_logs
        Match             kube.*
        Region            ${region}
        Log_Group_Name    /myapps/logs
        Log_Stream_Prefix my
        Auto_Create_Group On
        net.keepalive     Off

And this doesn't work. final output in /myapps/tomcatlogs has data from all the 3 remaining filters except from the kubernetes.

inputs: |
    [INPUT]
        Name    tail
        Tag     kube.*
        Path    /var/log/containers/*.log
        Parser  docker
    [INPUT]
        Name    tail
        Tag     tomcat.*
        Path    /var/log/containers/tomcat*.log. (checked even *.log doesn't work)
        Parser  docker

  filters: |
    [FILTER]
        Name             kubernetes
        Match            *
        Kube_URL         https://kubernetes.default.svc:443
        Kube_CA_File     /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File  /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix  kube.var.log.containers.
        Merge_Log        On
        Merge_Log_Key    log_processed
    [FILTER]
        Name    Lua
        Match   tomcat.*
        code    function dummy_filter(a,b,c)local n=c;n["dummy"]="dummy";return 2,b,n end
        call    dummy_filter
    [FILTER]
        Name             parser
        Match            tomcat.*
        Key_Name         log
        Parser           tomcat_parser
        Preserve_Key     On
        Reserve_Data     On
    [FILTER]
        Name    Lua
        Match   tomcat.*
        code    function dummy_filter1(a,b,c)local n=c;n["dummy1"]="dummy1";return 2,b,n end
        call    dummy_filter1

  customParsers: |
    [PARSER]
       Format regex
       Name   tomcat_parser
       Regex  ^(?<apptime>[0-9-a-zA-Z]+\s[0-9:\.]+)\s+(?<level>[a-zA-Z]+)\s+\[(?<thread>[a-zA-Z]+)\]\s+(?<applog>.*$)

  outputs: |
    [OUTPUT]
        Name              cloudwatch_logs
        Match             kube.*
        Region            ${region}
        Log_Group_Name    /myapps/logs
        Log_Stream_Prefix my
        Auto_Create_Group On
        net.keepalive     Off
    [OUTPUT]
        Name              cloudwatch_logs
        Match             tomcat.*
        Region            ${region}
        Log_Group_Name    /myapps/tomcatlogs
        Log_Stream_Prefix my
        Auto_Create_Group On
        net.keepalive     Off

I don't like the existing sol as non-tomcat logs too gets evaluated in the tomcat filter. Any guidance will be appreciated.

Upvotes: 0

Views: 2494

Answers (1)

Jose David Palacio
Jose David Palacio

Reputation: 143

Your tomcat Input tag the records as tomcat.* which means that they will managed as:

tomcat.var.log.containers.tomcat*.log

And your kubernetes filter has

Kube_Tag_Prefix   kube.var.log.containers.

So the tomcat tagged records won fit the kube tag prefix and so it won't be able to parse correctly the log files names. You can set the Log_level as debug for fluent-bit inside the [SERVICE]. This will give you a more detail information about what is happening.

Hope this helps!

Upvotes: 2

Related Questions