Logstats: Xml filter: How to split attributes of multiple nodes,Parent in separate rows

Question

I am new to Kibana, logstash. I am trying push XML (generated as the output of nunit) in logstash. I want to have the XML elements attributes, its Parents attributes and child attributes in the same row in Kibana. However, I can get XML elements attribute of different elements in one single as an array. However, I would want to split each element of XML into separate rows

Below is the xml file:

<test-suite type="TestFixture" name="bootAPIs" executed="True" result="Failure" success="False" time="12.811" asserts="0">
  <results>
    <test-case name="NunitBETests.bootAPIs.GetChannelMultiFilter" executed="True" result="Failure" success="False" time="1.582" asserts="0">
      <failure>
        <message>
          <![CDATA[Either there are no linear channels or response is incorrect,
 REQUEST is : {"initObj":{"Locale":{"LocaleLanguage":"","LocaleCountry":"","LocaleDevice":"","LocaleUserState":"Unknown"},"Platform":"ConnectedTV","SiteGuid":"958863","DomainID":543335,"UDID":"616439088037","ApiUser":"tvpapi_185si","ApiPass":"A2d4G6","Token":""},"ChannelID":"3407167","picSize":"302x170","pageSize":0,"pageIndex":0,"orderBy":"None","tagsMetas":[],"cutWith":"AND"}
 RESPONSE is : []]]>
        </message>
        <stack-trace>
          <![CDATA[at NunitBETests.bootAPIs.GetChannelMultiFilter() in c:\vinayDoNotDelete\ComponentTestallFiles\Softwares\OtherCsharpSOlutions\NunitBETests\bootAPIs.cs:line 183
]]>
        </stack-trace>
      </failure>
      kccndgjjw
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetDomainInfo" executed="True" result="Success" success="True" time="0.939" asserts="0">
      <reason>
        <message />
      </reason>
    </test-case>
    xptmu
    <test-case name="NunitBETests.bootAPIs.GetDomainPermittedItems" executed="True" result="Success" success="True" time="0.910" asserts="0">
      <reason>
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetDomainPermittedSubscriptions" executed="True" result="Success" success="True" time="0.921" asserts="0">
      <reason>
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetEPGMultiChannelProgram" executed="True" result="Success" success="True" time="1.574" asserts="0">
      <reason>
        qvlbshks
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetItemFromList" executed="True" result="Success" success="True" time="0.908" asserts="0">
      <reason>
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetMenu" executed="True" result="Success" success="True" time="0.913" asserts="0">
      <reason>
        kwagbbo
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetRecordings" executed="True" result="Success" success="True" time="1.787" asserts="0">
      <reason>
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetSecuredSiteGuid" executed="True" result="Success" success="True" time="0.903" asserts="0">
      obzldvhx
      <reason>
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetSeriesRecordings" executed="True" result="Success" success="True" time="0.944" asserts="0">
      <reason>
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.GetUserData" executed="True" result="Success" success="True" time="0.910" asserts="0">
      bw
      <reason>
        <message />
      </reason>
    </test-case>
    <test-case name="NunitBETests.bootAPIs.RefreshAccessToken" executed="True" result="Success" success="True" time="0.462" asserts="0">
      <reason>
        <message />
      </reason>
    </test-case>
  </results>
</test-suite>

In Logstash config I tried options like force_array => false, force_content => true and variations in xpath like:

xpath =>
   [
    "/test-suite/@name", "suitename",
    "//test-case[1]/@name", "testcase",
    "//test-case[1]/@result", "res",
    "//test-case[1]/reason/@name","reasonattr",
    "//test-case[1]/@time", "timetest"
   ]


   xpath =>
   [
    "/test-suite/@name", "suitename",
    "//test-case[*]/@name", "testcase",
    "//test-case[*]/@result", "res",
    "//test-case[*]/reason/@name","reasonattr",
    "//test-case[*]/@time", "timetest"
   ]

   xpath =>
   [
    "/test-suite/results/test-case/@name","testcase",
    "/test-suite/@name", "suitename"
   ]

but I got output like below in almost all cases, where the data of all test cases is packed in a single row:

{
     "suitename" => [
        [0] "bootAPIs"
    ],
      "timetest" => [
        [ 0] 1.582,
        [ 1] 0.939,
        [ 2] 0.91,
        [ 3] 0.921,
        [ 4] 1.574,
        [ 5] 0.908,
        [ 6] 0.913,
        [ 7] 1.787,
        [ 8] 0.903,
        [ 9] 0.944,
        [10] 0.91,
        [11] 0.462
    ],
      "testcase" => [
        [ 0] "NunitBETests.bootAPIs.GetChannelMultiFilter",
        [ 1] "NunitBETests.bootAPIs.GetDomainInfo",
        [ 2] "NunitBETests.bootAPIs.GetDomainPermittedItems",
        [ 3] "NunitBETests.bootAPIs.GetDomainPermittedSubscriptions",
        [ 4] "NunitBETests.bootAPIs.GetEPGMultiChannelProgram",
        [ 5] "NunitBETests.bootAPIs.GetItemFromList",
        [ 6] "NunitBETests.bootAPIs.GetMenu",
        [ 7] "NunitBETests.bootAPIs.GetRecordings",
        [ 8] "NunitBETests.bootAPIs.GetSecuredSiteGuid",
        [ 9] "NunitBETests.bootAPIs.GetSeriesRecordings",
        [10] "NunitBETests.bootAPIs.GetUserData",
        [11] "NunitBETests.bootAPIs.RefreshAccessToken"
    ],
      "@version" => "1",
          "host" => "DESKTOP-PC8JBMK",
           "res" => [
        [ 0] "Failure",
        [ 1] "Success",
        [ 2] "Success",
        [ 3] "Success",
        [ 4] "Success",
        [ 5] "Success",
        [ 6] "Success",
        [ 7] "Success",
        [ 8] "Success",
        [ 9] "Success",
        [10] "Success",
        [11] "Success"
    ]
}

What I am wanting is each of the test case attribute, Suite Name attribute (parent of test case) to be collected as a single row for EACH test cases present in the xml file. Something like below is what i Want to get: Please let me know how I could do this:

      {
         "suitename" => [
            [0] "bootAPIs"
        ],
          "timetest" => [
            [ 0] 1.582,

        ],
          "testcase" => [
            [ 0] "NunitBETests.bootAPIs.GetChannelMultiFilter",

        ],
          "@version" => "1",
              "host" => "DESKTOP-PC8JBMK",
               "res" => [
            [ 0] "Failure",

        ],
    }

    {
         "suitename" => [
            [1] "bootAPIs"
        ],
          "timetest" => [
            [ 1] 0.939,

        ],
          "testcase" => [
            [ 1] ""NunitBETests.bootAPIs.bootAPIs.GetDomainInfo"",

        ],
          "@version" => "1",
              "host" => "DESKTOP-PC8JBMK",
               "res" => [
            [ 1] "Success",

        ],
    }

    {
         "suitename" => [
            [2] "bootAPIs"
        ],
          "timetest" => [
            [ 2] 0.91,

        ],
          "testcase" => [
            [ 2] "NunitBETests.bootAPIs.GetDomainPermittedItems"",

        ],
          "@version" => "1",
              "host" => "DESKTOP-PC8JBMK",
               "res" => [
            [ 2] "Success",

        ],
    }

Answer 1

After some work was able to get to the answer. Pasting here the complete details, just in case that it may assit some one else tooo...

Problem statement: Below xml is an output generate out of Nunit . I need to collect attributes of all test-case nodes along with the attributes of its Parent and child nodes in a single event per testcase so that every event generated by Logstash has testcase name, testsuite name(parent node) and Failure message (child node) for all testcases in the xml.

XMl below:

<test-suite type="Assembly" name="C:\vinayDoNotDelete\ComponentTestallFiles\Softwares\OtherCsharpSOlutionsunitBETests\bin\Debugunitbetests.exe" executed="True" result="Failure" success="False" time="18.987" asserts="0">
  <results>
    <test-suite type="Namespace" name="NunitBETests" executed="True" result="Failure" success="False" time="18.978" asserts="0">
      <results>
        <test-suite type="TestFixture" name="bootAPIs" executed="True" result="Failure" success="False" time="12.811" asserts="0">
          <results>
            <test-case name="NunitBETests.bootAPIs.GetMOvieFilter" executed="True" result="Failure" success="False" time="1.582" asserts="0">
              <failure>
                ycaddztg
                <message>
                  <![CDATA[Either there are no linear channels or response is incorrect,
 REQUEST is : {"initObj":{"Locale":{"LocaleLanguage":"","LocaleCountry":"","LocaleDevice":"","LocaleUserState":"Unknown"},"Platform":"ConnectedTV","SiteGuid":"958863","DomainID":543335,"UDID":"616439088037","ApiUser":"tvpapi_185si","ApiPass":"A2d4G6","Token":""},"ChannelID":"3407167","picSize":"302x170","pageSize":0,"pageIndex":0,"orderBy":"None","tagsMetas":[],"cutWith":"AND"}
 RESPONSE is : []]]>
                </message>
                <stack-trace>
                  <![CDATA[at NunitBETests.bootAPIs.GetMOvieFilter() in c:\vinayDoNotDelete\ComponentTestallFiles\Softwares\OtherCsharpSOlutionsunitBETests\bootAPIs.cs:line 183
]]>
                </stack-trace>
              </failure>
            </test-case>

            <test-case name="NunitBETests.bootAPIs.GetSubscriptions" executed="True" result="Success" success="True" time="0.921" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
            kvlrw
            <test-case name="NunitBETests.bootAPIs.GetProgram" executed="True" result="Success" success="True" time="1.574" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>

            <test-case name="NunitBETests.bootAPIs.GetSchedule" executed="True" result="Success" success="True" time="1.787" asserts="0">
              <reason>
                dxgarfj
                <message />
              </reason>
            </test-case>
            <test-case name="NunitBETests.bootAPIs.GetGuid" executed="True" result="Success" success="True" time="0.903" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
            <test-case name="NunitBETests.bootAPIs.GetAllVeggies" executed="True" result="Success" success="True" time="0.944" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
            <test-case name="NunitBETests.bootAPIs.GetAllNoNVegies" executed="True" result="Success" success="True" time="0.910" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
            <test-case name="NunitBETests.bootAPIs.Refresh" executed="True" result="Success" success="True" time="0.462" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
          </results>
        </test-suite>
        <test-suite type="TestFixture" name="epgAPIs" executed="True" result="Failure" success="False" time="3.392" asserts="0">
          <results>
            <test-case name="NunitBETests.epgAPIs.GetMOvieFilter" executed="True" result="Failure" success="False" time="0.912" asserts="0">
              <failure>
                <message>
                  <![CDATA[Either there are no series recordings or response incorrect, response is : []]]>
                </message>
                kuxxlc
                <stack-trace>
                  <![CDATA[at NunitBETests.epgAPIs.GetMOvieFilter() in c:\vinayDoNotDelete\ComponentTestallFiles\Softwares\OtherCsharpSOlutionsunitBETests\epgAPIs.cs:line 43
]]>
                </stack-trace>
              </failure>
            </test-case>
            <test-case name="NunitBETests.epgAPIs.GetProgram" executed="True" result="Success" success="True" time="1.565" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
            jlsb
            <test-case name="NunitBETests.epgAPIs.GetData" executed="True" result="Success" success="True" time="0.908" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
          </results>
        </test-suite>
        <test-suite type="TestFixture" name="recordingsAPIs" executed="True" result="Success" success="True" time="2.759" asserts="0">
          <results>
            <test-case name="NunitBETests.recordingsAPIs.GetRecordings" executed="True" result="Success" success="True" time="1.810" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
            <test-case name="NunitBETests.recordingsAPIs.GetVeggies" executed="True" result="Success" success="True" time="0.944" asserts="0">
              <reason>
                <message />
              </reason>
            </test-case>
            oagfymbn
          </results>
        </test-suite>
      </results>
    </test-suite>
  </results>
</test-suite>

Logtash filter below:

input { stdin { }

}
filter {
xml {

store_xml => true
target => "poc"
source => "message"

}
split {
field => "poc[results][0][test-suite][0][results][0][test-suite]"
}

split {
field => "poc[results][0][test-suite][0][results][0][test-suite][results][0][test-case]"
}

date {
match => [ "date" , "dd-MM-yyyy HH:mm:ss" ]
timezone => "Europe/Amsterdam"
}

mutate {
add_field => { "number22" => "%{poc[results][0][test-suite][0][results][0][test-suite][name]}" }
add_field => { "name22" => "%{poc[results][0][test-suite][0][results][0][test-suite][results][0][test-case][name]}" }
add_field => { "res22" => "%{poc[results][0][test-suite][0][results][0][test-suite][results][0][test-case][result]}" }
add_field => { "reason22" => "%{poc[results][0][test-suite][0][results][0][test-suite][results][0][test-case][failure][0][message][0]}" }

}

if [reason22] == "%{poc[results][0][test-suite][0][results][0][test-suite][results][0][test-case][failure][0][message][0]}" {
mutate { update => { "reason22" => "" } }
}
}
output {
elasticsearch {
action => "index"
index => "xm64"
hosts => ["localhost:9200"]
}
stdout { codec => rubydebug }
}