Nested getline in AWK script

Question

Please let me know if we can use nested getline within AWK scripts like:

while ( ("tail -f log" |& getline var0) > 0) {

    while ( ("ls" | getline ) > 0) {
    }
    close("ls")
    while ( ("date" | getline ) > 0) {
    }
    close("date")
}
    close("tail -f log")

What is the depth we can make use of nested getline functionality and will there be any data loss of output at any level of the nested getline? What are the things we should make sure in implementing this style?

==================================================================================

UPDATE===================UPDATE==============UPDATE===============UPDATE=======

Requirement : Provide real time statistical data and errors by probing QA box and webserver / services logs and system status. Report would be generated in following format:

Local Date And Time | Category| Component | Condition

Assumption -: AWK script would execute faster than shell script with added advantage of using its inbuilt parsing and other functionalities.

Implementation : - The main command loop is command0="tail -f -n 0 -s 5 ...........". This command would start an infinite loop extracting appended logs of services / webserver of QA box. . Note the -f, -s and –n options which makes to dump all appended data to logs, sleep for 5 seconds after each iterations and start without printing any default content from the existing logs.

After each iteration, capture and verify the system time and execute various OS resource commands after 10 seconds interval (5 seconds sleep in-between each iteration and 4 seconds after processing the tail output – assuming that processing all tail command roughly take 1 sec, hence in all 10 seconds)

Various command I have used for extracting OS resources are:

I.  command1="vmstat | nl | tr -s '\\t '"
II. command2="sar -W 0"
III.    command3="top -b -n 1 | nl | tr -s '\\t '"
IV. command4="ls -1 /tmp | grep EXIT"

Search for respective command(?) in the script and go thru the while loop of it in the script to figure output processing of the respective command. Note I nave used ‘nl’ command for development / coding ease

Ultimately presence of /tmp/EXIT file on the box will make the script to exit after removing the same from the box

Below is my script - I have added comments as much as possible for self explanatory:

#Useage - awk -f script.awk
BEGIN {
    command0="tail -f -n 0 -s 5 /x/web/webserver/*/logs/error_log /x/web/webserver/service/*/logs/log"
    command1="vmstat | nl | tr -s '\\t '"
    command2="sar -W 0"
    command3="top -b -n 1 | nl | tr -s '\\t '"
    command4="ls -1 /tmp | grep EXIT"
    format = "%a %b %e %H:%M:%S %Z %Y"
    split("", details)
    split("", fields)
    split("", data)
    split("", values)
    start_time=0

    printf "\n>%s:\n\n", command0 #dummy print for debuggng command being executed

    while ( (command0 |& getline var0) > 0) { #get the command output
        if (start_time == 0) #if block to reset the start_time variable
        {
                start_time = systime() + 4
        }
        if (var0 ~ /==>.*<==/) { #if block to extract the file name from the tail output - outputted in '==>FileName<==' format
                gsub(/[=><]/, "", var0)
                len = split(var0, name, "/")
                if(len == 7) {file = name[5]} else {file = name[6]}
        }
        if (len == 7 && var0 ~ /[Ee]rror|[Ee]xception|ORA|[Ff]atal/) {  #extract the logs error statements
                print strftime(format,systime()) " | Error Log | " file " | Error :" var0
        }
        if(systime() >= start_time) #check if curernt system time is greater than start_time as computed above
        {
                start_time = 0 #reset the start_time variable and now execute the system resource command
                printf "\n>%s:\n\n", command1
                while ( (command1 |& getline) > 0) { #process output of first command
                if($1 <= 1)
                        continue #not needed for processing skip this one
                if($1 == 2) #capture the fieds name and skip to next line
                {
                        for (i = 1; i <= NF; i++){fields[$i] = i;}
                        continue
                }
                if ($1 == 3) #store the command data output in data array
                        split($0, data);
                print strftime(format,systime()) " | System Resource | System | Time spent running non-kernel code :" data[fields["us"]]
                print strftime(format,systime()) " | System Resource | System | Time spent running kernel code :" data[fields["sy"]]
                print strftime(format,systime()) " | System Resource | System | Amount of memory swapped in from disk :" data[fields["si"]]
                print strftime(format,systime()) " | System Resource | System | Amount of memory swapped to disk :" data[fields["so"]]
                }
                close(command1)

                printf "\n>%s:\n\n", command2 #start processing second command
                while ( (command2 |& getline) > 0) {
                        if ($4 ~ /[0-9]+[\.][0-9]+/) #check for 4th positional value if its format is of "int.intint" format
                        {
                                if( $4 > 0.0) #dummy check now to print if page swapping
                                        print strftime(format,systime()) " | System Resource | Disk | Page rate is > 0.0 reads/second: " $4
                        }
                }
                close(command2)

                printf "\n>%s:\n\n", command3 # start processing command number 3
                while ( (command3 |& getline ) > 0) {
                        if($1 == 1 && $0 ~ /load average:/) #get the load average from the output if this is the first line
                        {
                                split($0, arr, ",")
                                print strftime(format,systime())" | System Resource | System |" arr[4]
                        }
                        if($1 > 7 && $1 <= 12) # print first top 5 process that are consuming most of the CPUs time
                        {
                                f=split($0, arr, " ")
                                if(f == 13)
                                        print strftime(format,systime())" | System Resource | System | CPU% "arr[10]" Process No: "arr[1] - 7" Name: "arr[13]
                        }
                }
                close(command3)
                printf "\n>%s:\n\n", command4 #process command number 4 to check presence of file
                while ( (command4 |& getline var4) > 0) {
                        system("rm -rf /tmp/EXIT")
                        exit 0 #if file is there then remove the file and exit this script execution
                }
                close(command4)
        }
    }
        close(command0)
}

Output -:

>tail -f -n 0 -s 5 /x/web/webserver/*/logs/error_log /x/web/webserver/service/*/logs/log:


>vmstat | nl | tr -s '\t ':

Sun Dec 16 23:05:12 PST 2012 | System Resource | System | Time spent running non-kernel code :9
Sun Dec 16 23:05:12 PST 2012 | System Resource | System | Time spent running kernel code :9
Sun Dec 16 23:05:12 PST 2012 | System Resource | System | Amount of memory swapped in from disk :0
Sun Dec 16 23:05:12 PST 2012 | System Resource | System | Amount of memory swapped to disk :2

>sar -W 0:

Sun Dec 16 23:05:12 PST 2012 | System Resource | Disk | Page rate is > 0.0 reads/second: 3.89

>top -b -n 1 | nl | tr -s '\t ':

Sun Dec 16 23:05:13 PST 2012 | System Resource | System | load average: 3.63
Sun Dec 16 23:05:13 PST 2012 | System Resource | System | CPU% 12.0 Process No: 1 Name: occworker
Sun Dec 16 23:05:13 PST 2012 | System Resource | System | CPU% 10.3 Process No: 2 Name: occworker
Sun Dec 16 23:05:13 PST 2012 | System Resource | System | CPU% 6.9 Process No: 3 Name: caldaemon
Sun Dec 16 23:05:13 PST 2012 | System Resource | System | CPU% 6.9 Process No: 4 Name: occmux
Sun Dec 16 23:05:13 PST 2012 | System Resource | System | CPU% 6.9 Process No: 5 Name: top

>ls -1 /tmp | grep EXIT:

Answer 1

This is your second post that I can recall about using getline this way. I mentioned last time that it was the wrong approach but it looks like you didn't believe me so let me try one more time.

Your question of "how do I use awk to execute commands with getline to read their output?" is like asking "how do I use a drill to cut glass?". You could get an answer telling you to tape over the part of the glass where you'll be drilling to avoid fracturing it and that WOULD answer your question but the more useful answer would probably be - don't do that, use a glass cutter.

Using awk as a shell from which to call commands is 100% the wrong approach. Simply use the right tool for the right job. If you need to parse a text file, use awk. If you need to manipulate files or processes or invoke commands, use shell (or your OS equivalent).

Finally, please read http://awk.freeshell.org/AllAboutGetline and don't even think about using getline until you fully understand all the caveats.

EDIT: here's a shell script to do what your posted awk script does:

tail -f log |
while IFS= read -r var0; do
    ls
    date
done

Look simpler? Not saying it makes sense to do that, but if you did want to do it, THAT's the way to implement it, not in awk.

EDIT: here's how to write the first part of your awk script in shell (bash in this case), I ran out of enthusiasm for translating the rest of it for you and I think this shows you how to do the rest yourself:

format = "%a %b %e %H:%M:%S %Z %Y"
start_time=0

tail -f -n 0 -s 5 /x/web/webserver/*/logs/error_log /x/web/webserver/service/*/logs/log |
while IFS= read -r line; do

    systime=$(date +"%s")

    #block to reset the start_time variable
    if ((start_time == 0)); then
        start_time=(( systime + 4 ))
    fi

    #block to extract the file name from the tail output - outputted in '==>FileName<==' format
    case $var0 in
        "==>"*"<==" )
            path="${var0%% <==}"
            path="${path##==> }"
            name=( ${path//\// } )
            len="${#name[@]}"
            if ((len == 7)); then
                file=name[4]
            else
                file=name[5]
            fi
            ;;
    esac

    if ((len == 7)); then
        case $var0 in
            [Ee]rror|[Ee]xception|ORA|[Ff]atal )    #extract the logs error statements
                printf "%s | Error Log | %s | Error :%s\n" "$(date +"$format")" "$file" "$var0"
                ;;
        esac
    fi

    #check if curernt system time is greater than start_time as computed above
    if (( systime >= start_time )); then

        start_time=0 #reset the start_time variable and now execute the system resource command
        ....

Note that this would execute slightly faster than your awk script but that absolutely does not matter at all since your tail is taking 5 second breaks between iterations.

Also note that all I'm doing above is translating your awk script into shell, it doesn't necessarily mean it'd be the best way to write this tool from scratch.