How to wire up multiple programs reading/writing via STDIN/STDOUT in Golang concurrently?

Question

At a high level I would like to accomplish the following. Each box is a running program reading from STDIN and writing to STDOUT. I want to write a golang program which sets this up and runs it so that all production/consumption is happening in parallel. I am thinking of using io.Pipe, channels, and os.Exec etc.

                            +-----------+                                  
                            |  PROG-1   +-----------------------+          
                +---------> |           |                       v          
                |           +-----------+                                  
                |                                           +-------+      
    +-----------+                                           | DIFF  +----->
    | GENERATOR |                                           |       |      
    +-----------+                                           +---+---+      
                |                                               ^          
                |                                               |          
                |           +-----------+                       |          
                |           |           |                       |          
                +---------> |  PROG-2   +-----------------------+          
                            +-----------+                                  

Here's an attempt but it doesn't seem to be working reliably and also the "DIFF" part is not implemented.

package main

import (
    "io"
    "os"
    "os/exec"
)

const UPPER = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
const LOWER = "abcdefghijklmnopqrstuvwxyz"

func runProg(r io.Reader, cmd *exec.Cmd) {
    cmd.Stdin = r
    cmd.Stdout = os.Stdout // I want this to go to a third prog call "diff".
    cmd.Run()
}

func runIt(r io.Reader, prog1 *exec.Cmd, prog2 *exec.Cmd) {
    r1, w1 := io.Pipe()
    r2, w2 := io.Pipe()

    go runProg(r1, prog1)
    go runProg(r2, prog2)

    go func() {
        defer w1.Close()
        defer w2.Close()
        mw := io.MultiWriter(w1, w2)
        io.Copy(mw, r)
    }()

}

func main() {
    generator := exec.Command("ls", "-l")
    r, w := io.Pipe()
    generator.Stdout = w

    prog1 := exec.Command("tr", LOWER, UPPER)
    prog2 := exec.Command("tr", UPPER, LOWER)

    runIt(r, prog1, prog2)

    generator.Run()

}

Answer 1

There are a couple things here. You're adding work and complexity in creating all those pipes. Also, running the command concurrently is built-in using Cmd.Start() and Cmd.Wait().

package main

import (
        "fmt"
        "io"
        "os"
        "os/exec"
)

const UPPER = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
const LOWER = "abcdefghijklmnopqrstuvwxyz"

func runProg(cmd *exec.Cmd) (w io.WriteCloser, err error) {
        w, err := cmd.StdinPipe()
        if err != nil {
                fmt.Println(err)
        }
        cmd.Stdout = os.Stdout
        err = cmd.Start()
}



func runIt(r io.Reader, prog1 *exec.Cmd, prog2 *exec.Cmd) {

        w1, err := runProg(prog1)
        if err != nil {
                fmt.Println(err)
        }
        w2, err := runProg(prog2)
        if err != nil {
                fmt.Println(err)
        }

        go func() {
                defer w1.Close()
                defer w2.Close()
                mw := io.MultiWriter(w1, w2)
                io.Copy(mw, r)
        }()

}

func main() {
        generator := exec.Command("ls", "-l")
        r, err := generator.StdoutPipe()
        if err != nil {
                fmt.Println(err)
        }

        prog1 := exec.Command("tr", LOWER, UPPER)
        prog2 := exec.Command("tr", UPPER, LOWER)

        runIt(r, prog1, prog2)

        generator.Run()

        err = prog1.Wait()
        err1 := prog2.Wait()
        if err != nil || err1 != nil {
                fmt.Println(err, err1)
        }
}