How do you construct a read-write pipe with lua?

Question

I'd like to do the equivalent of:

foo=$(echo "$foo"|someprogram)

within lua -- ie, I've got a variable containing a bunch of text, and I'd like to run it through a filter (implemented in python as it happens).

Any hints?

Added: would really like to do this without using a temporary file

Answer 1

I stumbled on this post while trying to do the same thing and never found a good solution, see the code below for how I solved my issues. This implementation allows users to access stdin, stdout, stderr and get the return status code. A simple wrapper is called for simple pipe calls.

Brief note: popen uses fork under the hood. Fork's documentation states "The child inherits copies of the parent's set of open file descriptors", so what we are doing is creating new file descriptors for the new process to inherit.

require("posix")
 
--
-- Simple popen3() implementation
--
function popen3(path, ...)
    local r1, w1 = posix.pipe()
    local r2, w2 = posix.pipe()
    local r3, w3 = posix.pipe()
 
    assert((r1 ~= nil or r2 ~= nil or r3 ~= nil), "pipe() failed")
 
    local pid, err = posix.fork()
    assert(pid ~= nil, "fork() failed")
    if pid == 0 then
        posix.close(w1)
        posix.close(r2)
        posix.dup2(r1, posix.fileno(io.stdin))
        posix.dup2(w2, posix.fileno(io.stdout))
        posix.dup2(w3, posix.fileno(io.stderr))
        posix.close(r1)
        posix.close(w2)
        posix.close(w3)
 
        local ret, err = posix.execp(path, unpack({...}))
        assert(ret ~= nil, "execp() failed")
 
        posix._exit(1)
        return
    end
 
    posix.close(r1)
    posix.close(w2)
    posix.close(w3)
 
    return pid, w1, r2, r3
end
 
--
-- Pipe input into cmd + optional arguments and wait for completion
-- and then return status code, stdout and stderr from cmd.
--
function pipe_simple(input, cmd, ...)
    --
    -- Launch child process
    --
    local pid, w, r, e = popen3(cmd, unpack({...}))
    assert(pid ~= nil, "filter() unable to popen3()")
 
    --
    -- Write to popen3's stdin, important to close it as some (most?) proccess
    -- block until the stdin pipe is closed
    --
    posix.write(w, input)
    posix.close(w)
 
    local bufsize = 4096
    --
    -- Read popen3's stdout via Posix file handle
    --
    local stdout = {}
    local i = 1
    while true do
        buf = posix.read(r, bufsize)
        if buf == nil or #buf == 0 then break end
        stdout[i] = buf
        i = i + 1
    end
    
    --
    -- Read popen3's stderr via Posix file handle
    --
    local stderr = {}
    local i = 1
    while true do
        buf = posix.read(e, bufsize)
        if buf == nil or #buf == 0 then break end
        stderr[i] = buf
        i = i + 1
    end
 
    --
    -- Clean-up child (no zombies) and get return status
    --
    local wait_pid, wait_cause, wait_status = posix.wait(pid)
 
    return wait_status, table.concat(stdout), table.concat(stderr)
end
 
--
-- Example usage
--
local my_in = io.stdin:read("*all")
--local my_cmd = "wc"
--local my_args = {"-l"}
local my_cmd = "spamc"
local my_args = {} -- no arguments
local my_status, my_out, my_err = pipe_simple(my_in, my_cmd, unpack(my_args))
 
-- Obviously not interleaved as they would have been if printed in realtime
io.stdout:write(my_out)
io.stderr:write(my_err)
 
os.exit(my_status)

Answer 2

For a system I have running Lua 5.1 and luaposix 35.0-1, I started with the solution from Anthony Towns from this current page and made it work for this luaposix version for the purpose of calling openssl for encryption on a system without any other encryption capabilities. I have attempted to make the code more explicit in order to allow others to handle any potential API changes in luaposix in the future.

local posix = require('posix');
require('os');
require('io');

local function getOutputFromProcessProvidedInput( dataForProcess, command, commandArguments )
        local MAXIMUM_BYTE_READ_COUNT = 100;
        local readFileHandle1,writeFileHandle1 = posix.pipe()
        io.flush();
        local childProcessId1 = posix.fork();
        if (childProcessId1 == 0)
        then
                posix.close( readFileHandle1 );
                posix.write( writeFileHandle1, dataForProcess );
                io.flush();
                os.exit( 1 );
        end
        posix.close( writeFileHandle1 );

        local readFileHandle2,writeFileHandle2 = posix.pipe();
        io.flush();
        local childProcessId2 = posix.fork();
        if (childProcessId2 == 0)
        then
                posix.close( readFileHandle2 );
                posix.dup2( readFileHandle1, posix.fileno( io.stdin ) );
                posix.dup2( writeFileHandle2, posix.fileno( io.stdout ) );
                posix.execp( command, commandArguments );
                os.exit( 2 );
        end
        posix.close( writeFileHandle2 );
        posix.close( readFileHandle1 );

        local dataFromProcess = posix.read( readFileHandle2, MAXIMUM_BYTE_READ_COUNT );
        posix.close( readFileHandle2 );

        posix.wait( childProcessId2 );
        posix.wait( childProcessId1 );

        return dataFromProcess;
end

-- Command being executed
-- echo -n AAAAAAAAAAAAAAAA | openssl aes-128-cbc -e -nopad -a -K 30313233343536373839616263646566 -iv 1FF1ECB9000000000000000000000000
-- Expected result
-- 28iudIC31lHfDDxfa1/g9w==
result = openReadWritePipe("AAAAAAAAAAAAAAAA","openssl",{"aes-128-cbc", "-e", "-nopad", "-a", "-K", "30313233343536373839616263646566", "-iv",  "1FF1ECB9000000000000000000000000"});
print("Result: "..result);

Answer 3

It's easy, no extensions necessary (tested with lua 5.3).

#!/usr/bin/lua
-- use always locals
local stdin = io.stdin:lines()
local stdout = io.write

for line in stdin do
    stdout (line)
end 

save as inout.lua and do chmod +x /tmp/inout.lua

20:30 $ foo=$(echo "bla"|  /tmp/inout.lua)
20:30 $ echo $foo
bla

Answer 4

Here is how I solved the problem, it require lua posix.

          p = require 'posix'
          local r,w = p.pipe()
          local r1,w1 = p.pipe()
          local cpid = p.fork()
          if cpid == 0 then -- child reads from pipe                                     
             w:close()
             r1:close()
             p.dup(r, io.stdin)
             p.dup(w1 ,io.stdout)
             p.exec('./myProgram')
             r:close()
             w1:close()
             p._exit(0)
          else -- parent writes to pipe                                                  
             IN = r1
             OUT = w
          end

During myProgram execution, you'l read and write from normal io and after this part of code you just have to write/read on IN and OUT to comunicate with child program.

Answer 5

As long as your Lua supports io.popen, this problem is easy. The solution is exactly as you have outlined, except instead of $(...) you need a function like this one:

function os.capture(cmd, raw)
  local f = assert(io.popen(cmd, 'r'))
  local s = assert(f:read('*a'))
  f:close()
  if raw then return s end
  s = string.gsub(s, '^%s+', '')
  s = string.gsub(s, '%s+$', '')
  s = string.gsub(s, '[\n\r]+', ' ')
  return s
end

You can then call

local foo = ...
local cmd = ("echo $foo | someprogram"):gsub('$foo', foo)
foo = os.capture(cmd)

I do stuff like this all the time. Here's a related useful function for forming commands:

local quote_me = '[^%w%+%-%=%@%_%/]' -- complement (needn't quote)
local strfind = string.find

function os.quote(s)
  if strfind(s, quote_me) or s == '' then
    return "'" .. string.gsub(s, "'", [['"'"']]) .. "'"
  else
    return s
  end
end

Answer 6

Aha, a possibly better solution:

require('posix')
require('os')
require('io')

function splat_popen(data,cmd)
   rd,wr = posix.pipe()
   io.flush()
   child = posix.fork()
   if child == 0 then
      rd:close()
      wr:write(data)
      io.flush()
      os.exit(1)
   end
   wr:close()

   rd2,wr2 = posix.pipe()
   io.flush()
   child2 = posix.fork()
   if child2 == 0 then
      rd2:close()
      posix.dup(rd,io.stdin)
      posix.dup(wr2,io.stdout)
      posix.exec(cmd)
      os.exit(2)
   end
   wr2:close()
   rd:close()

   y = rd2:read("*a")
   rd2:close()

   posix.wait(child2)
   posix.wait(child)

   return y
end

munged=splat_popen("hello, world","/usr/games/rot13")
print("munged: "..munged.." !")

Answer 7

A not very nice solution that avoids a temporary file...

require("io")
require("posix")

x="hello\nworld"

posix.setenv("LUA_X",x)
i=popen('echo "$LUA_X" | myfilter')
x=i.read("*a")

Answer 8

There is nothing in the Lua standard library to allow this.

Here is an in-depth exploration of the difficulties of doing bidirectional communication properly, and a proposed solution:

if possible, redirect one end of the stream (input or output) to a file. I.e.:

fp = io.popen("foo >/tmp/unique", "w")
fp:write(anything)
fp:close()
fp = io.open("/tmp/unique")
x = read("*a")
fp:close()

You may be interested in this extension which adds functions to the os and io namespaces to make bidirectional communication with a subprocess possible.