big awesome update for devfs, command substitution, and io redirection

Added another 60 tests, now totalling 901 all-passing tests. This change also reduces memory cost for boot by about 1300 bytes.

This change should be fully compatible with any 1.6 user scripts. This makes no change to api, only improves shell support and makes important process handle cleanup (when pipes and redirection is used - we are not auto-closing user file handles).

src/main/resources/assets/opencomputers/loot/OpenOS/boot/03_io.lua

@@ -33,14 +33,19 @@ function stdoutStream:write(str)
 end
 
 function stderrStream:write(str)
-  local component = require("component")
+  local gpu = term.gpu()
-  if component.isAvailable("gpu") and component.gpu.getDepth() and component.gpu.getDepth() > 1 then
+  local set_depth = gpu and gpu.getDepth() and gpu.getDepth() > 1
-    local foreground = component.gpu.setForeground(0xFF0000)
+
-    term.write(str, true)
+  if set_depth then
-    component.gpu.setForeground(foreground)
+    set_depth = gpu.setForeground(0xFF0000)
-  else
-    term.write(str, true)
   end
+    
+  term.drawText(str, true)
+
+  if set_depth then
+    gpu.setForeground(set_depth)
+  end
+
   return self
 end
 

src/main/resources/assets/opencomputers/loot/OpenOS/boot/10_devfs.lua

@@ -0,0 +1,7 @@
+require("filesystem").mount(
+setmetatable({
+  isReadOnly = function()return true end
+},
+{
+  __index=function(tbl,key)return require("devfs")[key]end
+}), "/dev")

src/main/resources/assets/opencomputers/loot/OpenOS/lib/devfs.lua

@@ -0,0 +1,87 @@
+local fs = require("filesystem")
+
+local proxy = {points={},address=require("guid").next()}
+
+local nop = function()end
+
+function proxy.getLabel()
+  return "devfs"
+end
+
+function proxy.list()
+  local keys = {}
+  for k,v in pairs(proxy.points) do
+    table.insert(keys, k)
+  end
+  return keys
+end
+
+function proxy.exists(path)
+  return not not proxy.points[path]
+end
+
+function proxy.remove(path)
+  if not proxy.exists(path) then return false end
+  proxy.points[path] = nil
+  return true
+end
+
+function proxy.isDirectory(path)
+  return false
+end
+
+function proxy.size(path)
+  return 0
+end
+
+function proxy.lastModified(path)
+  return fs.lastModified("/dev/")
+end
+
+function proxy.read(h,...)
+  return h:read(...)
+end
+
+function proxy.write(h,...)
+  return h:write(...)
+end
+
+proxy.close = nop
+
+function proxy.open(path, mode)
+  checkArg(1, path, "string")
+
+  local handle = proxy.points[path]
+  if not handle then return nil, "device point [" .. path .. "] does not exist" end
+
+  local msg = "device point [" .. path .. "] cannot be opened for "
+
+  if mode == "r" then
+    if not handle.read then
+      return nil, msg .. "read"
+    end
+  else
+    if not handle.write then
+      return nil, msg .. "write"
+    end
+  end
+
+  return handle
+end
+
+function proxy.create(path, handle)
+  handle.close = handle.close or nop
+  proxy.points[path] = handle
+  return true
+end
+
+proxy.create("null", {write = nop})
+proxy.create("random", {read = function(_,n)
+  local chars = {}
+  for i=1,n do
+    table.insert(chars,string.char(math.random(0,255)))
+  end
+  return table.concat(chars)
+end})
+
+return proxy

src/main/resources/assets/opencomputers/loot/OpenOS/lib/pipes.lua

@@ -15,16 +15,14 @@ function pipeStream.new(pm)
   return setmetatable(stream, metatable)
 end
 function pipeStream:resume()
-  local yield_args = table.pack(self.pm.pco.resume_all(table.unpack(self.pm.args)))
+  local yield_args = table.pack(self.pm.pco.resume_all())
   if not yield_args[1] then
-    self.pm.args = {false}
     self.pm.dead = true
 
     if not yield_args[1] and yield_args[2] then
       io.stderr:write(tostring(yield_args[2]) .. "\n")
     end
   end
-  self.pm.args = {true}
   return table.unpack(yield_args)
 end
 function pipeStream:close()
@@ -134,7 +132,6 @@ function plib.internal.create(fp)
   local pco = setmetatable(
   {
     stack = {},
-    args = {},
     next = nil,
     create = _co.create,
     wrap = _co.wrap,
@@ -318,7 +315,7 @@ function pipeManager.new(prog, mode, env)
   end
 
   local pm = setmetatable(
-    {dead=false,closed=false,args={},prog=prog,mode=mode,env=env},
+    {dead=false,closed=false,prog=prog,mode=mode,env=env},
     {__index=pipeManager}
   )
   pm.prog_id = pm.mode == "r" and 1 or 2
@@ -328,37 +325,29 @@ function pipeManager.new(prog, mode, env)
     function()pm.dead=true end
 
   pm.commands = {}
-  pm.commands[pm.prog_id] = {shellPath, sh.internal.buildCommandRedirects({})}
+  pm.commands[pm.prog_id] = {shellPath, {}}
-  pm.commands[pm.self_id] = {pm.handler, sh.internal.buildCommandRedirects({})}
+  pm.commands[pm.self_id] = {pm.handler, {}}
 
   pm.root = function()
+    local startup_args = {}
+
     local reason
-    pm.threads, reason, pm.inputs, pm.outputs = 
+    pm.threads, reason = sh.internal.createThreads(pm.commands, {}, pm.env)
-      sh.internal.buildPipeStream(pm.commands, pm.env)
 
     if not pm.threads then
       pm.dead = true
-      return false, reason -- 2nd return is reason, not pipes, on error :)
+      return false, reason
     end
-    pm.pipe = reason[1] -- an array of pipes of length 1
 
-    local startup_args = {}
+    pm.pipe = process.info(pm.threads[1]).data.io[1]
+    process.info(pm.threads[pm.prog_id]).data.args = {pm.env,pm.prog}
+    
     -- if we are the writer, we need args to resume prog
     if pm.mode == "w" then
-      pm.pipe.stream.args = {pm.env,pm.prog,n=2}
+      pm.pipe.stream.redirect[0] = plib.internal.redirectRead(pm)
-      startup_args = {true,n=1}
-      -- also, if we are the writer, we need to intercept the reader
-      pm.pipe.stream.redirect.read = plib.internal.redirectRead(pm)
-    else
-      startup_args = {true,pm.env,pm.prog,n=3}
     end
 
-    return sh.internal.executePipeStream(
+    return sh.internal.runThreads(pm.threads)
-      pm.threads,
-      {pm.pipe},
-      pm.inputs,
-      pm.outputs,
-      startup_args)
   end
 
   return pm

src/main/resources/assets/opencomputers/loot/OpenOS/lib/process.lua

@@ -81,7 +81,7 @@ function process.load(path, env, init, name)
         return string.format('%s:\n%s', msg or '', stack)
       end, ...)
     }
-    process.list[thread] = nil
+    process.internal.close(thread)
     if not result[1] then
       -- msg can be a custom error object
       local msg = result[2]
@@ -100,6 +100,7 @@ function process.load(path, env, init, name)
     env = env,
     data = setmetatable(
     {
+      handles = {},
       io = setmetatable({}, {__index=p and p.data and p.data.io or nil}),
       coroutine_handler = setmetatable({}, {__index=p and p.data and p.data.coroutine_handler or nil}),
     }, {__index=p and p.data or nil}),
@@ -133,4 +134,16 @@ function process.info(levelOrThread)
   end
 end
 
+--table of undocumented api subject to change and intended for internal use
+process.internal = {}
+--this is a future stub for a more complete method to kill a process
+function process.internal.close(thread)
+  checkArg(1,thread,"thread")
+  local pdata = process.info(thread).data
+  for k,v in pairs(pdata.handles) do
+    v:close()
+  end
+  process.list[thread] = nil
+end
+
 return process

src/main/resources/assets/opencomputers/loot/OpenOS/lib/sh.lua

@@ -8,13 +8,7 @@ local tx = require("transforms")
 local unicode = require("unicode")
 
 local sh = {}
-
+sh.internal = {}
-sh.internal = setmetatable({},
-{
-  __tostring=function()
-    return "table of undocumented api subject to change and intended for internal use"
-  end
-})
 
 -- --[[@@]] are not just comments, but custom annotations for delayload methods.
 -- See package.lua and the api wiki for more information
@@ -133,29 +127,36 @@ function sh.internal.isIdentifier(key)
 end
 
 function sh.expand(value)
-  return value
+  local expanded = value
   :gsub("%$([_%w%?]+)", function(key)
     if key == "?" then
       return tostring(sh.getLastExitCode())
     end
-    return os.getenv(key) or '' end)
+    return os.getenv(key) or ''
+  end)
   :gsub("%${(.*)}", function(key)
     if sh.internal.isIdentifier(key) then
       return sh.internal.expandKey(key)
     end
     error("${" .. key .. "}: bad substitution")
   end)
+  if expanded:find('`') then
+    expanded = sh.internal.parse_sub(expanded)
+  end
+  return expanded
 end
 
 function sh.internal.expand(word)
   if #word == 0 then return {} end
-
   local result = ''
   for i=1,#word do
     local part = word[i]
-    result = result .. (not (part.qr and part.qr[3]) and sh.expand(part.txt) or part.txt)
+    -- sh.expand runs command substitution on backticks
+    -- if the entire quoted area is backtick quoted, then
+    -- we can save some checks by adding them back in
+    local q = part.qr and part.qr[1] == '`' and '`' or ''
+    result = result .. (not (part.qr and part.qr[3]) and sh.expand(q..part.txt..q) or part.txt)
   end
-
   return {result}
 end
 
@@ -205,49 +206,6 @@ function sh.hintHandler(full_line, cursor)
   return sh.internal.hintHandlerImpl(full_line, cursor)
 end
 
-function sh.internal.buildCommandRedirects(args)
-  local input, output, mode = nil, nil, "write"
-  local tokens = args
-  args = {}
-  local function smt(call) -- state metatable factory
-    local function index(_, token)
-      if token == "<" or token == ">" or token == ">>" then
-        return "parse error near " .. token
-      end
-      call(token)
-      return "args" -- default, return to normal arg parsing
-    end
-    return {__index=index}
-  end
-  local sm = { -- state machine for redirect parsing
-    args   = setmetatable({["<"]="input", [">"]="output", [">>"]="append"},
-                              smt(function(token)
-                                    table.insert(args, token)
-                                  end)),
-    input  = setmetatable({}, smt(function(token)
-                                    input = token
-                                  end)),
-    output = setmetatable({}, smt(function(token)
-                                    output = token
-                                    mode = "write"
-                                  end)),
-    append = setmetatable({}, smt(function(token)
-                                    output = token
-                                    mode = "append"
-                                  end))
-  }
-  -- Run state machine over tokens.
-  local state = "args"
-  for i = 1, #tokens do
-    local token = tokens[i]
-    state = sm[state][token]
-    if not sm[state] then
-      return nil, state
-    end
-  end
-  return args, input, output, mode
-end
-
 function sh.internal.parseCommand(words)
   checkArg(1, words, "table")
   if #words == 0 then
@@ -263,10 +221,11 @@ function sh.internal.parseCommand(words)
   if not program then
     return nil, evaluated_words[1] .. ": " .. reason
   end
-  return program, sh.internal.buildCommandRedirects(tx.sub(evaluated_words, 2))
+  evaluated_words = tx.sub(evaluated_words, 2)
+  return program, evaluated_words
 end
 
-function sh.internal.buildPipeStream(commands, env)
+function sh.internal.createThreads(commands, eargs, env)
   -- Piping data between programs works like so:
   -- program1 gets its output replaced with our custom stream.
   -- program2 gets its input replaced with our custom stream.
@@ -274,109 +233,70 @@ function sh.internal.buildPipeStream(commands, env)
   -- custom stream triggers execution of "next" program after write.
   -- custom stream triggers yield before read if buffer is empty.
   -- custom stream may have "redirect" entries for fallback/duplication.
-  local threads, pipes, inputs, outputs = {}, {}, {}, {}
+  local threads = {}
   for i = 1, #commands do
-    local program, args, input, output, mode = table.unpack(commands[i])
+    local program, args = table.unpack(commands[i])
-    local process_name = tostring(program)
+    local name, thread = tostring(program)
-    local reason
     local thread_env = type(program) == "string" and env or nil
-    threads[i], reason = process.load(program, thread_env, function()
+    local thread, reason = process.load(program, thread_env, function()
-      os.setenv("_", program)
+      os.setenv("_", name)
-      if input then
+      -- popen expects each process to first write an empty string
-        local file, reason = io.open(shell.resolve(input))
+      -- this is required for proper thread order
-        if not file then
+      io.write('')
-          error("could not open '" .. input .. "': " .. reason, 0)
+    end, name)
-        end
+
-        table.insert(inputs, file)
+    threads[i] = thread
-        if pipes[i - 1] then
+    
-          pipes[i - 1].stream.redirect.read = file
+    if thread then
-          io.input(pipes[i - 1])
+      -- smart check if ios should be loaded
-        else
+      if tx.first(args, function(token) return token == "<" or token:find(">") end) then
-          io.input(file)
+        args, reason = sh.internal.buildCommandRedirects(thread, args)
-        end
-      elseif pipes[i - 1] then
-        io.input(pipes[i - 1])
       end
-      if output then
+    end
-        local file, reason = io.open(shell.resolve(output), mode == "append" and "a" or "w")
+    
-        if not file then
+    if not args or not thread then
-          error("could not open '" .. output .. "': " .. reason, 0)
+      for i,t in ipairs(threads) do
-        end
+        process.internal.close(t)
-        table.insert(outputs, file)
-        if pipes[i] then
-          pipes[i].stream.redirect.write = file
-          io.output(pipes[i])
-        else
-          io.output(file)
-        end
-      elseif pipes[i] then
-        io.output(pipes[i])
       end
-    io.write('')
+      return nil, reason
-    end, process_name)
-    if not threads[i] then
-      return false, reason
     end
 
-    if i < #commands then
+    process.info(thread).data.args = tx.concat(args, eargs or {})
-      pipes[i] = require("buffer").new("rw", sh.internal.newMemoryStream())
-      pipes[i]:setvbuf("no")
-    end
-    if i > 1 then
-      pipes[i - 1].stream.next = threads[i]
-      pipes[i - 1].stream.args = args
-    end
   end
-  return threads, pipes, inputs, outputs
+
+  if #threads > 1 then
+    sh.internal.buildPipeChain(threads)
+  end
+
+  return threads
 end
 
-function sh.internal.executePipeStream(threads, pipes, inputs, outputs, args)
+function sh.internal.runThreads(threads)
   local result = {}
   for i = 1, #threads do
     -- Emulate CC behavior by making yields a filtered event.pull()
-    while args[1] and coroutine.status(threads[i]) ~= "dead" do
+    local thread, args = threads[i]
-      result = table.pack(coroutine.resume(threads[i], table.unpack(args, 2, args.n)))
+    while coroutine.status(thread) ~= "dead" do
-      if coroutine.status(threads[i]) ~= "dead" then
+      args = args or process.info(thread).data.args
-        local action = result[2]
+      result = table.pack(coroutine.resume(thread, table.unpack(args)))
-        if action == nil or type(action) == "number" then
+      if coroutine.status(thread) ~= "dead" then
-          args = table.pack(pcall(event.pull, table.unpack(result, 2, result.n)))
+        args = sh.internal.handleThreadYield(result)
-        else
-          args = table.pack(coroutine.yield(table.unpack(result, 2, result.n)))
-        end
         -- in case this was the end of the line, args is returned
         result = args
+        if table.remove(args, 1) then
+          break
+        end
       end
     end
-    if pipes[i] then
-      pcall(pipes[i].close, pipes[i])
-    end
     if not result[1] then
-      if type(result[2]) == "table" and result[2].reason == "terminated" then
+      sh.internal.handleThreadCrash(thread, result)
-        if result[2].code then
-          result[1] = true
-          result.n = 1
-        else
-          result[2] = "terminated"
-        end
-      elseif type(result[2]) == "string" then
-        result[2] = debug.traceback(threads[i], result[2])
-      end
       break
     end
   end
-  for _, input in ipairs(inputs) do input:close() end
-  for _, output in ipairs(outputs) do output:close() end
   return table.unpack(result)
 end
 
-function sh.internal.executeStatement(env, commands, eargs)
+function sh.internal.executePipes(pipe_parts, eargs, env)
-  local threads, pipes, inputs, outputs = sh.internal.buildPipeStream(commands, env)
-  if not threads then return false, pipes end
-  local args = tx.concat({true,n=1},commands[1][2] or {}, eargs)
-  return sh.internal.executePipeStream(threads, pipes, inputs, outputs, args)
-end
-
-function sh.internal.executePipes(pipe_parts, eargs)
   local commands = {}
   for i=1,#pipe_parts do
     commands[i] = table.pack(sh.internal.parseCommand(pipe_parts[i]))
@@ -388,9 +308,14 @@ function sh.internal.executePipes(pipe_parts, eargs)
       return sh.internal.ec.parseCommand
     end
   end
-  local result = table.pack(sh.internal.executeStatement(env,commands,eargs))
+  local threads, reason = sh.internal.createThreads(commands, eargs, env)  
-  local cmd_result = result[2]
+  if not threads then
-  if not result[1] then
+    io.stderr:write(reason,"\n")
+    return false
+  end
+  local result, cmd_result = sh.internal.runThreads(threads)
+
+  if not result then
     if cmd_result then
       if type(cmd_result) == "string" then
         cmd_result = cmd_result:gsub("^/lib/process%.lua:%d+: /", '/')
@@ -404,7 +329,6 @@ end
 
 function sh.execute(env, command, ...)
   checkArg(2, command, "string")
-  local eargs = {...}
   if command:find("^%s*#") then return true, 0 end
   local statements, reason = sh.internal.statements(command)
   if not statements or statements == true then
@@ -413,15 +337,105 @@ function sh.execute(env, command, ...)
     return true, 0
   end
 
+  local eargs = {...}
+
   -- simple
   if reason then
-    sh.internal.ec.last = sh.internal.command_result_as_code(sh.internal.executePipes(statements,eargs))
+    sh.internal.ec.last = sh.internal.command_result_as_code(sh.internal.executePipes(statements, eargs, env))
     return true
   end
 
-  return sh.internal.execute_complex(statements)
+  return sh.internal.execute_complex(statements, eargs, env)
 end
 
+function --[[@delayloaded-start@]] sh.internal.handleThreadYield(result)
+  local action = result[2]
+  if action == nil or type(action) == "number" then
+    return table.pack(pcall(event.pull, table.unpack(result, 2, result.n)))
+  else
+    return table.pack(coroutine.yield(table.unpack(result, 2, result.n)))
+  end
+end --[[@delayloaded-end@]]
+
+function --[[@delayloaded-start@]] sh.internal.handleThreadCrash(thread, result)
+  if type(result[2]) == "table" and result[2].reason == "terminated" then
+    if result[2].code then
+      result[1] = true
+      result.n = 1
+    else
+      result[2] = "terminated"
+    end
+  elseif type(result[2]) == "string" then
+    result[2] = debug.traceback(thread, result[2])
+  end
+end --[[@delayloaded-end@]]
+
+function --[[@delayloaded-start@]] sh.internal.buildCommandRedirects(thread, args)
+  local data = process.info(thread).data
+  local tokens, ios, handles = args, data.io, data.handles
+  args = {}
+  local from_io, to_io, mode
+  for i = 1, #tokens do
+    local token = tokens[i]
+    if token == "<" then
+      from_io = 0
+      mode = "r"
+    else
+      local first_index, last_index, from_io_txt, mode_txt, to_io_txt = token:find("(%d*)(>>?)(.*)")
+      if mode_txt then
+        mode = mode_txt == ">>" and "a" or "w"
+        from_io = from_io_txt and tonumber(from_io_txt) or 1
+        if to_io_txt ~= "" then
+          to_io = tonumber(to_io_txt:sub(2))
+          ios[from_io] = ios[to_io]
+          mode = nil
+        end
+      else -- just an arg
+        if not mode then
+          table.insert(args, token)
+        else
+          local file, reason = io.open(shell.resolve(token), mode)
+          if not file then
+            return nil, "could not open '" .. token .. "': " .. reason
+          end
+          table.insert(handles, file)
+          ios[from_io] = file
+        end
+        mode = nil
+      end
+    end
+  end
+
+  return args
+end --[[@delayloaded-end@]]
+
+function --[[@delayloaded-start@]] sh.internal.buildPipeChain(threads)
+  local prev_pipe
+  for i=1,#threads do
+    local thread = threads[i]
+    local data = process.info(thread).data
+    local pio = data.io
+
+    local pipe
+    if i < #threads then
+      pipe = require("buffer").new("rw", sh.internal.newMemoryStream())
+      pipe:setvbuf("no")
+      pipe.stream.redirect[1] = rawget(pio, 1)
+      pio[1] = pipe
+      table.insert(data.handles, pipe)
+    end
+
+    if prev_pipe then
+      prev_pipe.stream.redirect[0] = rawget(pio, 0)
+      prev_pipe.stream.next = thread
+      pio[0] = prev_pipe
+    end
+
+    prev_pipe = pipe
+  end
+
+end --[[@delayloaded-end@]]
+
 function --[[@delayloaded-start@]] sh.internal.glob(glob_pattern)
   local segments = text.split(glob_pattern, {"/"}, true)
   local hiddens = tx.select(segments,function(e)return e:match("^%%%.")==nil end)
@@ -549,7 +563,7 @@ function --[[@delayloaded-start@]] sh.internal.hintHandlerImpl(full_line, cursor
     return {}
   end
   local result
-  local prefix, partial = line:match("^(.*=)(.*)$")
+  local prefix, partial = line:match("^(.*[=><])(.*)$")
   if not prefix then prefix, partial = line:match("^(.+%s+)(.*)$") end
   local partialPrefix = (partial or line)
   local name = partialPrefix:gsub(".*/", "")
@@ -583,10 +597,11 @@ function --[[@delayloaded-start@]] sh.internal.hasValidPiping(words, pipes)
     return true
   end
 
-  pipes = pipes or tx.sub(text.syntax, 2) -- first text syntax is ; which CAN be repeated
+  local semi_split = tx.find(text.syntax, {";"}) -- all symbols before ; in syntax CAN be repeated
+  pipes = pipes or tx.sub(text.syntax, semi_split + 1)
 
-  local pies = tx.select(words, function(parts, i, t)
+  local pies = tx.select(words, function(parts, i)
-    return (#parts == 1 and tx.first(pipes, {{parts[1].txt}}) and true or false), i
+    return #parts == 1 and #text.split(parts[1].txt, pipes, true) == 0 and true or false
   end)
 
   local bad_pipe
@@ -718,6 +733,7 @@ function --[[@delayloaded-start@]] sh.internal.newMemoryStream()
 
   function memoryStream:close()
     self.closed = true
+    self.redirect = {}
   end
 
   function memoryStream:seek()
@@ -728,12 +744,12 @@ function --[[@delayloaded-start@]] sh.internal.newMemoryStream()
     if self.closed then
       return nil -- eof
     end
-    if self.redirect.read then
+    if self.redirect[0] then
       -- popen could be using this code path
       -- if that is the case, it is important to leave stream.buffer alone
-      return self.redirect.read:read(n)
+      return self.redirect[0]:read(n)
     elseif self.buffer == "" then
-      self.args = table.pack(coroutine.yield(table.unpack(self.result)))
+      process.info(self.next).data.args = table.pack(coroutine.yield(table.unpack(self.result)))
     end
     local result = string.sub(self.buffer, 1, n)
     self.buffer = string.sub(self.buffer, n + 1)
@@ -741,16 +757,17 @@ function --[[@delayloaded-start@]] sh.internal.newMemoryStream()
   end
 
   function memoryStream:write(value)
-    if not self.redirect.write and self.closed then
+    if not self.redirect[1] and self.closed then
       -- if next is dead, ignore all writes
       if coroutine.status(self.next) ~= "dead" then
         error("attempt to use a closed stream")
       end
-    elseif self.redirect.write then
+    elseif self.redirect[1] then
-      return self.redirect.write:write(value)
+      return self.redirect[1]:write(value)
     elseif not self.closed then
       self.buffer = self.buffer .. value
-      self.result = table.pack(coroutine.resume(self.next, table.unpack(self.args)))
+      local args = process.info(self.next).data.args
+      self.result = table.pack(coroutine.resume(self.next, table.unpack(args)))
       if coroutine.status(self.next) == "dead" then
         self:close()
       end
@@ -764,26 +781,52 @@ function --[[@delayloaded-start@]] sh.internal.newMemoryStream()
   end
 
   local stream = {closed = false, buffer = "",
-                  redirect = {}, result = {}, args = {}}
+                  redirect = {}, result = {}}
   local metatable = {__index = memoryStream,
                      __metatable = "memorystream"}
   return setmetatable(stream, metatable)
 end --[[@delayloaded-end@]]
 
-function --[[@delayloaded-start@]] sh.internal.execute_complex(statements)
+function --[[@delayloaded-start@]] sh.internal.execute_complex(statements, eargs, env)
   for si=1,#statements do local s = statements[si]
     local chains = sh.internal.groupChains(s)
-    local last_code,br = sh.internal.boolean_executor(chains, function(chain, chain_index)
+    local last_code = sh.internal.boolean_executor(chains, function(chain, chain_index)
       local pipe_parts = sh.internal.splitChains(chain)
-      return sh.internal.executePipes(pipe_parts,
+      local next_args = chain_index == #chains and si == #statements and eargs or {}
-        chain_index == #chains and si == #statements and eargs or {})
+      return sh.internal.executePipes(pipe_parts, next_args, env)
     end)
-    if br then
-      io.stderr:write(br,"\n")
-    end
     sh.internal.ec.last = sh.internal.command_result_as_code(last_code)
   end
-  return true, br
+  return true
+end --[[@delayloaded-end@]]
+
+
+function --[[@delayloaded-start@]] sh.internal.parse_sub(input)
+  -- cannot use gsub here becuase it is a [C] call, and io.popen needs to yield at times
+  local packed = {}
+  -- not using for i... because i can skip ahead
+  local i, len = 1, #input
+
+  while i < len do
+
+    local fi, si, capture = input:find("`([^`]*)`", i)
+
+    if not fi then
+      table.insert(packed, input:sub(i))
+      break
+    end
+
+    local sub = io.popen(capture)
+    local result = sub:read("*a")
+    sub:close()
+    -- all whitespace is replaced by single spaces
+    -- we requote the result because tokenize will respect this as text
+    table.insert(packed, (text.trim(result):gsub("%s+"," ")))
+
+    i = si+1
+  end
+
+  return table.concat(packed)
 end --[[@delayloaded-end@]]
 
 return sh, local_env

src/main/resources/assets/opencomputers/loot/OpenOS/lib/shell.lua

@@ -35,7 +35,7 @@ local function findFile(name, ext)
     dir = fs.concat(fs.concat(dir, name), "..")
     local name = fs.name(name)
     local list = fs.list(dir)
-    if list then
+    if list and name then
       local files = {}
       for file in list do
         files[file] = true

src/main/resources/assets/opencomputers/loot/OpenOS/lib/text.lua

@@ -8,14 +8,8 @@ local text = {}
 local local_env = {tx=tx,unicode=unicode}
 
 text.internal = {}
-setmetatable(text.internal,
-{
-  __tostring=function()
-    return 'table of undocumented api subject to change and intended for internal use'
-  end
-})
 
-text.syntax = {";","&&","||","|",">>",">","<"}
+text.syntax = {"^%d*>>?&%d+$",";","&&","||?","^%d*>>?",">>?","<"}
 
 function --[[@delayloaded-start@]] text.detab(value, tabWidth)
   checkArg(1, value, "string")
@@ -163,11 +157,12 @@ function text.internal.tokenize(value, quotes, delimiters)
   checkArg(1, value, "string")
   checkArg(2, quotes, "table", "nil")
   checkArg(3, delimiters, "table", "nil")
+  local custom = not not delimiters
   delimiters = delimiters or text.syntax
 
   local words, reason = text.internal.words(value, quotes)
 
-  local splitter = text.escapeMagic(table.concat(delimiters))
+  local splitter = text.escapeMagic(custom and table.concat(delimiters) or "<>|;&")
   if type(words) ~= "table" or 
     #splitter == 0 or
     not value:find("["..splitter.."]") then
@@ -182,7 +177,7 @@ function text.internal.words(input, quotes)
   checkArg(1, input, "string")
   checkArg(2, quotes, "table", "nil")
   local qr = nil
-  quotes = quotes or {{"'","'",true},{'"','"'}}
+  quotes = quotes or {{"'","'",true},{'"','"'},{'`','`'}}
   local function append(dst, txt, qr)
     local size = #dst
     if size == 0 or dst[size].qr ~= qr then
@@ -253,9 +248,6 @@ function --[[@delayloaded-start@]] text.internal.splitWords(words, delimiters)
     table.insert(split_words[#split_words], part)
     next_word = false
   end
-  local delimLookup = tx.select(delimiters, function(e,i)
-    return i, e
-  end)
   for wi=1,#words do local word = words[wi]
     next_word = true
     for pi=1,#word do local part = word[pi]
@@ -265,7 +257,7 @@ function --[[@delayloaded-start@]] text.internal.splitWords(words, delimiters)
       else
         local part_text_splits = text.split(part.txt, delimiters)
         tx.foreach(part_text_splits, function(sub_txt, spi)
-          local delim = delimLookup[sub_txt]
+          local delim = #text.split(sub_txt, delimiters, true) == 0
           next_word = next_word or delim
           add_part({txt=sub_txt,qr=qr})
           next_word = delim