Hello dear community,
I'm trying to create a new process from a python file that should write its output either on the console or in a seperate file (depending on the users choice).
Writing to the console is easy using poParentStreams. But I don't know how to redirect the output stream to a file stream.
I've tried using this code: https://forum.nim-lang.org/t/6909#43301 by doing this: discard dup2(file.getFileHandle(), process.outputHandle())
But it just kept writing the output to the console. (It also doesn't work if I don't put poParentStreams )
Do you know how to do this? Here is my full code:
proc dup(oldfd: FileHandle): FileHandle {.importc, header: "unistd.h".}
proc dup2(oldfd: FileHandle, newfd: FileHandle): cint {.importc, header: "unistd.h".}
proc runPythonFile() =
let filePath= "path"
let command = "python"
var args = ["fileName", "arg0"]
let options: set[ProcessOption] = {poStdErrToStdOut, poEchoCmd, poParentStreams}
let process = startProcess(command, filePath, args, nil, options)
let file = open("stdOut.txt", fmAppend)
discard dup2(file.getFileHandle(), process.outputHandle())
Thanks a lot!
First, since this question I've added dup and dup2 to fusion/ioutils : https://nim-lang.github.io/fusion/src/fusion/ioutils so you don't need to manually importc proc.
Second, look at this :
let process = startProcess(command, filePath, args, nil, options)
let file = open("stdOut.txt", fmAppend)
discard dup2(file.getFileHandle(), process.outputHandle())
If what you want is redirecting stdout during your process execution you should do this :
proc runPythonFile() =
let filePath= "pypathexample"
let command = "/usr/bin/python3"
var args = ["example.py", "arg0"]
let options: set[ProcessOption] = {poStdErrToStdOut, poEchoCmd, poParentStreams}
let stdoutFileno = stdout.getFileHandle()
let stdoutDupFd = duplicate(stdoutFileno)
let file = open("stdOut.txt", fmAppend)
echo "-----------------------------------------------------------" # No redirected
duplicateTo(file.getFileHandle(), stdoutFileno)
let process = startProcess(command, filePath, args, nil, options)
duplicateTo(stdoutDupFd, stdoutFileno)
echo "-----------------------------------------------------------" # Not redirected
file.close()
process.close()
Thank you very much, both of these code snippets are exactly what I was looking for.
I have one questions regarding the second one: What do I do if the process runs for a while and I don't want to wait for that?
That includes:
And then concerning my situation: Some output might be generated later/over time and it could also be a lot (so maybe to much for a buffer).
So what I'd need is something like this: "Hey process, please append all your output to that file (no matter how long you run or how much output you have) and close yourself (and the file) after you have finished. (because I [the nim programm] am not going to be running by then anymore)"
Is something like this possible? Sorry if it's kind of complicated.. ^^
I've been experimenting with the code from Clonk and I noticed, that process.outputStream.readAll() makes the nim programm wait for all output of the process (also if the process sleeps for several seconds). That's kind of surprising in my opinion because it's not described like this in the documentation. But I noticed that after the call of
let s = process.outputStream.readAll()
file.write(s)
file.close()
Apart from these minor questions, my main problem still remains: Can I redirect the output of a process to a file without waiting for the process to finish? Or do I have to create a separate thread for that because it is not possible?
Thank you :)
From your description, it sounds like you want async process, which isn't what osproc does.
I see 2 solutions to your problem :
1)Don't use a process and instead use Nimpy to call your Python code from a Nim thread.
# File = ./pypath/example.py
def mainPythonEntryPoint(arg1, arg2):
print("> BEGIN <")
i = 0
while i < 100:
i = i+1
print(arg1)
print(arg2)
time.sleep(0.01)
print("> DONE <")
import fusion/ioutils
import std/os
import nimpy
proc threadednimpy(args: tuple[a1: int, a2: string]) {.gcsafe.} =
echo "Thread Start"
# Import your python file
doAssert(pyImport("os").getcwd().to(string) == getCurrentDir())
discard pyImport("sys").path.append("pypath")
let example = pyImport("example")
# Redirect output for this thread
let stdoutFileno = stdout.getFileHandle()
let stdoutDupFd = duplicate(stdoutFileno)
let file = open("stdOut3.txt", fmAppend)
defer: file.close()
duplicateTo(file.getFileHandle(), stdoutFileno)
let res = example.mainPythonEntryPoint(args.a1, args.a2) # Calling python function
duplicateTo(stdoutDupFd, stdoutFileno)
echo "Thread Done"
proc runPythonFile3() =
# These are just dummy arguments;
var thr : Thread[tuple[a1: int, a2: string]]
createThread(thr, threadednimpy, (a1: 1, a2: "Nimpy is great"))
# Check main thread is not redirected
echo "This is not redirected"
echo "Do Stuff"
echo "Do more Stuff"
echo "Do more Stuff"
echo "Stop doing Stuff"
joinThread(thr)
# Compile with --threads:on.
import os, osproc
var chan: Channel[string]
proc firstWorker() =
var tmp: tuple[output: TaintedString, exitCode: int]
tmp = execCmdEx("ls -lh")
chan.send("hi there")
chan.send(tmp.output)
chan.open()
var worker1: Thread[void]
createThread(worker1, firstWorker)
echo chan.recv()
echo chan.recv()
worker1.joinThread()
chan.close()
# Compile with --threads:on.
import osproc, streams
type
execChan = object
val: string
ecode: int
type
ThreadData = tuple[cmd: string, wd: string, args: seq[string]]
var chan: Channel[execChan]
proc firstWorker(cmdInfo: ThreadData) =
var
tmp: tuple[output : string, exitCode : int]
chval: execChan
p: owned(Process)
try:
p = startProcess(command = cmdInfo.cmd, args = cmdInfo.args, workingDir = cmdInfo.wd, options = {poStdErrToStdOut, poUsePath})
except:
chval.val = "Error: " & getCurrentExceptionMsg()
chval.ecode = -12345
chan.send(chval)
return
var outp = outputStream(p)
close inputStream(p)
tmp = ("", -1)
var line = newStringOfCap(120).TaintedString
while true:
if outp.readLine(line):
tmp[0].string.add(line.string)
tmp[0].string.add("\n")
else:
tmp[1] = peekExitCode(p)
if tmp[1] != -1: break
close(p)
chval.val = tmp.output
chval.ecode = tmp.exitCode
chan.send(chval)
chan.open()
var
worker1: Thread[ThreadData]
argSeq: seq[string]
recvVal: execChan
argSeq.add("-lh")
createThread[ThreadData](worker1, firstWorker, (cmd: "/usr/bin/ls", wd: "/tmp", args: argSeq))
recvVal = chan.recv()
echo recvVal.val
echo recvVal.ecode
# Wait for the thread to exit
worker1.joinThread()
chan.close()