os: implement os.fd_is_pending/1, os.Process.pipe_read/1, os.Process.is_pending/1 (#19787)

This commit is contained in:
Delyan Angelov 2023-11-07 13:47:25 +02:00 committed by GitHub
parent a92700e93b
commit 50c22b5a12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 288 additions and 136 deletions

View File

@ -76,7 +76,7 @@ jobs:
- name: v_win.c can be compiled and run with -os windows
run: |
./v -cc msvc -os windows -o /tmp/v_win.c cmd/v
x86_64-w64-mingw32-gcc /tmp/v_win.c -std=c99 -w -municode -o v_from_vc.exe
x86_64-w64-mingw32-gcc /tmp/v_win.c -std=c99 -w -municode -o v_from_vc.exe -lws2_32
ls -lart v_from_vc.exe
wine64 ./v_from_vc.exe version

View File

@ -20,7 +20,7 @@ jobs:
- name: Test new v.c
run: |
.\v.exe -o v.c cmd/v
gcc -Werror -municode -w v.c
gcc -Werror -municode -w v.c -lws2_32
- name: Install dependencies
run: |
.\v.exe setup-freetype
@ -119,7 +119,7 @@ jobs:
- name: Test new v.c
run: |
.\v.exe -o v.c cmd/v
.\thirdparty\tcc\tcc.exe -Werror -w -ladvapi32 -bt10 v.c
.\thirdparty\tcc\tcc.exe -Werror -w -ladvapi32 -lws2_32 -bt10 v.c
- name: Install dependencies
run: |
.\v.exe setup-freetype
@ -163,7 +163,7 @@ jobs:
# .\v.exe wipe-cache
# .\make.bat -tcc32
# - name: Test new v.c
# run: .\v.exe -o v.c cmd/v && .\thirdparty\tcc\tcc.exe -Werror -g -w -ladvapi32 -bt10 v.c
# run: .\v.exe -o v.c cmd/v && .\thirdparty\tcc\tcc.exe -Werror -g -w -ladvapi32 -lws2_32 -bt10 v.c
# - name: v doctor
# run: ./v doctor
#

View File

@ -4,7 +4,7 @@ LABEL maintainer="Delyan Angelov <delian66@gmail.com>"
COPY . .
RUN make
RUN ./v -os windows -o v.c cmd/v
RUN x86_64-w64-mingw32-gcc v.c -std=c99 -w -municode -o v.exe
RUN x86_64-w64-mingw32-gcc v.c -std=c99 -w -municode -o v.exe -lws2_32
RUN file v.exe
CMD [ "bash" ]

View File

@ -98,7 +98,7 @@ endif
all: latest_vc latest_tcc latest_legacy
ifdef WIN32
$(CC) $(CFLAGS) -std=c99 -municode -w -o v1.exe $(VC)/$(VCFILE) $(LDFLAGS)
$(CC) $(CFLAGS) -std=c99 -municode -w -o v1.exe $(VC)/$(VCFILE) $(LDFLAGS) -lws2_32
v1.exe -no-parallel -o v2.exe $(VFLAGS) cmd/v
v2.exe -o $(VEXE) $(VFLAGS) cmd/v
del v1.exe

View File

@ -198,6 +198,10 @@ pub fn (mut vgit_context VGitContext) compile_oldv_if_needed() {
// after 53ffee1 2020-05-18, gcc builds on windows do need `-municode`
c_flags += '-municode'
}
// after 2023-11-07, windows builds need linking to ws2_32:
if vgit_context.commit_v__ts >= 1699341818 && !vgit_context.cc.contains('msvc') {
c_flags += '-lws2_32'
}
command_for_building_v_from_c_source = '${vgit_context.cc} ${c_flags} -o cv.exe "${vc_source_file_location}" ${c_ldflags}'
command_for_selfbuilding = '.\\cv.exe -o ${vgit_context.vexename} {SOURCE}'
} else {

View File

@ -1,74 +0,0 @@
module main
import os
// this is a example script to show you stdin can be used and keep a process open
fn exec(cmd string) (string, int) {
mut cmd2 := cmd
mut out := ''
mut line := ''
mut rc := 0
mut p := os.new_process('/bin/bash')
// there are methods missing to know if stderr/stdout has data as such its better to redirect bot on same FD
// not so nice trick to run bash in bash and redirect stderr, maybe someone has a better solution
p.set_args(['-c', 'bash 2>&1'])
p.set_redirect_stdio()
p.run()
p.stdin_write('${cmd2} && echo **OK**')
os.fd_close(p.stdio_fd[0]) // important: close stdin so cmd can end by itself
for p.is_alive() {
line = p.stdout_read()
println(line)
// line_err = p.stderr_read() //IF WE CALL STDERR_READ will block
// we need a mechanism which allows us to check if stderr/stdout has data or it should never block
// is not a good way, need to use a string buffer, is slow like this
out += line
if line.ends_with('**OK**\n') {
out = out[0..(out.len - 7)]
break
}
}
// println("read from stdout, should not block")
// is not really needed but good test to see behaviour
out += p.stdout_read()
println('read done')
println(p.stderr_read())
p.close()
p.wait()
if p.code > 0 {
rc = 1
println('ERROR:')
println(cmd2)
print(out)
}
return out, rc
}
fn main() {
mut out := ''
mut rc := 0
// find files from /tmp excluding files unlistable by current user
out, rc = exec("find /tmp/ -user \$UID; echo '******'")
println(out)
assert out.ends_with('******\n')
out, rc = exec('echo to stdout')
assert out.contains('to stdout')
out, rc = exec('echo to stderr 1>&2')
assert out.contains('to stderr')
out, rc = exec('ls /sssss')
assert rc > 0 // THIS STILL GIVES AN ERROR !
println('test ok stderr & stdout is indeed redirected')
}

View File

@ -0,0 +1,97 @@
module main
// This example shows how to communicate with a child process (`bash` in this case), by sending
// commands to its stdin pipe, and reading responses from its stdout and stderr pipes.
// Note, you can use `if p.is_pending(.stdout) {` and `if p.is_pending(.stderr) {`, to check if
// there is available data in the pipes, without having to block in your main loop, if the data
// is missing or just not available yet.
import os
import time
const tmp_folder = os.join_path(os.temp_dir(), 'process_folder')
const max_txt_files = 20
fn exec(cmd string) (string, int, string) {
mut out := []string{}
mut er := []string{}
mut rc := 0
mut p := os.new_process('/bin/bash')
p.set_redirect_stdio()
p.run()
p.stdin_write('echo "START " && sleep 0.1 && ${cmd};\n')
p.stdin_write('ECODE=\$?;\n')
p.stdin_write('sleep 0.1;\n')
p.stdin_write('exit \$ECODE;\n')
// Note, that you can also ensure that `bash` will exit, when the command finishes,
// by closing its stdin pipe. In the above example, that is not needed however, since
// the last `exit` command, will make it quit as well.
// os.fd_close(p.stdio_fd[0])
for p.is_alive() {
if data := p.pipe_read(.stderr) {
eprintln('p.pipe_read .stderr, len: ${data.len:4} | data: `${data#[0..10]}`...')
er << data
}
if data := p.pipe_read(.stdout) {
eprintln('p.pipe_read .stdout, len: ${data.len:4} | data: `${data#[0..10]}`...')
out << data
}
// avoid a busy loop, by sleeping a bit between each iteration
time.sleep(2 * time.millisecond)
}
// the process finished, slurp all the remaining data in the pipes:
out << p.stdout_slurp()
er << p.stderr_slurp()
p.close()
p.wait()
if p.code > 0 {
eprintln('----------------------------------------------------------')
eprintln('COMMAND: ${cmd}')
eprintln('STDOUT:\n${out}')
eprintln('STDERR:\n${er}')
eprintln('----------------------------------------------------------')
rc = 1
}
return out.join(''), rc, er.join('')
}
fn main() {
mut out := ''
mut er := ''
mut ecode := 0
// prepare some files in a temporary folder
defer {
os.rmdir_all(tmp_folder) or {}
}
os.mkdir_all(tmp_folder) or {}
for i in 0 .. max_txt_files {
os.write_file(os.join_path(tmp_folder, '${i}.txt'), '${i}\n${i}\n')!
}
out, ecode, er = exec("find ${os.quoted_path(tmp_folder)} ; sleep 0.1; find ${os.quoted_path(tmp_folder)} ; echo '******'")
assert out.ends_with('******\n')
assert er == ''
out, ecode, er = exec('echo to stdout')
assert out.contains('to stdout')
assert er == ''
out, ecode, er = exec('echo to stderr 1>&2')
assert out.starts_with('START')
assert er.contains('to stderr')
out, ecode, er = exec('ls /sssss')
assert out.starts_with('START')
assert er != ''
assert ecode > 0 // THIS STILL GIVES AN ERROR !
println('test ok stderr & stdout is indeed redirected, ecode: ${ecode}')
}

View File

@ -134,7 +134,7 @@ REM By default, use tcc, since we have it prebuilt:
:tcc_strap
:tcc32_strap
echo ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with "!tcc_exe!"
"!tcc_exe!" -Bthirdparty/tcc -bt10 -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32
"!tcc_exe!" -Bthirdparty/tcc -bt10 -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32 -lws2_32
if %ERRORLEVEL% NEQ 0 goto :compile_error
echo ^> Compiling "%V_EXE%" with "%V_BOOTSTRAP%"
"%V_BOOTSTRAP%" -keepc -g -showcc -cc "!tcc_exe!" -cflags -Bthirdparty/tcc -o "%V_UPDATED%" cmd/v
@ -151,7 +151,7 @@ if %ERRORLEVEL% NEQ 0 (
)
echo ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with Clang
clang -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32
clang -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32 -lws2_32
if %ERRORLEVEL% NEQ 0 (
echo In most cases, compile errors happen because the version of Clang installed is too old
clang --version
@ -173,7 +173,7 @@ if %ERRORLEVEL% NEQ 0 (
)
echo ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with GCC
gcc -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32
gcc -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32 -lws2_32
if %ERRORLEVEL% NEQ 0 (
echo In most cases, compile errors happen because the version of GCC installed is too old
gcc --version
@ -214,7 +214,7 @@ if exist "%InstallDir%/Common7/Tools/vsdevcmd.bat" (
set ObjFile=.v.c.obj
echo ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with MSVC
cl.exe /volatile:ms /Fo%ObjFile% /W0 /MD /D_VBOOTSTRAP "vc/v_win.c" user32.lib kernel32.lib advapi32.lib shell32.lib /link /nologo /out:"%V_BOOTSTRAP%" /incremental:no
cl.exe /volatile:ms /Fo%ObjFile% /W0 /MD /D_VBOOTSTRAP "vc/v_win.c" user32.lib kernel32.lib advapi32.lib shell32.lib ws2_32.lib /link /nologo /out:"%V_BOOTSTRAP%" /incremental:no
if %ERRORLEVEL% NEQ 0 (
echo In some cases, compile errors happen because of the MSVC compiler version
cl.exe

View File

@ -1,8 +1,18 @@
module os
// file descriptor based operations:
// low level operations with file descriptors/handles
// close filedescriptor
$if !windows {
#include <sys/select.h>
}
$if windows {
#include <winsock2.h>
}
#flag windows -lws2_32
// fd_close closes the file descriptor. It returns 0 on success.
pub fn fd_close(fd int) int {
if fd == -1 {
return 0
@ -10,6 +20,8 @@ pub fn fd_close(fd int) int {
return C.close(fd)
}
// fd_write writes the given string to the file descriptor.
// It blocks until all the data in the string is written.
pub fn fd_write(fd int, s string) {
if fd == -1 {
return
@ -26,7 +38,7 @@ pub fn fd_write(fd int, s string) {
}
}
// read from filedescriptor, block until data
// fd_slurp reads all the remaining data from the file descriptor.
pub fn fd_slurp(fd int) []string {
mut res := []string{}
if fd == -1 {
@ -42,8 +54,7 @@ pub fn fd_slurp(fd int) []string {
return res
}
// read from filedescriptor, don't block
// return [bytestring,nrbytes]
// fd_read reads data from the file descriptor. It returns the read data, and how many bytes were read.
pub fn fd_read(fd int, maxbytes int) (string, int) {
if fd == -1 {
return '', 0
@ -59,3 +70,37 @@ pub fn fd_read(fd int, maxbytes int) (string, int) {
return tos(buf, nbytes), nbytes
}
}
[typedef]
pub struct C.fd_set {}
pub struct C.timeval {
tv_sec u64
tv_usec u64
}
fn C.@select(ndfs int, readfds &C.fd_set, writefds &C.fd_set, exceptfds &C.fd_set, timeout &C.timeval) int
// These are C macros, but from the V's point of view, can be treated as C functions:
fn C.FD_ZERO(fdset &C.fd_set)
fn C.FD_SET(fd int, fdset &C.fd_set)
fn C.FD_ISSET(fd int, fdset &C.fd_set) int
// fd_is_pending returns true, when there is pending data, waiting to be read from file descriptor `fd`.
// If the file descriptor is closed, or if reading from it, will block (there is no data), fd_is_pending returns false.
pub fn fd_is_pending(fd int) bool {
read_set := C.fd_set{}
C.FD_ZERO(&read_set)
C.FD_SET(fd, &read_set)
mut ts := C.timeval{
tv_sec: 0
tv_usec: 0
}
res := C.@select(fd + 1, &read_set, C.NULL, C.NULL, &ts)
if res > 0 {
if C.FD_ISSET(fd, &read_set) != 0 {
return true
}
}
return false
}

View File

@ -93,9 +93,9 @@ fn test_read_bytes_into_newline_binary() {
bw[9] = 0xff
bw[12] = 10 // newline
n0_bytes := bw[0..10]
n1_bytes := bw[10..13]
n2_bytes := bw[13..]
n0_bytes := unsafe { bw[0..10] }
n1_bytes := unsafe { bw[10..13] }
n2_bytes := unsafe { bw[13..] }
mut f := os.open_file(tfile, 'w')!
f.write(bw)!

View File

@ -1,5 +1,12 @@
module os
// The kind of the pipe file descriptor, that is used for communicating with the child process
pub enum ChildProcessPipeKind {
stdin
stdout
stderr
}
// signal_kill - kills the process, after that it is no longer running
pub fn (mut p Process) signal_kill() {
if p.status !in [.running, .stopped] {
@ -109,71 +116,143 @@ fn (mut p Process) _spawn() int {
// is_alive - query whether the process p.pid is still alive
pub fn (mut p Process) is_alive() bool {
mut res := false
if p.status in [.running, .stopped] {
return p._is_alive()
res = p._is_alive()
}
return false
$if trace_process_is_alive ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, res: ${res}')
}
return res
}
//
pub fn (mut p Process) set_redirect_stdio() {
p.use_stdio_ctl = true
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}')
}
return
}
// stdin_write will write the string `s`, to the stdin pipe of the child process.
pub fn (mut p Process) stdin_write(s string) {
p._check_redirection_call('stdin_write')
$if windows {
p.win_write_string(0, s)
} $else {
fd_write(p.stdio_fd[0], s)
p._check_redirection_call(@METHOD)
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, s.len: ${s.len}, s: `${s}`')
}
p._write_to(.stdin, s)
}
// will read from stdout pipe, will only return when EOF (end of file) or data
// means this will block unless there is data
// stdout_slurp will read from the stdout pipe, and will block until it either reads all the data, or until the pipe is closed (end of file).
pub fn (mut p Process) stdout_slurp() string {
p._check_redirection_call('stdout_slurp')
$if windows {
return p.win_slurp(1)
} $else {
return fd_slurp(p.stdio_fd[1]).join('')
p._check_redirection_call(@METHOD)
res := p._slurp_from(.stdout)
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, res.len: ${res.len}, res: `${res}`')
}
return res
}
// read from stderr pipe, wait for data or EOF
// stderr_slurp will read from the stderr pipe, and will block until it either reads all the data, or until the pipe is closed (end of file).
pub fn (mut p Process) stderr_slurp() string {
p._check_redirection_call('stderr_slurp')
$if windows {
return p.win_slurp(2)
} $else {
return fd_slurp(p.stdio_fd[2]).join('')
p._check_redirection_call(@METHOD)
res := p._slurp_from(.stderr)
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, res.len: ${res.len}, res: `${res}`')
}
return res
}
// read from stdout, return if data or not
// stdout_read reads a block of data, from the stdout pipe of the child process. It will block, if there is no data to be read.
// Call .is_pending() to check if there is data to be read, if you do not want to block.
pub fn (mut p Process) stdout_read() string {
p._check_redirection_call('stdout_read')
p._check_redirection_call(@METHOD)
res := p._read_from(.stdout)
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, res.len: ${res.len}, res: `${res}`')
}
return res
}
// stderr_read reads a block of data, from the stderr pipe of the child process. It will block, if there is no data to be read.
// Call .is_pending() to check if there is data to be read, if you do not want to block.
pub fn (mut p Process) stderr_read() string {
p._check_redirection_call(@METHOD)
res := p._read_from(.stderr)
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, res.len: ${res.len}, res: `${res}`')
}
return res
}
// pipe_read reads a block of data, from the given pipe of the child process.
// It will return `none`, if there is no data to be read, *without blocking*.
pub fn (mut p Process) pipe_read(pkind ChildProcessPipeKind) ?string {
p._check_redirection_call(@METHOD)
if !p._is_pending(pkind) {
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, no pending data')
}
return none
}
res := p._read_from(pkind)
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, res.len: ${res.len}, res: `${res}`')
}
return res
}
// is_pending returns whether there is data to be read from child process's pipe corresponding to `pkind`.
// For example `if p.is_pending(.stdout) { dump( p.stdout_read() ) }` will not block indefinitely.
pub fn (mut p Process) is_pending(pkind ChildProcessPipeKind) bool {
p._check_redirection_call(@METHOD)
res := p._is_pending(pkind)
$if trace_process_pipes ? {
eprintln('${@LOCATION}, pid: ${p.pid}, status: ${p.status}, pkind: ${pkind}, res: ${res}')
}
return res
}
// _read_from should be called only from .stdout_read/0, .stderr_read/0 and .pipe_read/1
fn (mut p Process) _read_from(pkind ChildProcessPipeKind) string {
$if windows {
s, _ := p.win_read_string(1, 4096)
s, _ := p.win_read_string(int(pkind), 4096)
return s
} $else {
s, _ := fd_read(p.stdio_fd[1], 4096)
s, _ := fd_read(p.stdio_fd[pkind], 4096)
return s
}
}
pub fn (mut p Process) stderr_read() string {
p._check_redirection_call('stderr_read')
// _slurp_from should be called only from stdout_slurp() and stderr_slurp()
fn (mut p Process) _slurp_from(pkind ChildProcessPipeKind) string {
$if windows {
s, _ := p.win_read_string(2, 4096)
return s
return p.win_slurp(int(pkind))
} $else {
s, _ := fd_read(p.stdio_fd[2], 4096)
return s
return fd_slurp(p.stdio_fd[pkind]).join('')
}
}
// _write_to should be called only from stdin_write()
fn (mut p Process) _write_to(pkind ChildProcessPipeKind, s string) {
$if windows {
p.win_write_string(int(pkind), s)
} $else {
fd_write(p.stdio_fd[pkind], s)
}
}
// _is_pending should be called only from is_pending()
fn (mut p Process) _is_pending(pkind ChildProcessPipeKind) bool {
$if windows {
// TODO
} $else {
return fd_is_pending(p.stdio_fd[pkind])
}
return false
}
// _check_redirection_call - should be called just by stdxxx methods
fn (mut p Process) _check_redirection_call(fn_name string) {
if !p.use_stdio_ctl {

View File

@ -325,7 +325,7 @@ pub fn (mut v Builder) cc_msvc() {
// Emily:
// Not all of these are needed (but the compiler should discard them if they are not used)
// these are the defaults used by msbuild and visual studio
mut real_libs := ['kernel32.lib', 'user32.lib', 'advapi32.lib']
mut real_libs := ['kernel32.lib', 'user32.lib', 'advapi32.lib', 'ws2_32.lib']
sflags := msvc_string_flags(v.get_os_cflags())
real_libs << sflags.real_libs
inc_paths := sflags.inc_paths

View File

@ -1,14 +1,14 @@
#include "@VMODROOT/epoll.h"
#include "@VMODROOT/netdb.h"
pub struct C.epoll_event {
pub struct C.zz_epoll_event {
mut:
events u32
data C.epoll_data_t
data C.zz_epoll_data_t
}
[typedef]
pub union C.epoll_data_t {
pub union C.zz_epoll_data_t {
mut:
ptr voidptr
fd int
@ -17,10 +17,10 @@ mut:
}
struct Epoll {
ev C.epoll_event
ev C.zz_epoll_event
}
pub struct C.hostent {
pub struct C.zz_hostent {
h_name &char
h_aliases &&char
h_addrtype int
@ -29,18 +29,18 @@ pub struct C.hostent {
}
fn test_dump_c_struct() {
ev := C.epoll_event{}
ev := C.zz_epoll_event{}
unsafe { C.memset(&ev, 0, sizeof(ev)) }
dump(ev)
println(ev)
e := Epoll{
ev: C.epoll_event{}
ev: C.zz_epoll_event{}
}
dump(e)
println(e)
//
mut hostent := &C.hostent{
mut hostent := &C.zz_hostent{
h_addr_list: unsafe { nil }
h_aliases: unsafe { nil }
h_name: unsafe { nil }

View File

@ -1,11 +1,12 @@
typedef union epoll_data {
// Note: the name zz_epoll_data is deliberately chosen to minimise the chance of conflicts with `epoll_data` in the future.
typedef union zz_epoll_data {
void *ptr;
int fd;
uint32_t u32;
uint64_t u64;
} epoll_data_t;
} zz_epoll_data_t;
struct epoll_event {
struct zz_epoll_event {
uint32_t events; /* Epoll events */
epoll_data_t data; /* User data variable */
zz_epoll_data_t data; /* User data variable */
};

View File

@ -1,6 +1,6 @@
/* Description of data base entry for a single host. */
struct hostent {
struct zz_hostent {
char *h_name; /* Official name of host. */
char **h_aliases; /* Alias list. */
int h_addrtype; /* Host address type. */