native: initial linking support for linux (#15326)

.github/workflows/containers_ci.yml

@@ -40,6 +40,13 @@ jobs:
           echo "C Compiler:"
           gcc --version
 
+## TODO: fix the container thevlang/vlang:alpine-build, instead of using apk add here:
+      - name: Add dependencies
+        run: |
+          apk add libexecinfo-static
+          apk add libexecinfo-dev
+          apk add libc6-compat
+
       - name: Build V
         run: CC=gcc make
 

.github/workflows/native_backend_tests.yml

@@ -55,6 +55,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Install linker
+        if: ${{ startsWith(matrix.os, 'ubuntu')}}
+        run: |
+          sudo apt-get install --quiet -y binutils
+
       - name: Build V with make.bat
         if: ${{ startsWith(matrix.os, 'windows') }}
         run: |

Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /opt/vlang
 ARG USE_LOCAL
 
 RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc clang make git && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc clang make git binutils && \
     apt-get clean && rm -rf /var/cache/apt/archives/* && \
     rm -rf /var/lib/apt/lists/*
 

Dockerfile.alpine

@@ -12,11 +12,14 @@ RUN mkdir -p /opt/vlang && ln -s /opt/vlang/v /usr/bin/v
 
 ARG USE_LOCAL
 
+## libexecinfo is needed for the bundled tcc
 RUN apk --no-cache add \
   git make upx gcc bash \
-  musl-dev \
+  musl-dev libc-dev \
   openssl-dev sqlite-dev \
-  libx11-dev glfw-dev freetype-dev
+  libx11-dev glfw-dev freetype-dev \
+  libexecinfo-dev libexecinfo-static \
+  libc6-compat gcompat binutils
 
 ## RUN apk --no-cache add --virtual sdl2deps sdl2-dev sdl2_ttf-dev sdl2_mixer-dev sdl2_image-dev
 COPY . /vlang-local

vlib/v/gen/native/amd64.v

@@ -2207,7 +2207,7 @@ fn (mut g Gen) fn_decl_amd64(node ast.FnDecl) {
 	g.println('; stack frame size: $g.stack_var_pos')
 	g.write32_at(local_alloc_pos + 3, g.stack_var_pos)
 	is_main := node.name == 'main.main'
-	if is_main {
+	if is_main && g.pref.os != .linux {
 		// println('end of main: gen exit')
 		zero := ast.IntegerLiteral{}
 		g.gen_exit(zero)

vlib/v/gen/native/elf.v

@@ -3,6 +3,8 @@
 // that can be found in the LICENSE file.
 module native
 
+import os
+
 const (
 	elf_class32              = 1
 	elf_class64              = 2
@@ -62,13 +64,24 @@ const (
 	elf_sht_shlib            = 0x0a
 	elf_sht_dynsym           = 0x0b
 
+	// elf section flags
+	elf_shf_write            = u64(0x01)
+	elf_shf_alloc            = u64(0x02)
+	elf_shf_execinstr        = u64(0x04)
+	elf_shf_merge            = u64(0x10)
+	elf_sht_strings          = u64(0x20)
+	elf_shf_info_link        = u64(0x40)
+	elf_shf_link_order       = u64(0x80)
+	elf_shf_os_nonconforming = u64(0x100)
+	elf_shf_groub            = u64(0x200)
+	elf_shf_tls              = u64(0x400)
+
 	// elf symbol tables
 	elf_symtab_size          = 0x18
 	elf_dynamic_size         = 0x10
 	elf_rela_size            = 0x18
 	elf_rel_size             = 0x10
 	elf_sh_symtab_entsize    = elf_symtab_size
-	elf_sh_symtab_info    = 1
 	elf_sh_symtab_align      = 8
 
 	// elf symbol bining
@@ -80,6 +93,7 @@ const (
 	elf_stt_object           = u8(1)
 	elf_stt_func             = u8(2)
 	elf_stt_section          = u8(3)
+	elf_stt_file             = u8(4)
 
 	// elf symbol visibility
 	elf_stv_default          = i8(0)
@@ -370,11 +384,15 @@ fn (mut g Gen) create_symtab(mut sections []Section, mut table []SymbolTableSect
 	mut names := []string{len: table.len}
 	mut offset := 1
 
+	text_section := g.find_section_header('.text', sections)
+
 	for i, mut entry in table {
 		names[i] = entry.str_name
 
 		entry.name = offset
-		entry.shndx = i16(sections.len + 1)
+		if entry.name != 1 {
+			entry.shndx = i16(text_section) // i16(sections.len + 1)
+		}
 
 		offset += entry.str_name.len + 1
 	}
@@ -382,10 +400,17 @@ fn (mut g Gen) create_symtab(mut sections []Section, mut table []SymbolTableSect
 	sections << g.create_section('.strtab', native.elf_sht_strtab, 0, 0, 1, 0, g.create_string_table_section(names))
 
 	sections << // index of .strtab
-	g.create_section('.symtab', native.elf_sht_symtab, sections.len - 1, native.elf_sh_symtab_info,
+	g.create_section('.symtab', native.elf_sht_symtab, sections.len - 1, table.len - 1,
 		native.elf_sh_symtab_align, native.elf_sh_symtab_entsize, table)
 }
 
+fn (mut g Gen) create_progbits(name string, flags u64, data []u8) Section {
+	mut section := g.create_section(name, native.elf_sht_progbits, 0, 0, 1, data.len,
+		ProgBitsSection{data})
+	section.header.flags = i64(flags)
+	return section
+}
+
 fn (mut g Gen) find_section_header(name string, sections []Section) int {
 	for i, section in sections {
 		if name == section.name {
@@ -434,7 +459,7 @@ fn (mut g Gen) gen_sections(mut sections []Section) {
 
 fn (mut g Gen) gen_symtab_data(section Section, data []SymbolTableSection) {
 	for symbol in data {
-		if symbol.str_name == '_start' {
+		if symbol.str_name == 'main' { // represents the C main function
 			g.start_symbol_addr = g.pos()
 		}
 
@@ -477,13 +502,7 @@ fn (mut g Gen) gen_section_data(sections []Section) {
 				g.gen_symtab_data(section, data)
 			}
 			ProgBitsSection {
-				start := g.pos()
+				// progbits have to be handled by the user.
-
-				g.write(data.bytes)
-				g.println('; SHT_PROGBITS Section')
-
-				size := g.pos() - start
-				g.write64_at(section.header.offset + 32, i64(size))
 			}
 			RelASection {
 				g.write64(data.offset)
@@ -579,7 +598,7 @@ mut:
 */
 
 pub fn (mut g Gen) generate_elf_header() {
-	elf_type := native.elf_type_exec // PIE (use _exec for non-relocatable executables)
+	elf_type := native.elf_type_rel // PIE (use _exec for non-relocatable executables)
 
 	// generate program headers
 	mut program_headers := []ProgramHeader{}
@@ -588,12 +607,19 @@ pub fn (mut g Gen) generate_elf_header() {
 	// generate sections
 	mut sections := [
 		Section{}, // null section as first section
+		g.create_progbits('.text', native.elf_shf_alloc | native.elf_shf_execinstr, []),
+		g.create_progbits('.data', native.elf_shf_write | native.elf_shf_alloc, []),
+		g.create_progbits('.bss', native.elf_shf_write | native.elf_shf_alloc, []),
 	]
 
 	mut symbols := [
 		SymbolTableSection{}, // first is null
-		g.create_symbol_table_section('_start', native.elf_stt_notype, native.elf_stb_global,
+		g.create_symbol_table_section('test.v', native.elf_stt_file, native.elf_stb_local,
-			native.elf_stv_default, 0, 0), // _start label points to entry point address
+			native.elf_stv_default, 0, 0), // source file TODO: replace test.v with actual source file name
+		g.create_symbol_table_section('.text', native.elf_stt_section, native.elf_stb_local,
+			native.elf_stv_default, 0, 0), // .text section
+		g.create_symbol_table_section('main', native.elf_stt_func, native.elf_stb_global,
+			native.elf_stv_default, 0, 0), // main label points to entry point address
 	]
 	g.create_symtab(mut sections, mut symbols) // create the .symtab section
 	g.create_shstrtab(mut sections) // create the .shstrtab section (this must be the last section!)
@@ -615,7 +641,6 @@ pub fn (mut g Gen) generate_elf_header() {
 	}
 
 	g.write32(native.elf_version)
-	e_entry_addr := g.pos()
 	g.write64(0) // e_entry (temp value)
 	g.write64(native.elf_header_size) // e_phoff
 	g.write64(native.elf_header_size + native.elf_phentry_size * program_headers.len) // e_shoff
@@ -646,17 +671,25 @@ pub fn (mut g Gen) generate_elf_header() {
 
 	g.code_start_pos = g.pos()
 	g.debug_pos = int(g.pos())
-	if elf_type == native.elf_type_exec {
+	// if g.start_symbol_addr > 0 {
-		g.write64_at(e_entry_addr, g.code_start_pos + native.segment_start)
+	//	g.write64_at(g.start_symbol_addr + native.elf_symtab_size - 16, g.code_start_pos + native.segment_start)
-	}
+	//}
-	if g.start_symbol_addr > 0 {
-		g.write64_at(g.start_symbol_addr + native.elf_symtab_size - 16, g.code_start_pos +
-			native.segment_start)
-	}
 
+	text_section := sections[g.find_section_header('.text', sections)]
+	g.elf_text_header_addr = text_section.header.offset
+	g.write64_at(g.elf_text_header_addr + 24, g.pos()) // write the code start pos to the text section
+
+	g.call(native.placeholder)
+	g.println('; call main.main')
+	if g.pref.arch == .arm64 {
+	} else {
+		g.mov64(.rax, 0)
+	}
+	g.ret()
+	g.println('; return 0')
+
+	g.write64_at(g.start_symbol_addr + native.elf_symtab_size - 8, g.pos() - g.code_start_pos) // write 'main' function size
 	g.debug_pos = g.buf.len
-	g.call(native.placeholder) // call main function, it's not guaranteed to be the first, we don't know its address yet
-	g.println('; call fn main')
 }
 
 fn (mut g Gen) elf_string_table() {
@@ -695,5 +728,88 @@ pub fn (mut g Gen) generate_elf_footer() {
 		// -5 is for "e8 00 00 00 00"
 		g.write32_at(g.code_start_pos + 1, int(g.main_fn_addr - g.code_start_pos) - 5)
 	}
+
+	// write size of text section into section header
+	g.write64_at(g.elf_text_header_addr + 32, g.pos() - g.code_start_pos)
+
 	g.create_executable()
 }
+
+pub fn (mut g Gen) prepend_vobjpath(paths []string) []string {
+	vopath := os.getenv('VOBJPATH')
+	if vopath == '' {
+		return paths
+	}
+	mut res := paths.clone()
+	res.insert(0, vopath)
+	return res
+}
+
+pub fn (mut g Gen) find_o_path(fname string) string {
+	opaths := g.prepend_vobjpath(['/usr/lib', '/usr/lib/x86_64-linux-gnu'])
+	for opath in opaths {
+		fpath := os.join_path_single(opath, fname)
+		if os.is_file(fpath) {
+			return fpath
+		}
+	}
+	return '/dev/null'
+}
+
+pub fn (mut g Gen) get_lpaths() string {
+	lpaths := g.prepend_vobjpath(['/usr/lib/x86_64-linux-gnu', '/usr/lib64', '/lib64', '/usr/lib',
+		'/lib'])
+	return lpaths.map('-L$it').join(' ')
+}
+
+pub fn (mut g Gen) link_elf_file(obj_file string) {
+	crt1 := g.find_o_path('crt1.o')
+	crti := g.find_o_path('crti.o')
+	crtn := g.find_o_path('crtn.o')
+	lpaths := g.get_lpaths()
+	linker_args := [
+		'-v',
+		lpaths,
+		'-m elf_x86_64',
+		'-dynamic-linker',
+		'/lib64/ld-linux-x86-64.so.2',
+		crt1,
+		crti,
+		'-lc',
+		'-lm',
+		'-lpthread',
+		crtn,
+		'$obj_file',
+		'-o $g.out_name',
+	]
+	slinker_args := linker_args.join(' ')
+
+	mut ldlld := 'ld'
+	/*
+	match g.pref.os {
+		.linux { ldlld = 'ld.lld' }
+		.windows { ldlld = 'lld-link' }
+		.macos { ldlld = 'ld64.lld' }
+		else {}
+	}
+	*/
+	custom_linker := os.getenv('VLINKER')
+	if custom_linker != '' {
+		ldlld = custom_linker
+	}
+	linker_path := os.real_path(ldlld)
+	linker_cmd := '${os.quoted_path(linker_path)} $slinker_args'
+	if g.pref.is_verbose {
+		println(linker_cmd)
+	}
+
+	res := os.execute(linker_cmd)
+	if res.exit_code != 0 {
+		g.n_error('ELF linking failed ($ldlld):\n$res.output')
+		return
+	}
+
+	if g.pref.is_verbose {
+		println('linking with $ldlld finished successfully:\n$res.output')
+	}
+}

vlib/v/gen/native/gen.v

@@ -31,7 +31,9 @@ mut:
 	sect_header_name_pos int
 	offset               i64
 	file_size_pos        i64
+	elf_text_header_addr i64
 	main_fn_addr         i64
+	main_fn_size         i64
 	start_symbol_addr    i64
 	code_start_pos       i64 // location of the start of the assembly instructions
 	fn_addr              map[string]i64
@@ -222,6 +224,7 @@ pub fn gen(files []&ast.File, table &ast.Table, out_name string, pref &pref.Pref
 	}
 	g.generate_builtins()
 	g.generate_footer()
+
 	return g.nlines, g.buf.len
 }
 
@@ -252,8 +255,21 @@ pub fn (mut g Gen) generate_header() {
 }
 
 pub fn (mut g Gen) create_executable() {
-	// Create the binary // should be .o ?
+	obj_name := match g.pref.os {
-	os.write_file_array(g.out_name, g.buf) or { panic(err) }
+		.linux { g.out_name + '.o' }
+		else { g.out_name }
+	}
+
+	os.write_file_array(obj_name, g.buf) or { panic(err) }
+
+	match g.pref.os {
+		// TEMPORARY
+		.linux { // TEMPORARY
+			g.link(obj_name)
+		} // TEMPORARY
+		else {} // TEMPORARY
+	} // TEMPORARY
+
 	os.chmod(g.out_name, 0o775) or { panic(err) } // make it executable
 	if g.pref.is_verbose {
 		eprintln('\n$g.out_name: native binary has been successfully generated')
@@ -282,6 +298,17 @@ pub fn (mut g Gen) generate_footer() {
 	}
 }
 
+pub fn (mut g Gen) link(obj_name string) {
+	match g.pref.os {
+		.linux {
+			g.link_elf_file(obj_name)
+		}
+		else {
+			g.n_error('native linking is not implemented for $g.pref.os')
+		}
+	}
+}
+
 pub fn (mut g Gen) stmts(stmts []ast.Stmt) {
 	for stmt in stmts {
 		g.stmt(stmt)

vlib/v/gen/native/tests/native_test.v

@@ -36,7 +36,8 @@ fn test_native() {
 		relative_test_path := full_test_path.replace(vroot + '/', '')
 		work_test_path := '$wrkdir/$test_file_name'
 		exe_test_path := '$wrkdir/${test_file_name}.exe'
-		cmd := '${os.quoted_path(vexe)} -o ${os.quoted_path(exe_test_path)} -b native ${os.quoted_path(full_test_path)}'
+		tmperrfile := '$dir/${test}.tmperr'
+		cmd := '${os.quoted_path(vexe)} -o ${os.quoted_path(exe_test_path)} -b native ${os.quoted_path(full_test_path)} 2> ${os.quoted_path(tmperrfile)}'
 		if is_verbose {
 			println(cmd)
 		}
@@ -44,9 +45,14 @@ fn test_native() {
 		if res_native.exit_code != 0 {
 			bench.fail()
 			eprintln(bench.step_message_fail(cmd))
+
+			if os.exists(tmperrfile) {
+				err := os.read_file(tmperrfile) or { panic(err) }
+				eprintln(err)
+			}
+
 			continue
 		}
-		tmperrfile := '$dir/${test}.tmperr'
 		res := os.execute('${os.quoted_path(exe_test_path)} 2> ${os.quoted_path(tmperrfile)}')
 		if res.exit_code != 0 {
 			bench.fail()