net.http: add socks5|http(s) proxy support [Linux] (#19676)

vlib/net/http/backend_nix.c.v

@@ -30,7 +30,11 @@ fn (req &Request) ssl_do(port int, method Method, host_name string, path string)
 	$if trace_http_request ? {
 		eprintln('> ${req_headers}')
 	}
-	// println(req_headers)
+
+	return req.do_request(req_headers, mut ssl_conn)!
+}
+
+fn (req &Request) do_request(req_headers string, mut ssl_conn ssl.SSLConn) !Response {
 	ssl_conn.write_string(req_headers) or { return err }
 
 	mut content := strings.new_builder(100)

vlib/net/http/http_proxy.v

@@ -1,8 +1,10 @@
 module http
 
-import net.urllib
 import encoding.base64
 import net
+import net.urllib
+import net.ssl
+import net.socks
 
 [heap]
 struct HttpProxy {
@@ -21,7 +23,7 @@ pub fn new_http_proxy(raw_url string) !&HttpProxy {
 	mut url := urllib.parse(raw_url) or { return error('malformed proxy url') }
 	scheme := url.scheme
 
-	if scheme != 'http' && scheme != 'https' {
+	if scheme !in ['http', 'https', 'socks5'] {
 		return error('invalid scheme')
 	}
 
@@ -38,11 +40,14 @@ pub fn new_http_proxy(raw_url string) !&HttpProxy {
 	if port == 0 {
 		if scheme == 'https' {
 			port = 443
+			host += ':' + port.str()
 		} else if scheme == 'http' {
 			port = 80
+			host += ':' + port.str()
 		}
-
-		host += ':' + port.str()
+	}
+	if port == 0 {
+		return error('Unknown port')
 	}
 
 	return &HttpProxy{
@@ -56,21 +61,17 @@ pub fn new_http_proxy(raw_url string) !&HttpProxy {
 	}
 }
 
-fn (proxy &HttpProxy) build_proxy_headers(request &Request, host string, path string) string {
-	ua := request.user_agent
+// host format - ip:port
+fn (pr &HttpProxy) build_proxy_headers(host string) string {
 	mut uheaders := []string{}
-
-	uheaders << 'Host: ${host}\r\n'
-	uheaders << 'User-Agent: ${ua}\r\n'
-	uheaders << 'Upgrade-Insecure-Requests: 1\r\n'
-	// uheaders << 'Proxy-Connection: Keep-Alive\r\n'
-
-	if proxy.username != '' {
+	address := host.all_before_last(':')
+	uheaders << 'Proxy-Connection: Keep-Alive\r\n'
+	if pr.username != '' {
 		mut authinfo := ''
 
-		authinfo += proxy.username
-		if proxy.password != '' {
-			authinfo += ':${proxy.password}'
+		authinfo += pr.username
+		if pr.password != '' {
+			authinfo += ':${pr.password}'
 		}
 
 		encoded_authinfo := base64.encode(authinfo.bytes())
@@ -80,31 +81,87 @@ fn (proxy &HttpProxy) build_proxy_headers(request &Request, host string, path st
 
 	version := Version.v1_1
 
-	// we always make requests on http scheme
-	url := 'http://${host}${path}'
-
-	return '${request.method} ${url} ${version}\r\n' + uheaders.join('') + '\r\n'
+	return 'CONNECT ${host} ${version}\r\nHost: ${address}\r\n' + uheaders.join('') + '\r\n'
 }
 
-fn (proxy &HttpProxy) http_do(host string, method Method, path string, req &Request) !Response {
-	host_name, _ := net.split_address(host)!
-	s := proxy.build_proxy_headers(req, host_name, path)
-	mut client := net.dial_tcp(proxy.host)!
-	client.set_read_timeout(req.read_timeout)
-	client.set_write_timeout(req.write_timeout)
-	// TODO this really needs to be exposed somehow
-	client.write_string(s)!
-	$if trace_http_request ? {
-		eprintln('> ${s}')
+fn (pr &HttpProxy) http_do(host urllib.URL, method Method, path string, req &Request) !Response {
+	host_name, _ := net.split_address(host.hostname())!
+
+	s := req.build_request_headers(req.method, host_name, path)
+	if host.scheme == 'https' {
+		mut client := pr.ssl_dial('${host.host}:443')!
+
+		$if windows {
+			return error('Windows Not SUPPORTED') // todo windows ssl
+			// response_text := req.do_request(req.build_request_headers(req.method, host_name,
+			// 	path))!
+			// client.shutdown()!
+			// return response_text
+		} $else {
+			response_text := req.do_request(req.build_request_headers(req.method, host_name,
+				path), mut client)!
+			client.shutdown()!
+			return response_text
+		}
+	} else if host.scheme == 'http' {
+		mut client := pr.dial('${host.host}:80')!
+		client.set_read_timeout(req.read_timeout)
+		client.set_write_timeout(req.write_timeout)
+		client.write_string(s)!
+		$if trace_http_request ? {
+			eprintln('> ${s}')
+		}
+		mut bytes := req.read_all_from_client_connection(client)!
+		client.close()!
+		response_text := bytes.bytestr()
+		$if trace_http_response ? {
+			eprintln('< ${response_text}')
+		}
+		if req.on_finish != unsafe { nil } {
+			req.on_finish(req, u64(response_text.len))!
+		}
+		return parse_response(response_text)
+	}
+	return error('Invalid Scheme')
+}
+
+fn (pr &HttpProxy) dial(host string) !&net.TcpConn {
+	if pr.scheme in ['http', 'https'] {
+		mut tcp := net.dial_tcp(pr.host)!
+		tcp.write(pr.build_proxy_headers(host).bytes())!
+		mut bf := []u8{len: 4096}
+
+		tcp.read(mut bf)!
+		return tcp
+	} else if pr.scheme == 'socks5' {
+		return socks.socks5_dial(pr.host, host, pr.username, pr.password)!
+	} else {
+		return error('http_proxy dial: invalid proxy scheme')
+	}
+}
+
+fn (pr &HttpProxy) ssl_dial(host string) !&ssl.SSLConn {
+	if pr.scheme in ['http', 'https'] {
+		mut tcp := net.dial_tcp(pr.host)!
+		tcp.write(pr.build_proxy_headers(host).bytes())!
+		mut bf := []u8{len: 4096}
+		tcp.read(mut bf)!
+		if !bf.bytestr().contains('HTTP/1.1 200') {
+			return error('ssl dial error: ${bf.bytestr()}')
+		}
+
+		mut ssl_conn := ssl.new_ssl_conn(
+			verify: ''
+			cert: ''
+			cert_key: ''
+			validate: false
+			in_memory_verification: false
+		)!
+		ssl_conn.connect(mut tcp, host.all_before_last(':'))!
+		return ssl_conn
+	} else if pr.scheme == 'socks5' {
+		return socks.socks5_ssl_dial(pr.host, host, pr.username, pr.password)!
+	} else {
+		return error('http_proxy ssl_dial: invalid proxy scheme')
 	}
-	mut bytes := req.read_all_from_client_connection(client)!
-	client.close()!
-	response_text := bytes.bytestr()
-	$if trace_http_response ? {
-		eprintln('< ${response_text}')
-	}
-	if req.on_finish != unsafe { nil } {
-		req.on_finish(req, u64(response_text.len))!
-	}
-	return parse_response(response_text)
 }

vlib/net/http/http_proxy_test.v

@@ -33,22 +33,18 @@ fn test_proxy_fields() ? {
 
 fn test_proxy_headers() ? {
 	sample_proxy := new_http_proxy(http.sample_proxy_url)!
-	headers := sample_proxy.build_proxy_headers(http.sample_request, http.sample_host,
-		http.sample_path)
+	headers := sample_proxy.build_proxy_headers(http.sample_host)
 
-	assert headers == 'GET ${http.sample_request.url}${http.sample_path} HTTP/1.1\r\n' +
-		'Host: ${http.sample_host}\r\n' + 'User-Agent: ${http.sample_request.user_agent}\r\n' +
-		'Upgrade-Insecure-Requests: 1\r\n\r\n'
+	assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' +
+		'Proxy-Connection: Keep-Alive\r\n\r\n'
 }
 
 fn test_proxy_headers_authenticated() ? {
 	sample_proxy := new_http_proxy(http.sample_auth_proxy_url)!
-	headers := sample_proxy.build_proxy_headers(http.sample_request, http.sample_host,
-		http.sample_path)
+	headers := sample_proxy.build_proxy_headers(http.sample_host)
 
 	auth_token := base64.encode(('${sample_proxy.username}:' + '${sample_proxy.password}').bytes())
 
-	assert headers == 'GET ${http.sample_request.url}${http.sample_path} HTTP/1.1\r\n' +
-		'Host: ${http.sample_host}\r\n' + 'User-Agent: ${http.sample_request.user_agent}\r\n' +
-		'Upgrade-Insecure-Requests: 1\r\n' + 'Proxy-Authorization: Basic ${auth_token}\r\n\r\n'
+	assert headers == 'CONNECT 127.0.0.1:1337 HTTP/1.1\r\n' + 'Host: 127.0.0.1\r\n' +
+		'Proxy-Connection: Keep-Alive\r\nProxy-Authorization: Basic ${auth_token}\r\n\r\n'
 }

vlib/net/http/request.v

@@ -148,7 +148,7 @@ fn (req &Request) method_and_url_to_response(method Method, url urllib.URL) !Res
 	} else if req.proxy != unsafe { nil } {
 		mut retries := 0
 		for {
-			res := req.proxy.http_do(host_name, method, path, req) or {
+			res := req.proxy.http_do(url, method, path, req) or {
 				retries++
 				if is_no_need_retry_error(err.code()) || retries >= req.max_retries {
 					return err

vlib/net/socks/socks5.v

@@ -0,0 +1,162 @@
+module socks
+
+import net.ssl
+import net
+
+const socks_version5 = u8(5)
+
+const addr_type_ipv4 = u8(1)
+
+const addr_type_fqdn = u8(3)
+
+const addr_type_ipv6 = u8(4)
+
+const no_auth = u8(0)
+
+const auth_user_password = u8(2)
+
+// socks5_dial create new instance of &net.TcpConn
+pub fn socks5_dial(proxy_url string, host string, username string, password string) !&net.TcpConn {
+	mut con := net.dial_tcp(proxy_url)!
+	return handshake(mut con, host, username, password)!
+}
+
+// socks5_ssl_dial create new instance of &ssl.SSLConn
+pub fn socks5_ssl_dial(proxy_url string, host string, username string, password string) !&ssl.SSLConn {
+	mut ssl_conn := ssl.new_ssl_conn(
+		verify: ''
+		cert: ''
+		cert_key: ''
+		validate: false
+		in_memory_verification: false
+	)!
+	mut con := socks5_dial(proxy_url, host, username, password)!
+	ssl_conn.connect(mut con, host.all_before_last(':')) or { panic(err) }
+	return ssl_conn
+}
+
+fn handshake(mut con net.TcpConn, host string, username string, password string) !&net.TcpConn {
+	mut v := [socks.socks_version5, 1]
+	if username.len > 0 {
+		v << socks.auth_user_password
+	} else {
+		v << socks.no_auth
+	}
+
+	con.write(v)!
+	mut bf := []u8{len: 2}
+	con.read(mut bf)!
+
+	if bf[0] != socks.socks_version5 {
+		con.close()!
+		return error('unexpected protocol version ${bf[0]}')
+	}
+	if username.len == 0 {
+		if bf[1] != 0 {
+			con.close()!
+			return error(reply(bf[1]))
+		}
+	}
+	if username.len > 0 {
+		v.clear()
+		v << u8(1)
+		v << u8(username.len)
+		v << username.bytes()
+		v << u8(password.len)
+		v << password.bytes()
+
+		con.write(v)!
+		mut resp := []u8{len: 2}
+		con.read(mut resp)!
+
+		if resp[0] != 1 {
+			con.close()!
+			return error('server does not support user/password version 1')
+		} else if resp[1] != 0 {
+			con.close()!
+			return error('user/password login failed')
+		}
+	}
+	v.clear()
+	v = [socks.socks_version5, 1, 0]
+
+	mut port := host.all_after_last(':').u64()
+	if port == 0 {
+		port = u64(80)
+	}
+	address := host.all_before_last(':')
+
+	if address.contains_only('.1234567890') { // ipv4
+		v << socks.addr_type_ipv4
+		v << parse_ipv4(address)!
+	} else if address.contains_only(':1234567890abcdf') {
+		// v << addr_type_ipv6
+		// v << parse_ipv4(address)!
+		// todo support ipv6
+	} else { // domain
+		if address.len > 255 {
+			return error('${address} is too long')
+		} else {
+			v << socks.addr_type_fqdn
+			v << u8(address.len)
+			v << address.bytes()
+		}
+	}
+	v << u8(port >> 8)
+	v << u8(port)
+
+	con.write(v)!
+
+	mut bff := []u8{len: v.len}
+
+	con.read(mut bff)!
+	if bff[1] != 0 {
+		con.close()!
+		return error(reply(bff[1]))
+	}
+	return con
+}
+
+fn reply(code u8) string {
+	match code {
+		0 {
+			return 'succeeded'
+		}
+		1 {
+			return 'general SOCKS server failure'
+		}
+		2 {
+			return 'connection not allowed by ruleset'
+		}
+		3 {
+			return 'network unreachable'
+		}
+		4 {
+			return 'host unreachable'
+		}
+		5 {
+			return 'connection refused'
+		}
+		6 {
+			return 'TTL expired'
+		}
+		7 {
+			return 'command not supported'
+		}
+		8 {
+			return 'address type not supported'
+		}
+		else {
+			return 'unknown code: ${code}'
+		}
+	}
+}
+
+fn parse_ipv4(addr string) ![]u8 {
+	mut ip := []u8{}
+	for part in addr.split('.') {
+		ip << part.u8()
+	}
+
+	return ip
+}

vlib/net/socks/socks5_test.v

@@ -0,0 +1,19 @@
+module socks
+
+fn ipv4_socks() ! {
+	mut v := socks5_dial('127.0.0.1:9150', '1.1.1.1:80', '', '')!
+	assert v != unsafe { nil }
+	v.close()!
+}
+
+fn domain_socks() ! {
+	mut v := socks5_dial('127.0.0.1:9150', 'ifconfig.info:80', '', '')!
+	assert v != unsafe { nil }
+	v.close()!
+}
+
+fn test_parse_ipv4() {
+	assert parse_ipv4('255.255.255.255')! == [u8(255), 255, 255, 255]
+	assert parse_ipv4('127.0.0.1')! == [u8(127), 0, 0, 1]
+	parse_ipv4('1.2..2') or { assert err.msg() == 'Ip address not valid' }
+}