net.http: support passing an HTTP proxy server in http.fetch (#19606)

vlib/net/http/http.v

@@ -23,6 +23,7 @@ pub mut:
 	user_agent string  = 'v.http'
 	user_ptr   voidptr = unsafe { nil }
 	verbose    bool
+	proxy      &HttpProxy = unsafe { nil }
 	//
 	validate               bool   // set this to true, if you want to stop requests, when their certificates are found to be invalid
 	verify                 string // the path to a rootca.pem file, containing trusted CA certificate(s)
@@ -159,6 +160,7 @@ pub fn fetch(config FetchConfig) !Response {
 		validate: config.validate
 		verify: config.verify
 		cert: config.cert
+		proxy: config.proxy
 		cert_key: config.cert_key
 		in_memory_verification: config.in_memory_verification
 		allow_redirect: config.allow_redirect

vlib/net/http/http_proxy.v

@@ -0,0 +1,110 @@
+module http
+
+import net.urllib
+import encoding.base64
+import net
+
+[heap]
+struct HttpProxy {
+mut:
+	scheme   string
+	username string
+	password string
+	host     string
+	hostname string
+	port     int
+	url      string
+}
+
+// new_http_proxy creates a new HttpProxy instance, from the given http proxy url in `raw_url`
+pub fn new_http_proxy(raw_url string) !&HttpProxy {
+	mut url := urllib.parse(raw_url) or { return error('malformed proxy url') }
+	scheme := url.scheme
+
+	if scheme != 'http' && scheme != 'https' {
+		return error('invalid scheme')
+	}
+
+	url.path = ''
+	url.raw_path = ''
+	url.raw_query = ''
+	url.fragment = ''
+
+	str_url := url.str()
+
+	mut host := url.host
+	mut port := url.port().int()
+
+	if port == 0 {
+		if scheme == 'https' {
+			port = 443
+		} else if scheme == 'http' {
+			port = 80
+		}
+
+		host += ':' + port.str()
+	}
+
+	return &HttpProxy{
+		scheme: scheme
+		username: url.user.username
+		password: url.user.password
+		host: host
+		hostname: url.hostname()
+		port: port
+		url: str_url
+	}
+}
+
+fn (proxy &HttpProxy) build_proxy_headers(request &Request, host string, path string) string {
+	ua := request.user_agent
+	mut uheaders := []string{}
+
+	uheaders << 'Host: ${host}\r\n'
+	uheaders << 'User-Agent: ${ua}\r\n'
+	uheaders << 'Upgrade-Insecure-Requests: 1\r\n'
+	// uheaders << 'Proxy-Connection: Keep-Alive\r\n'
+
+	if proxy.username != '' {
+		mut authinfo := ''
+
+		authinfo += proxy.username
+		if proxy.password != '' {
+			authinfo += ':${proxy.password}'
+		}
+
+		encoded_authinfo := base64.encode(authinfo.bytes())
+
+		uheaders << 'Proxy-Authorization: Basic ${encoded_authinfo}\r\n'
+	}
+
+	version := Version.v1_1
+
+	// we always make requests on http scheme
+	url := 'http://${host}${path}'
+
+	return '${request.method} ${url} ${version}\r\n' + uheaders.join('') + '\r\n'
+}
+
+fn (proxy &HttpProxy) http_do(host string, method Method, path string, req &Request) !Response {
+	host_name, _ := net.split_address(host)!
+	s := proxy.build_proxy_headers(req, host_name, path)
+	mut client := net.dial_tcp(proxy.host)!
+	client.set_read_timeout(req.read_timeout)
+	client.set_write_timeout(req.write_timeout)
+	// TODO this really needs to be exposed somehow
+	client.write_string(s)!
+	$if trace_http_request ? {
+		eprintln('> ${s}')
+	}
+	mut bytes := req.read_all_from_client_connection(client)!
+	client.close()!
+	response_text := bytes.bytestr()
+	$if trace_http_response ? {
+		eprintln('< ${response_text}')
+	}
+	if req.on_finish != unsafe { nil } {
+		req.on_finish(req, u64(response_text.len))!
+	}
+	return parse_response(response_text)
+}

vlib/net/http/http_proxy_test.v

@@ -0,0 +1,54 @@
+module http
+
+import encoding.base64
+
+const (
+	sample_proxy_url      = 'https://localhost'
+	sample_auth_proxy_url = 'http://user:pass@localhost:8888'
+
+	sample_host           = '127.0.0.1:1337'
+	sample_request        = &Request{
+		url: 'http://${sample_host}'
+	}
+	sample_path           = '/'
+)
+
+fn test_proxy_fields() ? {
+	sample_proxy := new_http_proxy(http.sample_proxy_url)!
+	sample_auth_proxy := new_http_proxy(http.sample_auth_proxy_url)!
+
+	assert sample_proxy.scheme == 'https'
+	assert sample_proxy.host == 'localhost:443'
+	assert sample_proxy.hostname == 'localhost'
+	assert sample_proxy.port == 443
+	assert sample_proxy.url == http.sample_proxy_url
+	assert sample_auth_proxy.scheme == 'http'
+	assert sample_auth_proxy.username == 'user'
+	assert sample_auth_proxy.password == 'pass'
+	assert sample_auth_proxy.host == 'localhost:8888'
+	assert sample_auth_proxy.hostname == 'localhost'
+	assert sample_auth_proxy.port == 8888
+	assert sample_auth_proxy.url == http.sample_auth_proxy_url
+}
+
+fn test_proxy_headers() ? {
+	sample_proxy := new_http_proxy(http.sample_proxy_url)!
+	headers := sample_proxy.build_proxy_headers(http.sample_request, http.sample_host,
+		http.sample_path)
+
+	assert headers == 'GET ${http.sample_request.url}${http.sample_path} HTTP/1.1\r\n' +
+		'Host: ${http.sample_host}\r\n' + 'User-Agent: ${http.sample_request.user_agent}\r\n' +
+		'Upgrade-Insecure-Requests: 1\r\n\r\n'
+}
+
+fn test_proxy_headers_authenticated() ? {
+	sample_proxy := new_http_proxy(http.sample_auth_proxy_url)!
+	headers := sample_proxy.build_proxy_headers(http.sample_request, http.sample_host,
+		http.sample_path)
+
+	auth_token := base64.encode(('${sample_proxy.username}:' + '${sample_proxy.password}').bytes())
+
+	assert headers == 'GET ${http.sample_request.url}${http.sample_path} HTTP/1.1\r\n' +
+		'Host: ${http.sample_host}\r\n' + 'User-Agent: ${http.sample_request.user_agent}\r\n' +
+		'Upgrade-Insecure-Requests: 1\r\n' + 'Proxy-Authorization: Basic ${auth_token}\r\n\r\n'
+}

vlib/net/http/request.v

@@ -30,6 +30,7 @@ pub mut:
 	user_agent string = 'v.http'
 	verbose    bool
 	user_ptr   voidptr
+	proxy      &HttpProxy = unsafe { nil }
 	// NOT implemented for ssl connections
 	// time = -1 for no timeout
 	read_timeout  i64 = 30 * time.second
@@ -117,14 +118,17 @@ fn (req &Request) method_and_url_to_response(method Method, url urllib.URL) !Res
 		}
 	}
 	// println('fetch $method, $scheme, $host_name, $nport, $path ')
-	if scheme == 'https' {
+	if scheme == 'https' && req.proxy == unsafe { nil } {
 		// println('ssl_do( $nport, $method, $host_name, $path )')
 		res := req.ssl_do(nport, method, host_name, path)!
 		return res
-	} else if scheme == 'http' {
+	} else if scheme == 'http' && req.proxy == unsafe { nil } {
 		// println('http_do( $nport, $method, $host_name, $path )')
 		res := req.http_do('${host_name}:${nport}', method, path)!
 		return res
+	} else if req.proxy != unsafe { nil } {
+		res := req.proxy.http_do(host_name, method, path, req)!
+		return res
 	}
 	return error('http.request.method_and_url_to_response: unsupported scheme: "${scheme}"')
 }