XHTTP client: Move x_padding into Referer header (#4298)

""Breaking"": Update the server side first, then client

transport/internet/browser_dialer/dialer.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	_ "embed"
 	"encoding/base64"
+	"encoding/json"
 	"net/http"
 	"time"
 
@@ -17,6 +18,12 @@ import (
 //go:embed dialer.html
 var webpage []byte
 
+type task struct {
+	Method string `json:"method"`
+	URL    string `json:"url"`
+	Extra  any    `json:"extra,omitempty"`
+}
+
 var conns chan *websocket.Conn
 
 var upgrader = &websocket.Upgrader{
@@ -55,23 +62,69 @@ func HasBrowserDialer() bool {
 	return conns != nil
 }
 
+type webSocketExtra struct {
+	Protocol string `json:"protocol,omitempty"`
+}
+
 func DialWS(uri string, ed []byte) (*websocket.Conn, error) {
-	data := []byte("WS " + uri)
+	task := task{
+		Method: "WS",
+		URL:    uri,
+	}
+
 	if ed != nil {
-		data = append(data, " "+base64.RawURLEncoding.EncodeToString(ed)...)
+		task.Extra = webSocketExtra{
+			Protocol: base64.RawURLEncoding.EncodeToString(ed),
+		}
 	}
 
-	return dialRaw(data)
+	return dialTask(task)
 }
 
-func DialGet(uri string) (*websocket.Conn, error) {
-	data := []byte("GET " + uri)
-	return dialRaw(data)
+type httpExtra struct {
+	Referrer string            `json:"referrer,omitempty"`
+	Headers  map[string]string `json:"headers,omitempty"`
 }
 
-func DialPost(uri string, payload []byte) error {
-	data := []byte("POST " + uri)
-	conn, err := dialRaw(data)
+func httpExtraFromHeaders(headers http.Header) *httpExtra {
+	if len(headers) == 0 {
+		return nil
+	}
+
+	extra := httpExtra{}
+	if referrer := headers.Get("Referer"); referrer != "" {
+		extra.Referrer = referrer
+		headers.Del("Referer")
+	}
+
+	if len(headers) > 0 {
+		extra.Headers = make(map[string]string)
+		for header := range headers {
+			extra.Headers[header] = headers.Get(header)
+		}
+	}
+
+	return &extra
+}
+
+func DialGet(uri string, headers http.Header) (*websocket.Conn, error) {
+	task := task{
+		Method: "GET",
+		URL:    uri,
+		Extra:  httpExtraFromHeaders(headers),
+	}
+
+	return dialTask(task)
+}
+
+func DialPost(uri string, headers http.Header, payload []byte) error {
+	task := task{
+		Method: "POST",
+		URL:    uri,
+		Extra:  httpExtraFromHeaders(headers),
+	}
+
+	conn, err := dialTask(task)
 	if err != nil {
 		return err
 	}
@@ -90,7 +143,12 @@ func DialPost(uri string, payload []byte) error {
 	return nil
 }
 
-func dialRaw(data []byte) (*websocket.Conn, error) {
+func dialTask(task task) (*websocket.Conn, error) {
+	data, err := json.Marshal(task)
+	if err != nil {
+		return nil, err
+	}
+
 	var conn *websocket.Conn
 	for {
 		conn = <-conns
@@ -100,7 +158,7 @@ func dialRaw(data []byte) (*websocket.Conn, error) {
 			break
 		}
 	}
-	err := CheckOK(conn)
+	err = CheckOK(conn)
 	if err != nil {
 		return nil, err
 	}

transport/internet/browser_dialer/dialer.html

@@ -14,10 +14,28 @@
 		let upstreamGetCount = 0;
 		let upstreamWsCount = 0;
 		let upstreamPostCount = 0;
+
+		function prepareRequestInit(extra) {
+			const requestInit = {};
+			if (extra.referrer) {
+				// note: we have to strip the protocol and host part.
+				// Browsers disallow that, and will reset the value to current page if attempted.
+				const referrer = URL.parse(extra.referrer);
+				requestInit.referrer = referrer.pathname + referrer.search + referrer.hash;
+				requestInit.referrerPolicy = "unsafe-url";
+			}
+
+			if (extra.headers) {
+				requestInit.headers = extra.headers;
+			}
+
+			return requestInit;
+		}
+
 		let check = function () {
 			if (clientIdleCount > 0) {
 				return;
-			};
+			}
 			clientIdleCount += 1;
 			console.log("Prepare", url);
 			let ws = new WebSocket(url);
@@ -29,12 +47,12 @@
 			// double-checking that this continues to work
 			ws.onmessage = function (event) {
 				clientIdleCount -= 1;
-				let [method, url, protocol] = event.data.split(" ");
-				switch (method) {
+				let task = JSON.parse(event.data);
+				switch (task.method) {
 					case "WS": {
 						upstreamWsCount += 1;
-						console.log("Dial WS", url, protocol);
-						const wss = new WebSocket(url, protocol);
+						console.log("Dial WS", task.url, task.extra.protocol);
+						const wss = new WebSocket(task.url, task.extra.protocol);
 						wss.binaryType = "arraybuffer";
 						let opened = false;
 						ws.onmessage = function (event) {
@@ -60,10 +78,12 @@
 							wss.close()
 						};
 						break;
-					};
+					}
 					case "GET": {
 						(async () => {
-							console.log("Dial GET", url);
+							const requestInit = prepareRequestInit(task.extra);
+
+							console.log("Dial GET", task.url);
 							ws.send("ok");
 							const controller = new AbortController();
 
@@ -83,58 +103,62 @@
 							ws.onclose = (event) => {
 								try {
 									reader && reader.cancel();
-								} catch(e) {};
+								} catch(e) {}
 
 								try {
 									controller.abort();
-								} catch(e) {};
+								} catch(e) {}
 							};
 
 							try {
 								upstreamGetCount += 1;
-								const response = await fetch(url, {signal: controller.signal});
+
+								requestInit.signal = controller.signal;
+								const response = await fetch(task.url, requestInit);
 
 								const body = await response.body;
 								reader = body.getReader();
 
 								while (true) {
 									const { done, value } = await reader.read();
-									ws.send(value);
+									if (value) ws.send(value);  // don't send back "undefined" string when received nothing
 									if (done) break;
-								};
+								}
 							} finally {
 								upstreamGetCount -= 1;
 								console.log("Dial GET DONE, remaining: ", upstreamGetCount);
 								ws.close();
-							};
+							}
 						})();
 						break;
-					};
+					}
 					case "POST": {
 						upstreamPostCount += 1;
-						console.log("Dial POST", url);
+
+						const requestInit = prepareRequestInit(task.extra);
+						requestInit.method = "POST";
+
+						console.log("Dial POST", task.url);
 						ws.send("ok");
 						ws.onmessage = async (event) => {
 							try {
-								const response = await fetch(
-									url,
-									{method: "POST", body: event.data}
-								);
+								requestInit.body = event.data;
+								const response = await fetch(task.url, requestInit);
 								if (response.ok) {
 									ws.send("ok");
 								} else {
 									console.error("bad status code");
 									ws.send("fail");
-								};
+								}
 							} finally {
 								upstreamPostCount -= 1;
 								console.log("Dial POST DONE, remaining: ", upstreamPostCount);
 								ws.close();
-							};
+							}
 						};
 						break;
-					};
-				};
+					}
+				}
 
 				check();
 			};

transport/internet/splithttp/browser_client.go

@@ -5,24 +5,26 @@ import (
 	"io"
 	gonet "net"
 
+	"github.com/xtls/xray-core/common/errors"
 	"github.com/xtls/xray-core/transport/internet/browser_dialer"
 	"github.com/xtls/xray-core/transport/internet/websocket"
 )
 
-// implements splithttp.DialerClient in terms of browser dialer
-// has no fields because everything is global state :O)
-type BrowserDialerClient struct{}
+// BrowserDialerClient implements splithttp.DialerClient in terms of browser dialer
+type BrowserDialerClient struct {
+	transportConfig *Config
+}
 
 func (c *BrowserDialerClient) IsClosed() bool {
 	panic("not implemented yet")
 }
 
 func (c *BrowserDialerClient) OpenStream(ctx context.Context, url string, body io.Reader, uploadOnly bool) (io.ReadCloser, gonet.Addr, gonet.Addr, error) {
 	if body != nil {
-		panic("not implemented yet")
+		return nil, nil, nil, errors.New("bidirectional streaming for browser dialer not implemented yet")
 	}
 
-	conn, err := browser_dialer.DialGet(url)
+	conn, err := browser_dialer.DialGet(url, c.transportConfig.GetRequestHeader())
 	dummyAddr := &gonet.IPAddr{}
 	if err != nil {
 		return nil, dummyAddr, dummyAddr, err
@@ -37,7 +39,7 @@ func (c *BrowserDialerClient) PostPacket(ctx context.Context, url string, body i
 		return err
 	}
 
-	err = browser_dialer.DialPost(url, bytes)
+	err = browser_dialer.DialPost(url, c.transportConfig.GetRequestHeader(), bytes)
 	if err != nil {
 		return err
 	}

transport/internet/splithttp/config.go

@@ -4,13 +4,16 @@ import (
 	"crypto/rand"
 	"math/big"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/xtls/xray-core/common"
 	"github.com/xtls/xray-core/core"
 	"github.com/xtls/xray-core/transport/internet"
 )
 
+const paddingQuery = "x_padding"
+
 func (c *Config) GetNormalizedPath() string {
 	pathAndQuery := strings.SplitN(c.Path, "?", 2)
 	path := pathAndQuery[0]
@@ -39,11 +42,6 @@ func (c *Config) GetNormalizedQuery() string {
 	}
 	query += "x_version=" + core.Version()
 
-	paddingLen := c.GetNormalizedXPaddingBytes().rand()
-	if paddingLen > 0 {
-		query += "&x_padding=" + strings.Repeat("0", int(paddingLen))
-	}
-
 	return query
 }
 
@@ -53,6 +51,28 @@ func (c *Config) GetRequestHeader() http.Header {
 		header.Add(k, v)
 	}
 
+	paddingLen := c.GetNormalizedXPaddingBytes().rand()
+	if paddingLen > 0 {
+		query, err := url.ParseQuery(c.GetNormalizedQuery())
+		if err != nil {
+			query = url.Values{}
+		}
+		// https://www.rfc-editor.org/rfc/rfc7541.html#appendix-B
+		// h2's HPACK Header Compression feature employs a huffman encoding using a static table.
+		// 'X' is assigned an 8 bit code, so HPACK compression won't change actual padding length on the wire.
+		// https://www.rfc-editor.org/rfc/rfc9204.html#section-4.1.2-2
+		// h3's similar QPACK feature uses the same huffman table.
+		query.Set(paddingQuery, strings.Repeat("X", int(paddingLen)))
+
+		referrer := url.URL{
+			Scheme:   "https", // maybe http actually, but this part is not being checked
+			Host:     c.Host,
+			Path:     c.GetNormalizedPath(),
+			RawQuery: query.Encode(),
+		}
+
+		header.Set("Referer", referrer.String())
+	}
 	return header
 }
 
@@ -63,7 +83,7 @@ func (c *Config) WriteResponseHeader(writer http.ResponseWriter) {
 	writer.Header().Set("X-Version", core.Version())
 	paddingLen := c.GetNormalizedXPaddingBytes().rand()
 	if paddingLen > 0 {
-		writer.Header().Set("X-Padding", strings.Repeat("0", int(paddingLen)))
+		writer.Header().Set("X-Padding", strings.Repeat("X", int(paddingLen)))
 	}
 }