remove Proxy-Raw-Headers

lyrebird/mitm/mitm_script.py

@@ -55,8 +55,6 @@ def to_mock_server(flow: http.HTTPFlow):
 
     flow.request.headers['Lyrebird-Client-Address'] = address
     flow.request.headers['Mitmproxy-Proxy'] = address
-    flow.request.headers['Proxy-Raw-Headers'] = json.dumps({name: flow.request.headers[name]
-                                                           for name in flow.request.headers if name.lower() not in ('host', 'proxy-raw-headers')}, ensure_ascii=False)
 
 
 def request(flow: http.HTTPFlow):

lyrebird/mock/extra_mock_server/server.py

@@ -26,25 +26,13 @@ def is_filtered(context: LyrebirdProxyContext):
 
     allow list like
     '''
-    global lb_config
     filters = lb_config.get('proxy.filters', [])
     for _filter in filters:
         if re.search(_filter, context.origin_url):
             return True
     return False
 
 
-def make_raw_headers_line(request: web.Request):
-    raw_headers = {}
-    for k, v in request.raw_headers:
-        raw_header_name = k.decode()
-        raw_header_value = v.decode()
-        if raw_header_name.lower() in ['cache-control', 'host', 'transfer-encoding', 'proxy-raw-headers']:
-            continue
-        raw_headers[raw_header_name] = raw_header_value
-    return json.dumps(raw_headers, ensure_ascii=False)
-
-
 def upgrade_request_report(context: LyrebirdProxyContext):
     if not context.request.headers.get('upgrade'):
         return
@@ -58,13 +46,9 @@ def make_request_headers(context: LyrebirdProxyContext, is_proxy):
     headers = {k: v for k, v in context.request.headers.items() if k.lower() not in [
             'cache-control', 'host', 'transfer-encoding']}
     if is_proxy:
-        if 'Proxy-Raw-Headers' in context.request.headers:
-            del headers['Proxy-Raw-Headers']
         if 'Lyrebird-Client-Address' in context.request.headers:
             del headers['Lyrebird-Client-Address']
     else:
-        if 'Proxy-Raw-Headers' not in context.request.headers:
-            headers['Proxy-Raw-Headers'] = make_raw_headers_line(context.request)
         if 'Lyrebird-Client-Address' not in context.request.headers:
             headers['Lyrebird-Client-Address'] = context.request.remote
     return headers

lyrebird/mock/handlers/handler_context.py

@@ -65,7 +65,10 @@ def _parse_request(self):
 
         raw_headers = None
         # Read raw headers
-        if 'Proxy-Raw-Headers' in self.request.headers:
+        if '_raw_header' in self.request.environ:
+            raw_headers = HeadersHelper.get_raw_headers(self.request)
+        # Proxy-Raw-Headers will be removed in future
+        elif 'Proxy-Raw-Headers' in self.request.headers:
             raw_headers = json.loads(self.request.headers['Proxy-Raw-Headers'])
 
         # parse path

lyrebird/mock/handlers/http_data_helper/__init__.py

@@ -1,5 +1,6 @@
 from collections import OrderedDict
 from . import content_encoding, content_type
+from ..http_header_helper import HeadersHelper
 from lyrebird.utils import CaseInsensitiveDict
 import json
 
@@ -26,7 +27,9 @@ def origin2flow(origin_obj, output=None, chain=None):
             return
 
         # Read raw headers, support the request from extra mock 9999 port
-        if 'Proxy-Raw-Headers' in origin_obj.headers:
+        if '_raw_header' in origin_obj.environ:
+            raw_headers = HeadersHelper.get_raw_headers(origin_obj)
+        elif 'Proxy-Raw-Headers' in origin_obj.headers:
             _origin_headers = json.loads(origin_obj.headers['Proxy-Raw-Headers'])
             raw_headers = CaseInsensitiveDict(_origin_headers)
         else:
@@ -70,7 +73,9 @@ def origin2string(origin_obj, output=None):
             return
 
         # Read raw headers, support the request from extra mock 9999 port
-        if 'Proxy-Raw-Headers' in origin_obj.headers:
+        if '_raw_header' in origin_obj.environ:
+            raw_headers = HeadersHelper.get_raw_headers(origin_obj)
+        elif 'Proxy-Raw-Headers' in origin_obj.headers:
             _origin_headers = json.loads(origin_obj.headers['Proxy-Raw-Headers'])
             raw_headers = CaseInsensitiveDict(_origin_headers)
         else:

lyrebird/mock/handlers/http_header_helper/__init__.py

@@ -1,6 +1,7 @@
 from collections import OrderedDict
 from .content_length import ContentLengthHandler
 from ..duplicate_header_key_handler import DuplicateHeaderKeyHandler
+from lyrebird.utils import CaseInsensitiveDict
 
 origin2flow_handlers = OrderedDict({
 })
@@ -42,3 +43,12 @@ def flow2origin(flow_obj, output=None, chain=None):
         else:
             return _headers
 
+    @staticmethod
+    def get_raw_headers(origin_request):
+        raw_headers = {}
+        if '_raw_header' in origin_request.environ:
+            raw_headers = CaseInsensitiveDict(origin_request.environ['_raw_header'])
+            for key in ('cache-control', 'host', 'transfer-encoding', 'lyrebird-client-address'):
+                if key in raw_headers:
+                    del raw_headers[key]
+        return raw_headers

lyrebird/mock/mock_server.py

@@ -5,12 +5,14 @@
 from .blueprints.ui import ui
 from .blueprints.core import core
 from flask_socketio import SocketIO
+from werkzeug.serving import WSGIRequestHandler
 from ..version import VERSION
 from lyrebird.base_server import ThreadServer
 from lyrebird import application
 from lyrebird import log
 import sys
 import traceback
+import functools
 
 """
 Mock server
@@ -121,3 +123,22 @@ def terminate(self):
         except Exception as e:
             pass
         print('CoreServer shutdown')
+
+
+def monkey_patch_wsgi_request_handler():
+    """
+    environ of Werkzeug lost some Header capitalization information when processing request headers. 
+    Although this is in compliance with RFC-7230, it may cause exceptions in some non-standard scenarios, 
+    so it is recorded here to achieve compatibility.
+    """
+    original_make_environ = WSGIRequestHandler.make_environ
+
+    @functools.wraps(original_make_environ)
+    def patched_make_environ(self):
+        environ = original_make_environ(self)
+        environ['_raw_header'] = self.headers._headers
+        return environ
+
+    WSGIRequestHandler.make_environ = patched_make_environ
+
+monkey_patch_wsgi_request_handler()

lyrebird/utils.py

@@ -347,10 +347,17 @@ class CaseInsensitiveDict(dict):
 
     def __init__(self, raw_dict=None):
         self.__key_map = {}
-        if raw_dict:
+        if not raw_dict:
+            return
+        if raw_dict and isinstance(raw_dict, dict):
             for k, v in raw_dict.items():
                 self.__setitem__(k, v)
-
+        elif raw_dict and isinstance(raw_dict, list):
+            for k, v in raw_dict:
+                self.__setitem__(k, v)
+        else:
+            raise TypeError("Unexpected type for CaseInsensitiveDict")
+
     def __getstate__(self):
         return {
             'key_map': self.__key_map,