feat(nfproxy): add HttpHistory support for pyfilters in HTTP requests

fgex-lib/firegex/nfproxy/__init__.py

@@ -35,7 +35,8 @@ def clear_pyfilter_registry():
         pyfilter.registry.clear()
 
 __all__ = [
-    "ACCEPT", "DROP", "REJECT", "UNSTABLE_MANGLE"
+    "ACCEPT", "DROP", "REJECT", "UNSTABLE_MANGLE",
     "Action", "FullStreamAction", "pyfilter",
-    "RawPacket", "TCPInputStream", "TCPOutputStream", "TCPClientStream", "TCPServerStream"
+    "RawPacket", "TCPInputStream", "TCPOutputStream", "TCPClientStream", "TCPServerStream",
+    "HttpHistory", "HttpStreamHistory"
 ]

fgex-lib/firegex/nfproxy/models/__init__.py

@@ -11,6 +11,8 @@ from firegex.nfproxy.models.http import (
     HttpResponseHeader,
     HttpFullRequest,
     HttpFullResponse,
+    HttpHistory,
+    HttpStreamHistory,
 )
 from firegex.nfproxy.internals.data import RawPacket
 from enum import Enum
@@ -31,6 +33,8 @@ type_annotations_associations = {
         HttpResponseHeader: HttpResponseHeader._fetch_packet,
         HttpFullRequest: HttpFullRequest._fetch_packet,
         HttpFullResponse: HttpFullResponse._fetch_packet,
+        HttpHistory: HttpHistory._fetch_packet,
+        HttpStreamHistory: HttpStreamHistory._fetch_packet,
     },
 }
 
@@ -52,5 +56,8 @@ __all__ = [
     "HttpResponseHeader",
     "HttpFullRequest",
     "HttpFullResponse",
+    "HttpHistory",
+    "HttpStreamHistory",
     "Protocols",
 ]
+

fgex-lib/firegex/nfproxy/models/http.py

@@ -48,6 +48,7 @@ class InternalHTTPMessage:
     ws_stream: list[Frame] = field(default_factory=list)  # Decoded websocket stream
     upgrading_to_h2: bool = field(default=False)
     upgrading_to_ws: bool = field(default=False)
+    added_to_history: bool = field(default=False)
 
 
 @dataclass
@@ -393,6 +394,59 @@ class InternalHttpResponse(InternalCallbackHandler, pyllhttp.Response):
         return False
 
 
+class HttpHistory:
+    """
+    HTTP History handler for pyfilters.
+    Provides access to completed previous requests and responses in the current TCP stream.
+    """
+
+    def __init__(
+        self,
+        requests: list["HttpFullRequest"] | None = None,
+        responses: list["HttpFullResponse"] | None = None,
+    ):
+        self._requests = list(requests) if requests is not None else []
+        self._responses = list(responses) if responses is not None else []
+
+    @property
+    def requests(self) -> list["HttpFullRequest"]:
+        """List of previous completed HTTP requests"""
+        return self._requests
+
+    @property
+    def responses(self) -> list["HttpFullResponse"]:
+        """List of previous completed HTTP responses"""
+        return self._responses
+
+    @classmethod
+    def _fetch_packet(cls, internal_data: DataStreamCtx):
+        if (
+            internal_data.current_pkt is None
+            or internal_data.current_pkt.is_tcp is False
+        ):
+            raise NotReadyToRun()
+
+        for key in ["http_module", "http_full", "http_header"]:
+            obj = internal_data.call_mem.get(f"_fetched_obj_{key}")
+            if obj is not None:
+                if isinstance(obj, list) and len(obj) > 0:
+                    return [item.history for item in obj]
+                elif hasattr(obj, "history"):
+                    return obj.history
+
+        fetch_cls = HttpRequest if internal_data.current_pkt.is_input else HttpResponse
+        obj = fetch_cls._fetch_packet(internal_data)
+        if isinstance(obj, list):
+            return [item.history for item in obj]
+        return obj.history
+
+    def __repr__(self):
+        return f"<HttpHistory requests={len(self._requests)} responses={len(self._responses)}>"
+
+
+HttpStreamHistory = HttpHistory
+
+
 class InternalBasicHttpMetaClass:
     """Internal class to handle HTTP requests and responses"""
 
@@ -404,11 +458,19 @@ class InternalBasicHttpMetaClass:
         self._parser = parser
         self.raised_error = False
         self._message: InternalHTTPMessage | None = msg
+        self._history: HttpHistory | None = None
         self._contructor_hook()
 
     def _contructor_hook(self):
         pass
 
+    @property
+    def history(self) -> HttpHistory:
+        """HTTP History for the current stream connection"""
+        if self._history is None:
+            return HttpHistory([], [])
+        return self._history
+
     @property
     def total_size(self) -> int:
         """Total size of the stream"""
@@ -493,6 +555,10 @@ class InternalBasicHttpMetaClass:
         """Get a header from the message without caring about the case"""
         return self._message.lheaders.get(header.lower(), default)
 
+    @classmethod
+    def _should_release_message_headers(cls) -> bool:
+        return True
+
     @staticmethod
     def _before_fetch_callable_checks(internal_data: DataStreamCtx) -> bool:
         raise NotImplementedError()
@@ -521,6 +587,9 @@ class InternalBasicHttpMetaClass:
             parser: InternalHttpRequest | InternalHttpResponse = ParserType()
             internal_data.data_handler_context[parser_key] = parser
 
+        parser.release_message_headers = cls._should_release_message_headers()
+
+
         if not internal_data.call_mem.get(
             cls._parser_class(), False
         ):  # Need to parse HTTP
@@ -620,10 +689,47 @@ class InternalBasicHttpMetaClass:
 
         if messages_to_call == 0:
             raise NotReadyToRun()
-        elif messages_to_call == 1:
-            return cls(parser, messages_tosend[0])
 
-        return [cls(parser, ele) for ele in messages_tosend]
+        max_history = int(internal_data.filter_glob.get("FGEX_MAX_HISTORY_SIZE", 100))
+        req_history_deque: deque = (
+            internal_data.data_handler_context.setdefault(
+                "http_history_requests", deque(maxlen=max_history)
+            )
+        )
+        resp_history_deque: deque = (
+            internal_data.data_handler_context.setdefault(
+                "http_history_responses", deque(maxlen=max_history)
+            )
+        )
+
+        built_instances = []
+        for msg in messages_tosend:
+            history_snapshot = HttpHistory(
+                list(req_history_deque), list(resp_history_deque)
+            )
+            instance = cls(parser, msg)
+            instance._history = history_snapshot
+            built_instances.append(instance)
+
+            if msg.message_complete and not getattr(msg, "added_to_history", False):
+                msg.added_to_history = True
+                if internal_data.current_pkt.is_input:
+                    full_req = HttpFullRequest(parser, msg)
+                    full_req._history = history_snapshot
+                    req_history_deque.append(full_req)
+                else:
+                    full_resp = HttpFullResponse(parser, msg)
+                    full_resp._history = history_snapshot
+                    resp_history_deque.append(full_resp)
+
+        if len(built_instances) == 1:
+            res = built_instances[0]
+            internal_data.call_mem[f"_fetched_obj_{cls._parser_class()}"] = res
+            return res
+
+        internal_data.call_mem[f"_fetched_obj_{cls._parser_class()}"] = built_instances
+        return built_instances
+
 
 
 class HttpRequest(InternalBasicHttpMetaClass):
@@ -639,7 +745,7 @@ class HttpRequest(InternalBasicHttpMetaClass):
     @property
     def method(self) -> bytes:
         """Method of the request"""
-        return self._parser.msg.method
+        return self._message.method
 
     @staticmethod
     def _parser_class() -> str:
@@ -662,7 +768,8 @@ class HttpResponse(InternalBasicHttpMetaClass):
     @property
     def status_code(self) -> int:
         """Status code of the response"""
-        return self._parser.msg.status
+        return self._message.status
+
 
     @staticmethod
     def _parser_class() -> str:
@@ -678,6 +785,10 @@ class HttpFullRequest(HttpRequest):
     This data handler will be called when the request data is complete
     """
 
+    @classmethod
+    def _should_release_message_headers(cls) -> bool:
+        return False
+
     def _contructor_hook(self):
         self._parser.release_message_headers = False
 
@@ -695,6 +806,10 @@ class HttpFullResponse(HttpResponse):
     This data handler will be called when the response data is complete
     """
 
+    @classmethod
+    def _should_release_message_headers(cls) -> bool:
+        return False
+
     def _contructor_hook(self):
         self._parser.release_message_headers = False
 
@@ -706,6 +821,7 @@ class HttpFullResponse(HttpResponse):
         return f"<HttpFullResponse status_code={self.status_code} url={self.url} headers={self.headers} body=[{0 if not self.body else len(self.body)} bytes] http_version={self.http_version} keep_alive={self.keep_alive} should_upgrade={self.should_upgrade} headers_complete={self.headers_complete} message_complete={self.message_complete} content_length={self.content_length} stream={self.stream} ws_stream={self.ws_stream}>"
 
 
+
 class HttpRequestHeader(HttpRequest):
     """
     HTTP Request Header handler

tests/test_http_history.py

@@ -0,0 +1,146 @@
+from firegex.nfproxy import pyfilter, ACCEPT, clear_pyfilter_registry
+from firegex.nfproxy.models import (
+    HttpRequest,
+    HttpResponse,
+    HttpFullRequest,
+    HttpFullResponse,
+    HttpHistory,
+    HttpStreamHistory,
+)
+from firegex.nfproxy.internals import compile, handle_packet
+
+
+def create_packet_info(payload: bytes, is_input: bool):
+    return {
+        "data": payload,
+        "raw_packet": b"\x00" * 40 + payload,
+        "is_input": is_input,
+        "is_ipv6": False,
+        "is_tcp": True,
+        "l4_size": len(payload),
+    }
+
+
+def test_http_history_model():
+    req1 = None
+    res1 = None
+    history_obj = HttpHistory(requests=[req1], responses=[res1])
+    assert history_obj.requests == [req1]
+    assert history_obj.responses == [res1]
+    assert "HttpHistory" in repr(history_obj)
+
+
+def test_http_history_stream_execution():
+    glob = {}
+    clear_pyfilter_registry()
+
+    code = """
+from firegex.nfproxy import pyfilter, ACCEPT
+from firegex.nfproxy.models import HttpFullRequest, HttpFullResponse, HttpHistory
+
+recorded_history = []
+
+@pyfilter
+def filter_req(req: HttpFullRequest, history: HttpHistory):
+    req_urls = [r.url for r in history.requests]
+    resp_statuses = [r.status_code for r in history.responses]
+    recorded_history.append(("req", req.url, req_urls, resp_statuses))
+    return ACCEPT
+
+@pyfilter
+def filter_resp(resp: HttpFullResponse, history: HttpHistory):
+    req_urls = [r.url for r in history.requests]
+    resp_statuses = [r.status_code for r in history.responses]
+    recorded_history.append(("resp", resp.status_code, req_urls, resp_statuses))
+    return ACCEPT
+"""
+
+    glob["__firegex_pyfilter_enabled"] = ["filter_req", "filter_resp"]
+    glob["__firegex_proto"] = "http"
+    exec(code, glob, glob)
+    compile(glob)
+
+    # 1. First Request
+    req1_data = b"GET /first HTTP/1.1\r\nHost: example.com\r\n\r\n"
+    glob["__firegex_packet_info"] = create_packet_info(req1_data, is_input=True)
+    handle_packet(glob)
+
+    # 2. First Response
+    resp1_data = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK"
+    glob["__firegex_packet_info"] = create_packet_info(resp1_data, is_input=False)
+    handle_packet(glob)
+
+    # 3. Second Request
+    req2_data = b"GET /second HTTP/1.1\r\nHost: example.com\r\n\r\n"
+    glob["__firegex_packet_info"] = create_packet_info(req2_data, is_input=True)
+    handle_packet(glob)
+
+    # 4. Second Response
+    resp2_data = b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n"
+    glob["__firegex_packet_info"] = create_packet_info(resp2_data, is_input=False)
+    handle_packet(glob)
+
+    recorded = glob["recorded_history"]
+
+    # 1. First Request: history is empty
+    assert recorded[0] == ("req", "/first", [], [])
+
+    # 2. First Response: past requests contains /first
+    assert recorded[1] == ("resp", "OK", ["/first"], [])
+
+    # 3. Second Request: past requests contains [/first], past responses contains [OK]
+    assert recorded[2] == ("req", "/second", ["/first"], ["OK"])
+
+    # 4. Second Response: past requests contains [/first, /second], past responses contains [OK]
+    assert recorded[3] == ("resp", "Not Found", ["/first", "/second"], ["OK"])
+
+
+def test_http_history_attribute_on_request():
+    glob = {}
+    clear_pyfilter_registry()
+
+    code = """
+from firegex.nfproxy import pyfilter, ACCEPT
+from firegex.nfproxy.models import HttpFullRequest, HttpFullResponse
+
+history_lengths = []
+
+@pyfilter
+def filter_req(req: HttpFullRequest):
+    history_lengths.append((len(req.history.requests), len(req.history.responses)))
+    return ACCEPT
+
+@pyfilter
+def filter_resp(resp: HttpFullResponse):
+    history_lengths.append((len(resp.history.requests), len(resp.history.responses)))
+    return ACCEPT
+"""
+
+    glob["__firegex_pyfilter_enabled"] = ["filter_req", "filter_resp"]
+    glob["__firegex_proto"] = "http"
+    exec(code, glob, glob)
+    compile(glob)
+
+    # Req 1
+    glob["__firegex_packet_info"] = create_packet_info(b"GET /a HTTP/1.1\r\nHost: a\r\n\r\n", is_input=True)
+    handle_packet(glob)
+
+    # Resp 1
+    glob["__firegex_packet_info"] = create_packet_info(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", is_input=False)
+    handle_packet(glob)
+
+    # Req 2
+    glob["__firegex_packet_info"] = create_packet_info(b"GET /b HTTP/1.1\r\nHost: b\r\n\r\n", is_input=True)
+    handle_packet(glob)
+
+    lengths = glob["history_lengths"]
+    assert lengths[0] == (0, 0)
+    assert lengths[1] == (1, 0)
+    assert lengths[2] == (1, 1)
+
+
+if __name__ == "__main__":
+    test_http_history_model()
+    test_http_history_stream_execution()
+    test_http_history_attribute_on_request()
+    print("All HTTP history unit tests passed successfully!")