diff --git a/fgex-lib/firegex/nfproxy/__init__.py b/fgex-lib/firegex/nfproxy/__init__.py index ae5d3e8..99ad65a 100644 --- a/fgex-lib/firegex/nfproxy/__init__.py +++ b/fgex-lib/firegex/nfproxy/__init__.py @@ -35,7 +35,8 @@ def clear_pyfilter_registry(): pyfilter.registry.clear() __all__ = [ - "ACCEPT", "DROP", "REJECT", "UNSTABLE_MANGLE" + "ACCEPT", "DROP", "REJECT", "UNSTABLE_MANGLE", "Action", "FullStreamAction", "pyfilter", - "RawPacket", "TCPInputStream", "TCPOutputStream", "TCPClientStream", "TCPServerStream" + "RawPacket", "TCPInputStream", "TCPOutputStream", "TCPClientStream", "TCPServerStream", + "HttpHistory", "HttpStreamHistory" ] \ No newline at end of file diff --git a/fgex-lib/firegex/nfproxy/models/__init__.py b/fgex-lib/firegex/nfproxy/models/__init__.py index c990fcf..c5dffb2 100644 --- a/fgex-lib/firegex/nfproxy/models/__init__.py +++ b/fgex-lib/firegex/nfproxy/models/__init__.py @@ -11,6 +11,8 @@ from firegex.nfproxy.models.http import ( HttpResponseHeader, HttpFullRequest, HttpFullResponse, + HttpHistory, + HttpStreamHistory, ) from firegex.nfproxy.internals.data import RawPacket from enum import Enum @@ -31,6 +33,8 @@ type_annotations_associations = { HttpResponseHeader: HttpResponseHeader._fetch_packet, HttpFullRequest: HttpFullRequest._fetch_packet, HttpFullResponse: HttpFullResponse._fetch_packet, + HttpHistory: HttpHistory._fetch_packet, + HttpStreamHistory: HttpStreamHistory._fetch_packet, }, } @@ -52,5 +56,8 @@ __all__ = [ "HttpResponseHeader", "HttpFullRequest", "HttpFullResponse", + "HttpHistory", + "HttpStreamHistory", "Protocols", ] + diff --git a/fgex-lib/firegex/nfproxy/models/http.py b/fgex-lib/firegex/nfproxy/models/http.py index 305f694..39f3f26 100644 --- a/fgex-lib/firegex/nfproxy/models/http.py +++ b/fgex-lib/firegex/nfproxy/models/http.py @@ -48,6 +48,7 @@ class InternalHTTPMessage: ws_stream: list[Frame] = field(default_factory=list) # Decoded websocket stream upgrading_to_h2: bool = field(default=False) upgrading_to_ws: bool = field(default=False) + added_to_history: bool = field(default=False) @dataclass @@ -393,6 +394,59 @@ class InternalHttpResponse(InternalCallbackHandler, pyllhttp.Response): return False +class HttpHistory: + """ + HTTP History handler for pyfilters. + Provides access to completed previous requests and responses in the current TCP stream. + """ + + def __init__( + self, + requests: list["HttpFullRequest"] | None = None, + responses: list["HttpFullResponse"] | None = None, + ): + self._requests = list(requests) if requests is not None else [] + self._responses = list(responses) if responses is not None else [] + + @property + def requests(self) -> list["HttpFullRequest"]: + """List of previous completed HTTP requests""" + return self._requests + + @property + def responses(self) -> list["HttpFullResponse"]: + """List of previous completed HTTP responses""" + return self._responses + + @classmethod + def _fetch_packet(cls, internal_data: DataStreamCtx): + if ( + internal_data.current_pkt is None + or internal_data.current_pkt.is_tcp is False + ): + raise NotReadyToRun() + + for key in ["http_module", "http_full", "http_header"]: + obj = internal_data.call_mem.get(f"_fetched_obj_{key}") + if obj is not None: + if isinstance(obj, list) and len(obj) > 0: + return [item.history for item in obj] + elif hasattr(obj, "history"): + return obj.history + + fetch_cls = HttpRequest if internal_data.current_pkt.is_input else HttpResponse + obj = fetch_cls._fetch_packet(internal_data) + if isinstance(obj, list): + return [item.history for item in obj] + return obj.history + + def __repr__(self): + return f"" + + +HttpStreamHistory = HttpHistory + + class InternalBasicHttpMetaClass: """Internal class to handle HTTP requests and responses""" @@ -404,11 +458,19 @@ class InternalBasicHttpMetaClass: self._parser = parser self.raised_error = False self._message: InternalHTTPMessage | None = msg + self._history: HttpHistory | None = None self._contructor_hook() def _contructor_hook(self): pass + @property + def history(self) -> HttpHistory: + """HTTP History for the current stream connection""" + if self._history is None: + return HttpHistory([], []) + return self._history + @property def total_size(self) -> int: """Total size of the stream""" @@ -493,6 +555,10 @@ class InternalBasicHttpMetaClass: """Get a header from the message without caring about the case""" return self._message.lheaders.get(header.lower(), default) + @classmethod + def _should_release_message_headers(cls) -> bool: + return True + @staticmethod def _before_fetch_callable_checks(internal_data: DataStreamCtx) -> bool: raise NotImplementedError() @@ -521,6 +587,9 @@ class InternalBasicHttpMetaClass: parser: InternalHttpRequest | InternalHttpResponse = ParserType() internal_data.data_handler_context[parser_key] = parser + parser.release_message_headers = cls._should_release_message_headers() + + if not internal_data.call_mem.get( cls._parser_class(), False ): # Need to parse HTTP @@ -620,10 +689,47 @@ class InternalBasicHttpMetaClass: if messages_to_call == 0: raise NotReadyToRun() - elif messages_to_call == 1: - return cls(parser, messages_tosend[0]) - return [cls(parser, ele) for ele in messages_tosend] + max_history = int(internal_data.filter_glob.get("FGEX_MAX_HISTORY_SIZE", 100)) + req_history_deque: deque = ( + internal_data.data_handler_context.setdefault( + "http_history_requests", deque(maxlen=max_history) + ) + ) + resp_history_deque: deque = ( + internal_data.data_handler_context.setdefault( + "http_history_responses", deque(maxlen=max_history) + ) + ) + + built_instances = [] + for msg in messages_tosend: + history_snapshot = HttpHistory( + list(req_history_deque), list(resp_history_deque) + ) + instance = cls(parser, msg) + instance._history = history_snapshot + built_instances.append(instance) + + if msg.message_complete and not getattr(msg, "added_to_history", False): + msg.added_to_history = True + if internal_data.current_pkt.is_input: + full_req = HttpFullRequest(parser, msg) + full_req._history = history_snapshot + req_history_deque.append(full_req) + else: + full_resp = HttpFullResponse(parser, msg) + full_resp._history = history_snapshot + resp_history_deque.append(full_resp) + + if len(built_instances) == 1: + res = built_instances[0] + internal_data.call_mem[f"_fetched_obj_{cls._parser_class()}"] = res + return res + + internal_data.call_mem[f"_fetched_obj_{cls._parser_class()}"] = built_instances + return built_instances + class HttpRequest(InternalBasicHttpMetaClass): @@ -639,7 +745,7 @@ class HttpRequest(InternalBasicHttpMetaClass): @property def method(self) -> bytes: """Method of the request""" - return self._parser.msg.method + return self._message.method @staticmethod def _parser_class() -> str: @@ -662,7 +768,8 @@ class HttpResponse(InternalBasicHttpMetaClass): @property def status_code(self) -> int: """Status code of the response""" - return self._parser.msg.status + return self._message.status + @staticmethod def _parser_class() -> str: @@ -678,6 +785,10 @@ class HttpFullRequest(HttpRequest): This data handler will be called when the request data is complete """ + @classmethod + def _should_release_message_headers(cls) -> bool: + return False + def _contructor_hook(self): self._parser.release_message_headers = False @@ -695,6 +806,10 @@ class HttpFullResponse(HttpResponse): This data handler will be called when the response data is complete """ + @classmethod + def _should_release_message_headers(cls) -> bool: + return False + def _contructor_hook(self): self._parser.release_message_headers = False @@ -706,6 +821,7 @@ class HttpFullResponse(HttpResponse): return f"" + class HttpRequestHeader(HttpRequest): """ HTTP Request Header handler diff --git a/tests/test_http_history.py b/tests/test_http_history.py new file mode 100644 index 0000000..0f1cc9e --- /dev/null +++ b/tests/test_http_history.py @@ -0,0 +1,146 @@ +from firegex.nfproxy import pyfilter, ACCEPT, clear_pyfilter_registry +from firegex.nfproxy.models import ( + HttpRequest, + HttpResponse, + HttpFullRequest, + HttpFullResponse, + HttpHistory, + HttpStreamHistory, +) +from firegex.nfproxy.internals import compile, handle_packet + + +def create_packet_info(payload: bytes, is_input: bool): + return { + "data": payload, + "raw_packet": b"\x00" * 40 + payload, + "is_input": is_input, + "is_ipv6": False, + "is_tcp": True, + "l4_size": len(payload), + } + + +def test_http_history_model(): + req1 = None + res1 = None + history_obj = HttpHistory(requests=[req1], responses=[res1]) + assert history_obj.requests == [req1] + assert history_obj.responses == [res1] + assert "HttpHistory" in repr(history_obj) + + +def test_http_history_stream_execution(): + glob = {} + clear_pyfilter_registry() + + code = """ +from firegex.nfproxy import pyfilter, ACCEPT +from firegex.nfproxy.models import HttpFullRequest, HttpFullResponse, HttpHistory + +recorded_history = [] + +@pyfilter +def filter_req(req: HttpFullRequest, history: HttpHistory): + req_urls = [r.url for r in history.requests] + resp_statuses = [r.status_code for r in history.responses] + recorded_history.append(("req", req.url, req_urls, resp_statuses)) + return ACCEPT + +@pyfilter +def filter_resp(resp: HttpFullResponse, history: HttpHistory): + req_urls = [r.url for r in history.requests] + resp_statuses = [r.status_code for r in history.responses] + recorded_history.append(("resp", resp.status_code, req_urls, resp_statuses)) + return ACCEPT +""" + + glob["__firegex_pyfilter_enabled"] = ["filter_req", "filter_resp"] + glob["__firegex_proto"] = "http" + exec(code, glob, glob) + compile(glob) + + # 1. First Request + req1_data = b"GET /first HTTP/1.1\r\nHost: example.com\r\n\r\n" + glob["__firegex_packet_info"] = create_packet_info(req1_data, is_input=True) + handle_packet(glob) + + # 2. First Response + resp1_data = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK" + glob["__firegex_packet_info"] = create_packet_info(resp1_data, is_input=False) + handle_packet(glob) + + # 3. Second Request + req2_data = b"GET /second HTTP/1.1\r\nHost: example.com\r\n\r\n" + glob["__firegex_packet_info"] = create_packet_info(req2_data, is_input=True) + handle_packet(glob) + + # 4. Second Response + resp2_data = b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n" + glob["__firegex_packet_info"] = create_packet_info(resp2_data, is_input=False) + handle_packet(glob) + + recorded = glob["recorded_history"] + + # 1. First Request: history is empty + assert recorded[0] == ("req", "/first", [], []) + + # 2. First Response: past requests contains /first + assert recorded[1] == ("resp", "OK", ["/first"], []) + + # 3. Second Request: past requests contains [/first], past responses contains [OK] + assert recorded[2] == ("req", "/second", ["/first"], ["OK"]) + + # 4. Second Response: past requests contains [/first, /second], past responses contains [OK] + assert recorded[3] == ("resp", "Not Found", ["/first", "/second"], ["OK"]) + + +def test_http_history_attribute_on_request(): + glob = {} + clear_pyfilter_registry() + + code = """ +from firegex.nfproxy import pyfilter, ACCEPT +from firegex.nfproxy.models import HttpFullRequest, HttpFullResponse + +history_lengths = [] + +@pyfilter +def filter_req(req: HttpFullRequest): + history_lengths.append((len(req.history.requests), len(req.history.responses))) + return ACCEPT + +@pyfilter +def filter_resp(resp: HttpFullResponse): + history_lengths.append((len(resp.history.requests), len(resp.history.responses))) + return ACCEPT +""" + + glob["__firegex_pyfilter_enabled"] = ["filter_req", "filter_resp"] + glob["__firegex_proto"] = "http" + exec(code, glob, glob) + compile(glob) + + # Req 1 + glob["__firegex_packet_info"] = create_packet_info(b"GET /a HTTP/1.1\r\nHost: a\r\n\r\n", is_input=True) + handle_packet(glob) + + # Resp 1 + glob["__firegex_packet_info"] = create_packet_info(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", is_input=False) + handle_packet(glob) + + # Req 2 + glob["__firegex_packet_info"] = create_packet_info(b"GET /b HTTP/1.1\r\nHost: b\r\n\r\n", is_input=True) + handle_packet(glob) + + lengths = glob["history_lengths"] + assert lengths[0] == (0, 0) + assert lengths[1] == (1, 0) + assert lengths[2] == (1, 1) + + +if __name__ == "__main__": + test_http_history_model() + test_http_history_stream_execution() + test_http_history_attribute_on_request() + print("All HTTP history unit tests passed successfully!")