mirror of
https://github.com/Av0idD4rk/firegex.git
synced 2026-10-23 21:42:05 +00:00
310 lines
13 KiB
Python
310 lines
13 KiB
Python
"""What an HTTP filter actually sees, pinned exchange by exchange.
|
|
|
|
These are **characterisation** tests: they exist to make the parser replaceable, and
|
|
then to make it updatable. The library parsed HTTP with `pyllhttp`, which published a
|
|
wheel for exactly one platform and has since been archived; llhttp now ships as source
|
|
in `fgex-lib/llhttp/` behind a binding of ours, and the release workflow builds the
|
|
wheels. Nobody upstream will bump that copy — `fgex-lib/llhttp/PROVENANCE.md` says which
|
|
version it is and how to move it.
|
|
|
|
Either kind of change is only safe if "the same" can be demonstrated rather than hoped,
|
|
and the coverage before this was four plain GETs. So everything below goes through the
|
|
public models — the surface a filter written from the documentation touches — rather than
|
|
through parser internals, which are exactly what is allowed to change.
|
|
|
|
Each case is a shape that occurs in real traffic and that a naive port gets wrong:
|
|
headers split across packets, headers repeated, chunked bodies, compressed bodies,
|
|
pipelined messages, protocol upgrades, and bodies delivered in pieces.
|
|
"""
|
|
|
|
import gzip
|
|
import zlib
|
|
|
|
import pytest
|
|
|
|
from firegex.pyfilters.internals import compile, handle_packet
|
|
|
|
FILTER_CODE = """
|
|
from firegex.pyfilters import pyfilter, ACCEPT
|
|
from firegex.pyfilters.models import HttpRequest, HttpResponse
|
|
|
|
seen = []
|
|
|
|
@pyfilter
|
|
def watch_request(req: HttpRequest):
|
|
seen.append({
|
|
"kind": "request",
|
|
"method": req.method,
|
|
"url": req.url,
|
|
"headers": dict(req.headers),
|
|
"lheaders": dict(req.lheaders) if hasattr(req, "lheaders") else None,
|
|
"body": req.body,
|
|
"http_version": req.http_version,
|
|
"keep_alive": req.keep_alive,
|
|
"content_length": req.content_length,
|
|
"headers_complete": req.headers_complete,
|
|
"message_complete": req.message_complete,
|
|
})
|
|
return ACCEPT
|
|
|
|
@pyfilter
|
|
def watch_response(res: HttpResponse):
|
|
seen.append({
|
|
"kind": "response",
|
|
"status_code": res.status_code,
|
|
"status_phrase": res.status_phrase,
|
|
"headers": dict(res.headers),
|
|
"body": res.body,
|
|
"http_version": res.http_version,
|
|
"keep_alive": res.keep_alive,
|
|
"content_length": res.content_length,
|
|
"headers_complete": res.headers_complete,
|
|
"message_complete": res.message_complete,
|
|
})
|
|
return ACCEPT
|
|
"""
|
|
|
|
|
|
def packet(payload: bytes, is_input: bool):
|
|
return {
|
|
"data": payload,
|
|
"is_input": is_input,
|
|
"is_ipv6": False,
|
|
"is_tcp": True,
|
|
"src_ip": "10.0.0.9" if is_input else "10.0.0.1",
|
|
"src_port": 51000 if is_input else 80,
|
|
"dst_ip": "10.0.0.1" if is_input else "10.0.0.9",
|
|
"dst_port": 80 if is_input else 51000,
|
|
}
|
|
|
|
|
|
def run(*chunks: tuple[bytes, bool]) -> list[dict]:
|
|
"""Feed `(payload, is_input)` chunks through a filter and return what it saw."""
|
|
return _feed(*chunks)[0]
|
|
|
|
|
|
def run_with_verdict(*chunks: tuple[bytes, bool]) -> tuple[list[dict], int]:
|
|
"""The same, plus what the chain answered — which is the whole story when the
|
|
parser refuses the traffic and the filter is therefore never called."""
|
|
return _feed(*chunks)
|
|
|
|
|
|
def _feed(*chunks: tuple[bytes, bool]) -> tuple[list[dict], int]:
|
|
glob = {"__firegex_pyfilter_enabled": ["watch_request", "watch_response"]}
|
|
exec(FILTER_CODE, glob, glob)
|
|
compile(glob)
|
|
for payload, is_input in chunks:
|
|
glob["__firegex_packet_info"] = packet(payload, is_input)
|
|
handle_packet(glob)
|
|
return glob["seen"], glob.get("__firegex_pyfilter_result", {}).get("action")
|
|
|
|
|
|
def final(seen: list[dict], kind: str) -> dict:
|
|
"""The last complete view of a message — what a filter acts on."""
|
|
complete = [s for s in seen if s["kind"] == kind and s["message_complete"]]
|
|
assert complete, f"no complete {kind} was ever handed to the filter: {seen}"
|
|
return complete[-1]
|
|
|
|
|
|
# --- the ordinary shapes --------------------------------------------------------------
|
|
|
|
|
|
def test_a_plain_request_is_reported_whole():
|
|
got = final(run((b"GET /shop?id=1 HTTP/1.1\r\nHost: example.com\r\n\r\n", True)),
|
|
"request")
|
|
assert got["method"] == "GET"
|
|
assert got["url"] == "/shop?id=1"
|
|
assert got["http_version"] == "1.1"
|
|
assert got["headers"]["Host"] == "example.com"
|
|
assert got["keep_alive"] is True
|
|
|
|
|
|
def test_a_plain_response_is_reported_whole():
|
|
got = final(run((b"HTTP/1.1 404 Not Found\r\nContent-Length: 3\r\n\r\nnop", False)),
|
|
"response")
|
|
assert got["body"] == b"nop"
|
|
assert got["content_length"] == 3
|
|
# Both, and each under the name that describes it. `status_code` held the phrase
|
|
# for a long time while being documented as an int, so a filter asking `== 404`
|
|
# matched nothing.
|
|
assert got["status_code"] == 404
|
|
assert got["status_phrase"] == "Not Found"
|
|
|
|
|
|
def test_a_post_body_reaches_the_filter():
|
|
got = final(run((b"POST /login HTTP/1.1\r\nHost: x\r\nContent-Length: 11\r\n"
|
|
b"\r\nuser=admin&", True)), "request")
|
|
assert got["method"] == "POST"
|
|
assert got["body"] == b"user=admin&"
|
|
|
|
|
|
# --- the shapes a naive port gets wrong -----------------------------------------------
|
|
|
|
|
|
def test_a_request_split_across_packets_is_reassembled():
|
|
"""The halves of a header can land in different packets, and routinely do."""
|
|
got = final(run((b"GET /split HTTP/1.1\r\nHo", True),
|
|
(b"st: example.com\r\nX-Trace: abc\r\n\r\n", True)), "request")
|
|
assert got["url"] == "/split"
|
|
assert got["headers"]["Host"] == "example.com"
|
|
assert got["headers"]["X-Trace"] == "abc"
|
|
|
|
|
|
def test_a_body_split_across_packets_is_reassembled():
|
|
got = final(run((b"POST /up HTTP/1.1\r\nHost: x\r\nContent-Length: 10\r\n\r\nabcde", True),
|
|
(b"fghij", True)), "request")
|
|
assert got["body"] == b"abcdefghij"
|
|
|
|
|
|
def test_a_repeated_header_is_kept_as_a_list_raw_and_joined_when_lowercased():
|
|
"""Two `Set-Cookie` lines are two cookies, not one — and the RFC says a
|
|
comma-joined single value is the equivalent reading."""
|
|
got = final(run((b"HTTP/1.1 200 OK\r\nSet-Cookie: a=1\r\nSet-Cookie: b=2\r\n"
|
|
b"Content-Length: 0\r\n\r\n", False)), "response")
|
|
assert got["headers"]["Set-Cookie"] == ["a=1", "b=2"], got["headers"]
|
|
|
|
|
|
def test_a_header_keeps_the_case_the_sender_used():
|
|
got = final(run((b"GET / HTTP/1.1\r\nHOST: example.com\r\nX-Mixed-Case: v\r\n\r\n",
|
|
True)), "request")
|
|
assert got["headers"]["HOST"] == "example.com"
|
|
assert got["headers"]["X-Mixed-Case"] == "v"
|
|
|
|
|
|
def test_a_chunked_body_is_delivered_dechunked():
|
|
got = final(run((b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
|
|
b"5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n", False)), "response")
|
|
assert got["body"] == b"hello world"
|
|
|
|
|
|
def test_a_chunked_body_split_across_packets_is_delivered_whole():
|
|
got = final(run((b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhel", False),
|
|
(b"lo\r\n0\r\n\r\n", False)), "response")
|
|
assert got["body"] == b"hello"
|
|
|
|
|
|
def test_pipelined_requests_are_reported_separately():
|
|
"""Two requests in one packet are two messages, and a filter has to see both."""
|
|
seen = run((b"GET /one HTTP/1.1\r\nHost: x\r\n\r\nGET /two HTTP/1.1\r\nHost: x\r\n\r\n",
|
|
True))
|
|
urls = [s["url"] for s in seen if s["kind"] == "request" and s["message_complete"]]
|
|
# Each exactly once. `on_message_complete` queues the finished message but leaves
|
|
# `parser.msg` pointing at it, so the branch that hands over a message whose headers
|
|
# are known used to deliver the second one a second time.
|
|
assert urls == ["/one", "/two"], urls
|
|
|
|
|
|
def test_a_request_with_no_body_reports_zero_content_length():
|
|
got = final(run((b"GET /empty HTTP/1.1\r\nHost: x\r\n\r\n", True)), "request")
|
|
assert got["content_length"] in (0, None), got["content_length"]
|
|
|
|
|
|
def test_connection_close_is_reported_as_not_keep_alive():
|
|
got = final(run((b"GET / HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n", True)),
|
|
"request")
|
|
assert got["keep_alive"] is False
|
|
|
|
|
|
def test_http_1_0_is_reported_as_such():
|
|
"""A minor version of zero is still a version.
|
|
|
|
Built behind `if self.major and self.minor` before, so HTTP/1.0 and HTTP/2.0 both
|
|
reported the empty string and a filter keying on the version never saw them.
|
|
"""
|
|
got = final(run((b"GET / HTTP/1.0\r\nHost: x\r\n\r\n", True)), "request")
|
|
assert got["http_version"] == "1.0"
|
|
|
|
|
|
# --- the bodies the filters are actually reading --------------------------------------
|
|
# A compressed body that reached a filter still compressed is a filter that silently
|
|
# stops matching, so the decoding is part of what a filter sees.
|
|
|
|
|
|
def test_a_gzipped_body_is_decompressed_before_the_filter_sees_it():
|
|
payload = gzip.compress(b"FLAG{secret}")
|
|
raw = (b"HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: "
|
|
+ str(len(payload)).encode() + b"\r\n\r\n" + payload)
|
|
assert final(run((raw, False)), "response")["body"] == b"FLAG{secret}"
|
|
|
|
|
|
def test_a_raw_deflated_body_is_decompressed_before_the_filter_sees_it():
|
|
"""Raw deflate, which is what most servers actually send under this name."""
|
|
compressor = zlib.compressobj(wbits=-zlib.MAX_WBITS)
|
|
payload = compressor.compress(b"FLAG{secret}") + compressor.flush()
|
|
raw = (b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\nContent-Length: "
|
|
+ str(len(payload)).encode() + b"\r\n\r\n" + payload)
|
|
assert final(run((raw, False)), "response")["body"] == b"FLAG{secret}"
|
|
|
|
|
|
def test_a_zlib_wrapped_deflate_body_is_decompressed_too():
|
|
"""RFC 7230 defines `deflate` as the zlib format of RFC 1950.
|
|
|
|
Servers send both that and the raw stream under the same name, browsers accept
|
|
either, and a decoder that takes only one lets half the compressed traffic reach a
|
|
filter still compressed — where a pattern looking for a flag finds nothing and the
|
|
only trace is a line on stdout saying it skipped.
|
|
"""
|
|
payload = zlib.compress(b"FLAG{secret}")
|
|
raw = (b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\nContent-Length: "
|
|
+ str(len(payload)).encode() + b"\r\n\r\n" + payload)
|
|
assert final(run((raw, False)), "response")["body"] == b"FLAG{secret}"
|
|
|
|
|
|
def test_a_zstd_body_is_decompressed_before_the_filter_sees_it():
|
|
"""The encoding whose stdlib module only exists on Python 3.14."""
|
|
try:
|
|
from firegex.pyfilters.internals import zstd_compat # noqa: F401
|
|
import zstandard
|
|
except ImportError:
|
|
try:
|
|
from compression import zstd as _z
|
|
payload = _z.compress(b"FLAG{secret}")
|
|
except ImportError:
|
|
pytest.skip("no zstd backend available here")
|
|
else:
|
|
payload = zstandard.ZstdCompressor().compress(b"FLAG{secret}")
|
|
raw = (b"HTTP/1.1 200 OK\r\nContent-Encoding: zstd\r\nContent-Length: "
|
|
+ str(len(payload)).encode() + b"\r\n\r\n" + payload)
|
|
assert final(run((raw, False)), "response")["body"] == b"FLAG{secret}"
|
|
|
|
|
|
def test_an_identity_encoding_is_left_alone():
|
|
raw = b"HTTP/1.1 200 OK\r\nContent-Encoding: identity\r\nContent-Length: 5\r\n\r\nplain"
|
|
assert final(run((raw, False)), "response")["body"] == b"plain"
|
|
|
|
|
|
# What the parser refuses, and what it has started tolerating. These two are the reason
|
|
# the vendored llhttp carries its version in `fgex-lib/llhttp/PROVENANCE.md`: both
|
|
# answers changed between 9.2.1 and 9.4.3, and both are decisions about what a filter is
|
|
# shown rather than details of how it is parsed.
|
|
|
|
def test_an_empty_transfer_encoding_never_reaches_a_filter():
|
|
"""`Transfer-Encoding:` with nothing after it is refused, and the connection with it.
|
|
|
|
This is the shape request smuggling is built out of: a header that one parser reads
|
|
as "chunked follows" and another as "no framing here" is two readings of where the
|
|
next request begins. llhttp accepted it until 9.4.3 — which means firegex parsed it,
|
|
showed a filter an ordinary-looking request, and forwarded whatever the service then
|
|
made of it. Now the parse fails, `invalid_encoding_action` decides, and its default
|
|
is to reject: the filter is never called, because there is no one message here to
|
|
call it with.
|
|
"""
|
|
seen, action = run_with_verdict(
|
|
(b"POST / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: \r\n\r\n", True))
|
|
assert seen == []
|
|
assert action == 2 # REJECT
|
|
|
|
|
|
def test_tabs_around_a_content_length_are_tolerated():
|
|
"""`Content-Length:\t5\t` is a length, not a malformed header.
|
|
|
|
Servers accept it and llhttp has since 9.4.0. Refusing it — which is what happened
|
|
before — meant a legitimate response was never parsed, so the filter meant to read
|
|
that body never ran on it, and with the default action the connection was rejected:
|
|
firegex breaking a service it was there to protect.
|
|
"""
|
|
raw = b"HTTP/1.1 200 OK\r\nContent-Length:\t5\t\r\n\r\nplain"
|
|
message = final(run((raw, False)), "response")
|
|
assert message["body"] == b"plain"
|
|
assert message["content_length"] == 5
|