"""The test that says whether the objective was reached. One filter file, unchanged, in front of an HTTP/1.1 service, an HTTP/2 service and an HTTP/3 service, blocking the same request on all three. It can only pass because each version is rendered to the chain as the HTTP/1.1 it would have been. On the wire that path is plain text on HTTP/1.1, an HPACK-compressed header block on HTTP/2 and a QPACK one on HTTP/3, so a filter asking for an `HttpRequest` used to be called on the first alone — and on the other two it was handed a compression format, never ran, and said nothing about it. A firewall whose rules quietly stop applying when a client picks a newer protocol is worse than one that refuses the protocol outright. The HTTP/2 leg is a gRPC service, because a gRPC service *is* an HTTP/2 service and the suite already stands a real one up. What the filter is shown there is an HTTP/2 request with a path, which is the thing under test; that it happens to be a gRPC method name changes nothing about the question. """ import pytest from integration import filter_code from integration.conftest import add_python_filter, add_regex_filter, start_and_settle from helpers.grpcserver import GrpcService, call, channel, needs_grpc from helpers.net import free_port from helpers.quicserver import h3_is_blocked, h3_request, needs_quic from helpers.traffic import Channel pytestmark = [pytest.mark.instance, pytest.mark.http2] #: The same request, in the three shapes a client can send it. BLOCKED_PATH = "/files/../secret" ALLOWED_PATH = "/files/ok" HTTP11_BLOCKED = f"GET {BLOCKED_PATH} HTTP/1.1\r\nHost: x\r\n\r\n".encode() HTTP11_ALLOWED = f"GET {ALLOWED_PATH} HTTP/1.1\r\nHost: x\r\n\r\n".encode() @pytest.fixture def grpc_stand_in(): started = [] def _serve(cert: str | None = None, key: str | None = None) -> GrpcService: service = GrpcService(free_port(), cert, key) service.start() started.append(service) return service yield _serve for service in started: try: service.stop() except Exception: pass def _grpc_blocked(port: int, path: str, cert: str | None) -> bool: """Whether the exchange was refused, as a client sees a refusal. A block closes the connection carrying the request rather than answering it, so from out here it is an error; getting an answer back is the request having gone through. """ import grpc with channel(port, cert) as chan: try: call(chan, path, b"hello") return False except grpc.RpcError: return True @needs_grpc @needs_quic def test_one_python_filter_covers_http1_http2_and_http3(api, service, stand_in, grpc_stand_in, quic_stand_in, certificate): """`filter_code.HTTP`, saved three times and edited never. Three services, three protocols, three real client libraries — and one file, which is the whole claim. If this passes and the three assertions below disagree with each other, the rendering has drifted between the protocols, which is the failure the shared `http1.rs` exists to make impossible. """ cert, key = certificate() # HTTP/1.1, on a plain TCP service. The stand-in echoes, so the *reply* is not itself # valid HTTP — which is why the shared file turns the invalid-encoding refusal off. plain = stand_in() http11 = service(f"v11-{plain.port}", "127.0.0.1", plain.port, "proxy") add_python_filter(api, http11, filter_code.HTTP, name="http") start_and_settle(api, http11) # HTTP/2, on an `http` service: TLS on the wire, `h2` agreed by ALPN, and each # exchange terminated and rendered. grpc_server = grpc_stand_in(cert, key) http2 = service(f"v2-{grpc_server.port}", "127.0.0.1", grpc_server.port, "proxy", proto="http", tls_cert=cert, tls_key=key) add_python_filter(api, http2, filter_code.HTTP, name="http") start_and_settle(api, http2) # HTTP/3, on a QUIC service. h3_server = quic_stand_in() h3_cert, h3_key = h3_server.material http3 = service(f"v3-{h3_server.port}", "127.0.0.1", h3_server.port, "proxy", proto="quic", tls_cert=h3_cert, tls_key=h3_key) add_python_filter(api, http3, filter_code.HTTP, name="http") start_and_settle(api, http3) # The innocent request reaches all three. channel11 = Channel(plain, plain.port, False) assert channel11.gets_through(HTTP11_ALLOWED), "HTTP/1.1 refused an innocent request" assert not _grpc_blocked(grpc_server.port, ALLOWED_PATH, cert), \ "HTTP/2 refused an innocent request" assert ALLOWED_PATH in h3_request(h3_server.host, h3_server.port, ALLOWED_PATH), \ "HTTP/3 refused an innocent request" # And the same file blocks the same one on all three. assert channel11.is_blocked(HTTP11_BLOCKED), \ "the filter did not run on HTTP/1.1, where it always did" assert _grpc_blocked(grpc_server.port, BLOCKED_PATH, cert), \ ("the filter did not run on HTTP/2: the request reached the service, which means " "the exchange was not rendered and an HttpRequest filter was never called") assert h3_is_blocked(h3_server.host, h3_server.port, BLOCKED_PATH), \ "the filter did not run on HTTP/3" @needs_grpc @needs_quic def test_one_pattern_covers_http1_http2_and_http3(api, service, stand_in, grpc_stand_in, quic_stand_in, certificate): """The same claim for a hyperscan pattern, which is the other half of a chain. A pattern is written against a request line, and a request line is what the rendering produces on every version. Before HTTP/2 was rendered, this pattern matched on two of the three and nothing said which. """ cert, key = certificate() plain = stand_in() http11 = service(f"rx11-{plain.port}", "127.0.0.1", plain.port, "proxy") add_regex_filter(api, http11, f"GET {BLOCKED_PATH}") start_and_settle(api, http11) grpc_server = grpc_stand_in(cert, key) http2 = service(f"rx2-{grpc_server.port}", "127.0.0.1", grpc_server.port, "proxy", proto="http", tls_cert=cert, tls_key=key) # The path is the part that was compressed; the method gRPC sends is a POST. add_regex_filter(api, http2, BLOCKED_PATH) start_and_settle(api, http2) h3_server = quic_stand_in() h3_cert, h3_key = h3_server.material http3 = service(f"rx3-{h3_server.port}", "127.0.0.1", h3_server.port, "proxy", proto="quic", tls_cert=h3_cert, tls_key=h3_key) add_regex_filter(api, http3, f"GET {BLOCKED_PATH}") start_and_settle(api, http3) channel11 = Channel(plain, plain.port, False) # The innocent one first, on every leg. Without it a setup broken in some way that # has nothing to do with filtering — a handshake that fails, a service that is not # there — reads as a block and the whole test passes for the wrong reason. assert channel11.gets_through(HTTP11_ALLOWED) assert not _grpc_blocked(grpc_server.port, ALLOWED_PATH, cert) assert ALLOWED_PATH in h3_request(h3_server.host, h3_server.port, ALLOWED_PATH) assert channel11.is_blocked(HTTP11_BLOCKED) assert _grpc_blocked(grpc_server.port, BLOCKED_PATH, cert), \ "a pattern did not match a path that arrived HPACK-compressed" assert h3_is_blocked(h3_server.host, h3_server.port, BLOCKED_PATH) @needs_grpc def test_an_http_service_carries_its_cleartext_and_its_tls_edge_at_once( api, service, grpc_stand_in, certificate): """One service, two addresses, one chain — which is what `http` is for. A daemon answering in the clear on one port and under TLS on another used to need two firegex services with the same filters copied between them by hand, and a chain kept in step by hand is one that stops protecting one of them silently. The edge is decided per connection, from what the client actually sent. """ cert, key = certificate() encrypted = grpc_stand_in(cert, key) clear = grpc_stand_in() service_id = service( f"http-both-{encrypted.port}", "127.0.0.1", encrypted.port, "proxy", proto="http", tls_cert=cert, tls_key=key, addresses=[{"ip_int": "127.0.0.1", "port": encrypted.port, "edge": "tls"}, {"ip_int": "127.0.0.1", "port": clear.port, "edge": "tcp"}], ) add_regex_filter(api, service_id, "/fgex.Echo/Forbidden") start_and_settle(api, service_id) # The innocent call works on both edges… with channel(encrypted.port, cert) as chan: assert call(chan, "/fgex.Echo/Unary", b"hello") == b"you sent hello" with channel(clear.port) as chan: assert call(chan, "/fgex.Echo/Unary", b"hello") == b"you sent hello" # …and the one chain refuses on both, which is the whole point of one service. assert _grpc_blocked(encrypted.port, "/fgex.Echo/Forbidden", cert), \ "the TLS edge of an http service went unfiltered" assert _grpc_blocked(clear.port, "/fgex.Echo/Forbidden", None), \ "the cleartext edge of an http service went unfiltered" @needs_grpc @needs_quic def test_all_three_edges_of_one_http_service_share_one_chain( api, service, grpc_stand_in, quic_stand_in, certificate): """The shape an `http` service exists for: three ports, one chain, one certificate. A daemon reached in the clear on one port, under TLS on another and over HTTP/3 on a third. Before this it took three firegex services with the same filters pasted into each of them, and a chain kept in step by hand is one that stops protecting one of them silently — the same argument that put several addresses under one service. The certificate has to be the QUIC stand-in's, because that one is a real service with its own TLS and firegex opens a second QUIC handshake towards it; the TCP legs are happy with any. """ quic_server = quic_stand_in() cert, key = quic_server.material encrypted = grpc_stand_in(cert, key) clear = grpc_stand_in() service_id = service( f"http-three-{quic_server.port}", "127.0.0.1", encrypted.port, "proxy", proto="http", tls_cert=cert, tls_key=key, addresses=[ {"ip_int": "127.0.0.1", "port": encrypted.port, "edge": "tls"}, {"ip_int": "127.0.0.1", "port": clear.port, "edge": "tcp"}, {"ip_int": "127.0.0.1", "port": quic_server.port, "edge": "quic"}, ], ) add_regex_filter(api, service_id, BLOCKED_PATH) start_and_settle(api, service_id) # Every edge carries the innocent request… with channel(encrypted.port, cert) as chan: assert call(chan, ALLOWED_PATH, b"hello") == b"you sent hello", \ "the TLS edge did not answer" with channel(clear.port) as chan: assert call(chan, ALLOWED_PATH, b"hello") == b"you sent hello", \ "the cleartext edge did not answer" assert ALLOWED_PATH in h3_request("127.0.0.1", quic_server.port, ALLOWED_PATH), \ "the HTTP/3 edge did not answer" # …and the one pattern refuses on all three, which is the whole point. assert _grpc_blocked(encrypted.port, BLOCKED_PATH, cert), \ "the TLS edge went unfiltered" assert _grpc_blocked(clear.port, BLOCKED_PATH, None), \ "the cleartext edge went unfiltered" assert h3_is_blocked("127.0.0.1", quic_server.port, BLOCKED_PATH), \ "the HTTP/3 edge went unfiltered" # --- and one service reached in a version it does not speak --------------------------- @needs_quic def test_an_http1_service_is_reached_by_an_http3_client(api, service, http_stand_in, certificate): """The other half of one rendering: it can be *sent*, not only shown. Most of the web is a service that speaks HTTP/1.1 and never will speak QUIC. Because each exchange is already turned into the HTTP/1.1 the filters are shown, that same rendering can go on to the service — so firegex terminates HTTP/3 in front of an ordinary web service, which is what `upstream="tcp"` asks for. Nothing about the chain changes, which is what the block below is here to say. """ cert, key = certificate("127.0.0.1") server = http_stand_in() service_id = service(f"h3toh1-{server.port}", "127.0.0.1", server.port, "proxy", proto="quic", tls_cert=cert, tls_key=key, upstream="tcp") add_regex_filter(api, service_id, BLOCKED_PATH) start_and_settle(api, service_id) assert ALLOWED_PATH in h3_request("127.0.0.1", server.port, ALLOWED_PATH), \ "an HTTP/3 client could not reach the HTTP/1.1 service behind it" assert h3_is_blocked("127.0.0.1", server.port, BLOCKED_PATH), \ "the chain stopped seeing the exchange once the service spoke another version" def _whole_answer(sock, timeout: float) -> bytes: """Read one HTTP/1.1 message, not the first segment that happens to arrive. The head and the body of a proxied response land separately often enough that reading once is a test failing for a reason belonging to TCP — which it did, on both legs of the one below, and in the same way. """ got = b"" while b"\r\n\r\n" not in got or not got.split(b"\r\n\r\n", 1)[1]: piece = sock.recv(65535) if not piece: break got += piece return got def _http11(port: int, payload: bytes, timeout: float = 3.0) -> bytes: """One HTTP/1.1 exchange in the clear, without the echo helper's assumptions. `Channel` compares what comes back with what went out, which is exactly right for the echo stand-in and exactly wrong for a service that answers. Here what matters is that an answer came back at all and that it is the service's. """ import socket try: with socket.create_connection(("127.0.0.1", port), timeout=timeout) as sock: sock.sendall(payload) return _whole_answer(sock, timeout) except OSError: return b"" def _https11(port: int, payload: bytes, timeout: float = 3.0) -> bytes: """The same, through a real TLS handshake.""" import socket import ssl ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE try: with socket.create_connection(("127.0.0.1", port), timeout=timeout) as raw: with ctx.wrap_socket(raw) as sock: sock.sendall(payload) return _whole_answer(sock, timeout) except OSError: return b"" @needs_quic def test_one_service_is_published_in_the_clear_under_tls_and_over_http3( api, service, http_stand_in, certificate): """The shape an operator actually has: one daemon, three ways in. A service answering HTTP/1.1 in the clear on one port, and nothing else — no TLS of its own, no QUIC, no second port. It is *published* on two more: TLS on one, HTTP/3 on another, both pointing back at the port it already listens on. Firegex terminates what each client speaks, renders every version as the same HTTP/1.1, runs one chain over all three, and sends the service what it has always understood. Three addresses and one filter. Before this the operator's options were to make the service speak TLS and QUIC itself, or to run three services with the chain copied between them by hand — which is the failure a list of addresses exists to prevent. """ cert, key = certificate("127.0.0.1") server = http_stand_in() tls_port, h3_port = free_port(), free_port(udp=True) service_id = service( f"pub-{server.port}", "127.0.0.1", server.port, "proxy", proto="http", tls_cert=cert, tls_key=key, addresses=[ # The service itself, in the clear, where it really is. {"ip_int": "127.0.0.1", "port": server.port, "edge": "tcp"}, # And the two it is published on: each says where the service is *and* that # it answers in the clear, because both are facts about the way in. {"ip_int": "127.0.0.1", "port": tls_port, "edge": "tls", "target_port": server.port, "upstream": "tcp"}, {"ip_int": "127.0.0.1", "port": h3_port, "edge": "quic", "target_port": server.port, "upstream": "tcp"}, ], ) add_regex_filter(api, service_id, BLOCKED_PATH) start_and_settle(api, service_id) # In the clear, on the service's own port. assert b"you asked for" in _http11(server.port, HTTP11_ALLOWED), \ "the service stopped answering on its own port" # Under TLS, on a port it has never listened on. assert b"you asked for" in _https11(tls_port, HTTP11_ALLOWED), \ "the TLS publication did not reach the cleartext service" # And over HTTP/3, on another one. assert ALLOWED_PATH in h3_request("127.0.0.1", h3_port, ALLOWED_PATH), \ "the HTTP/3 publication did not reach the cleartext service" # One chain over all three, which is the point of them being one service. assert b"you asked for" not in _http11(server.port, HTTP11_BLOCKED), \ "the chain did not run on the cleartext edge" assert b"you asked for" not in _https11(tls_port, HTTP11_BLOCKED), \ "the chain did not run on the TLS publication" assert h3_is_blocked("127.0.0.1", h3_port, BLOCKED_PATH), \ "the chain did not run on the HTTP/3 publication" def test_an_https_edge_on_an_interface_is_reached_as_one( api, service, http_stand_in, certificate): """An interface is a way of spelling an address, not a lesser kind of one. The engine keys what it is told about an address on what `SO_ORIGINAL_DST` hands back, which is an address and never an interface name — so a row naming an interface had nothing to key on and was simply left out of the announcement. The rules were installed and the port was redirected, so the connection arrived; the engine then met it knowing nothing, terminated it as an ordinary one and dialled the port it came in on, where nothing listens. What that looked like to an operator: HTTPS declared on `lo:443` did not answer at all, while the very same address written `127.0.0.1:443` worked. Reported that way, and found by trying both. """ cert, key = certificate("127.0.0.1") server = http_stand_in() tls_port = free_port() service_id = service( f"ifacetls-{server.port}", "lo", server.port, "proxy", proto="http", tls_cert=cert, tls_key=key, addresses=[ # The service itself, in the clear, named by the interface it answers on. {"ip_int": "lo", "port": server.port, "edge": "tcp"}, # And the encrypted way in, named the same way. {"ip_int": "lo", "port": tls_port, "edge": "tls", "target_port": server.port, "upstream": "tcp"}, ], ) add_regex_filter(api, service_id, BLOCKED_PATH) start_and_settle(api, service_id) assert b"you asked for" in _http11(server.port, HTTP11_ALLOWED), \ "the cleartext edge did not reach the service" assert b"you asked for" in _https11(tls_port, HTTP11_ALLOWED), \ "the HTTPS edge declared on an interface never reached the service" assert b"you asked for" not in _https11(tls_port, HTTP11_BLOCKED), \ "the chain did not run on the HTTPS edge"