From 4476da02bd7a9bab70614e77798ffbf38f065403 Mon Sep 17 00:00:00 2001 From: Domingo Dirutigliano Date: Fri, 18 Sep 2026 09:34:44 +0200 Subject: [PATCH] QUIC support and bug fixes --- .github/workflows/release.yml | 7 +- AGENTS.md | 431 ++++++++ CLAUDE.md | 306 +----- README.md | 6 +- backend/modules/services/firewall.py | 69 +- backend/modules/services/models.py | 244 ++++- backend/modules/services/nftables.py | 145 ++- backend/modules/services/pyworker.py | 9 +- backend/modules/services/transports.py | 247 ++++- backend/proxysrc/Cargo.lock | 578 +++++++++- backend/proxysrc/Cargo.toml | 44 + backend/proxysrc/src/capture.rs | 94 +- backend/proxysrc/src/control.rs | 87 +- backend/proxysrc/src/filter.rs | 50 +- backend/proxysrc/src/h1up.rs | 324 ++++++ backend/proxysrc/src/h2.rs | 800 ++++++++++++++ backend/proxysrc/src/h3.rs | 486 +++++++++ backend/proxysrc/src/http1.rs | 393 +++++++ backend/proxysrc/src/lib.rs | 6 + backend/proxysrc/src/main.rs | 259 ++++- backend/proxysrc/src/proxy.rs | 657 +++++++++--- backend/proxysrc/src/pyworker.rs | 10 +- backend/proxysrc/src/quic.rs | 994 ++++++++++++++++++ backend/proxysrc/src/relays.rs | 39 + backend/proxysrc/src/tls.rs | 18 + backend/proxysrc/src/transparent.rs | 23 +- backend/proxysrc/src/udp.rs | 4 +- backend/proxysrc/tests/h2.rs | 959 +++++++++++++++++ backend/proxysrc/tests/h3.rs | 703 +++++++++++++ backend/proxysrc/tests/pyworker.rs | 4 +- backend/proxysrc/tests/quic.rs | 386 +++++++ backend/proxysrc/tests/tls.rs | 3 + backend/routers/services.py | 289 ++++- docs/pyfilter.md | 95 +- docs/services.md | 390 ++++++- fgex-lib/firegex/pyfilters/internals/data.py | 37 +- fgex-lib/firegex/pyfilters/models/__init__.py | 13 + fgex-lib/firegex/pyfilters/models/http.py | 262 ++++- fgex-lib/firegex/pyfilters/models/tcp.py | 2 +- .../components/Services/AddEditService.tsx | 190 ++-- .../src/components/Services/AddressList.tsx | 149 ++- .../components/Services/AddressOptions.tsx | 155 +++ .../src/components/Services/AddressRow.tsx | 94 ++ .../src/components/Services/LayerChoice.tsx | 54 +- .../components/Services/ProtocolChoice.tsx | 233 ++++ .../src/components/Services/ServiceRow.tsx | 24 +- frontend/src/components/Services/utils.ts | 96 +- frontend/src/js/demo.ts | 56 +- .../src/pages/Services/ServiceDetails.tsx | 8 +- run.py | 89 +- tests/helpers/capture.py | 158 +++ tests/helpers/firegexapi.py | 37 +- tests/helpers/grpcserver.py | 191 ++++ tests/helpers/httpserver.py | 65 ++ tests/helpers/net.py | 42 +- tests/helpers/quicserver.py | 243 +++++ tests/integration/conftest.py | 55 +- tests/integration/filter_code.py | 15 + tests/integration/test_addresses.py | 243 +++++ tests/integration/test_grpc.py | 277 +++++ tests/integration/test_http_versions.py | 426 ++++++++ tests/integration/test_logs.py | 25 +- tests/integration/test_quic.py | 247 +++++ tests/integration/test_tls.py | 30 + tests/pytest.ini | 2 + tests/requirements.txt | 21 + tests/run_tests.sh | 10 +- tests/unit/test_grpc_framing.py | 180 ++++ tests/unit/test_nftables_addresses.py | 94 +- 69 files changed, 12232 insertions(+), 750 deletions(-) create mode 100644 AGENTS.md create mode 100644 backend/proxysrc/src/h1up.rs create mode 100644 backend/proxysrc/src/h2.rs create mode 100644 backend/proxysrc/src/h3.rs create mode 100644 backend/proxysrc/src/http1.rs create mode 100644 backend/proxysrc/src/quic.rs create mode 100644 backend/proxysrc/src/relays.rs create mode 100644 backend/proxysrc/tests/h2.rs create mode 100644 backend/proxysrc/tests/h3.rs create mode 100644 backend/proxysrc/tests/quic.rs create mode 100644 frontend/src/components/Services/AddressOptions.tsx create mode 100644 frontend/src/components/Services/AddressRow.tsx create mode 100644 frontend/src/components/Services/ProtocolChoice.tsx create mode 100644 tests/helpers/capture.py create mode 100644 tests/helpers/grpcserver.py create mode 100644 tests/helpers/httpserver.py create mode 100644 tests/helpers/quicserver.py create mode 100644 tests/integration/test_grpc.py create mode 100644 tests/integration/test_http_versions.py create mode 100644 tests/integration/test_quic.py create mode 100644 tests/unit/test_grpc_framing.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0135162..21829c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,8 +27,13 @@ jobs: # No apt-get: the tests' dependencies are pure python. `iperf3`, matplotlib and # numpy belong to the benchmarks, which CI does not run — a measurement has no # pass or fail to gate a release on. + # + # As root, because what needs root **skips** otherwise — and a case that always + # skips is not a case this gate is checking. Through the script rather than `sudo + # pytest`, because the script installs the dependencies into the environment it is + # running in; a root process does not read the invoking user's site-packages. - name: Run tests - run: cd tests && ./run_tests.sh + run: cd tests && sudo ./run_tests.sh docker_build: needs: run_tests diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..17b5969 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,431 @@ +# AGENTS.md + +Guidance for any coding agent working in this repository, and the one copy of it. Tool-specific +files — `CLAUDE.md` and anything like it — point here rather than repeating this, because two +copies of a document this long drift the first time one of them is edited, and the drift is +invisible: each agent reads its own file and neither is told the other now says something else. +That already happened once, and the stale copy went on describing rules that had been replaced. + +## What this is + +Firegex is a firewall built for CTF Attack-Defense competitions: it sits in front of a vulnerable service and blocks/mangles malicious traffic before it reaches it. Behind a single FastAPI backend and React frontend it ships **services** (a network layer plus an ordered chain of filters) and plain nftables rules (`firewall`). That is the whole product surface: `nfregex`, `nfproxy`, `tls` and `porthijack` were folded into `services` as choices rather than modules. + +## Commands + +### Running the whole app (Docker or standalone) +```bash +python3 run.py # build+start (Docker if available, else standalone chroot on Linux) +python3 run.py --standalone # force standalone mode (no Docker) +python3 run.py stop [--clear] # stop (optionally wipe the DB volume/rootfs) +python3 run.py restart # recreate the container WITHOUT rebuilding (stale image if source changed) +python3 run.py status +python3 run.py config --password [newpass] # change the running instance's password; omit the value to be prompted interactively +python3 run.py config --show # print the persisted settings (port, host, socket dir, auth) +python3 run.py start --unsafe-disable-auth # no authentication at all — see "Authentication" below +python3 run.py restart --env FGEX_PROXY_QUIC_ALPN=h3,doq # anything run.py has no opinion about +``` +`--env KEY=VALUE` (repeatable, `KEY=` removes one) is the *only* supported way to set what the engine and the backend read straight out of their environment — `FGEX_PROXY_QUIC_ALPN`, `FGEX_PROXY_FILTER_TIMEOUT_MS`, `FGEX_STREAM_MAX_SIZE`. It exists because `run.py` **rewrites** `.firegex-compose.yml` on every start, so a variable hand-edited into it is gone by the next one, which is what the QUIC documentation's "set it in the container's environment" silently ran into. It is stored in `.firegex-conf.json` like the other flags, reaches the standalone chroot as well as the container, and refuses the names `run.py` sets itself (`MANAGED_ENV`) rather than letting two writers of one variable decide by list order. +`run.py` has zero external dependencies (stdlib only) by design. `restart` runs `docker compose up -d --force-recreate`, so it re-reads the generated compose file and re-applies env vars (`ALLOWED_IPS`, `PROXY_IP_HEADER`, `UNSAFE_DISABLE_AUTH`) — a plain `docker compose restart` would keep the old container environment and silently ignore those. It still does **not** rebuild the image; after a source change use `stop` then the bare start command to force `docker compose up -d --build`. + +Most `start`/`restart` flags are persisted to `.firegex-conf.json` and become the default for later runs, so a bare `python3 run.py start` inherits whatever was last passed. + +### Frontend (`frontend/`) +```bash +bun install # bun.lock is canonical +bun run dev # vite dev server +bun run build # tsc && vite build +npx tsc --noEmit -p . # typecheck only (no separate lint script is configured) +``` + +### Backend (`backend/`) +Needs Linux with `NET_ADMIN` (nftables/nfqueue) and root, so it's normally only run via `run.py` (Docker or standalone). Direct iteration on a Linux box/VM: +```bash +cd backend && DEBUG=1 python3 app.py DEBUG +``` + +### Tests (`tests/`) +**One suite, one runner: `pytest`.** Everything drives a **live, already-running** firegex (`http://127.0.0.1:4444` by default), root/Linux, except `unit/` which needs nothing running at all. + +```bash +cd tests +sudo ./run_tests.sh # installs deps, waits for the instance, runs pytest +sudo ./run_tests.sh --set-pass # against one still in its initial-setup state +sudo ./run_tests.sh unit # the ones needing no instance +pytest -k udp # one subject +pytest --layer proxy --no-ipv6 -x # narrow, stop at the first failure +pytest standalone # restarts the instance; opt-in, see below +``` + +**Run it as root, through the script.** Most of the suite only drives the API over HTTP and +works fine without, which is why it was run unprivileged for a long time — but what needs +root then *skips*, and a case that always skips proves nothing. `sudo ./run_tests.sh` is +what makes those run, and going through the script rather than `sudo pytest` is what makes +it work at all: `pip install --user` put every dependency in the invoking user's +site-packages, which a root process does not read, and the editable `firegex` is a `.pth` +there — a file only processed for *site* directories, so putting that path in `PYTHONPATH` +imports pytest and still leaves `pyworker.py --check` unable to find the filter library. +The script installs into the environment it is running in, which resolves all of it. Two +things that trip a root run were found by trying it, and both are fixed rather than worked +around: `tests/requirements.txt` was missing `fastapi` (see the file for why one unit test +needs the backend's own dependency, and why its absence stops the *whole* run rather than +one file), and `./run_tests.sh unit` — advertised in the script's own help — took `unit` as +a **password** and ran everything against an instance it could not authenticate to. + +| Directory | Needs an instance | What it holds | +|---|---|---| +| `unit/` | no | `firegex.regex` against libhs, the `firegex.pyfilters` models, the address→nftables translation | +| `integration/` | yes | the product, parametrised over the combinations | +| `standalone/` | yes, and restarts it | settings read once at process startup | +| `helpers/` | — | the API client, stand-in services, certificates, the traffic `Channel` | +| `bench/` | yes | measurement, not testing; `pytest` does not collect it | + +**The combinations are parameters, not a shell loop.** `integration/conftest.py` holds `Layer(transport, ipv6, tls)` and the fixtures that fan out over it — `inspecting_layer`, `filtering_layer`, `proxy_layer`, `tls_layer`, `external_layer`, `any_layer`. Running both network layers with the same filters is the point: a filter that only works on one of them is exactly the failure the unified model exists to prevent. This was a shell script invoking one 1100-line program eight times, which meant the first failure ended the run and the other seven combinations were never reached, and a host without IPv6 on loopback failed rather than skipped. + +Three things that have to stay true of this layout: + +- **`helpers/`, never `utils/`.** The backend's own package is called `utils`, and this directory's used to be too — so whichever imported first won, and `unit/test_nftables_addresses.py` (which imports the backend) and everything importing `helpers.firegexapi` could not be collected in one pytest process. That name collision is why the suite *could not* be unified before. +- **The interception canary.** It is perfectly possible to reach the API of an instance whose datapath protects a different machine — a container without host networking, a VM with the port forwarded. Every call returns `ok` and not one packet goes near any of it, which without a check reads as dozens of "the filter did not block" failures rather than as a suite pointed somewhere it cannot reach. `datapath_in_our_path` asks once, plainly, and everything needing interception is skipped with the reason. +- **Skip with a reason rather than fail.** A laptop with no IPv6 loopback, a non-root run, an instance elsewhere — none of those are firegex bugs, and `-ra` is in `addopts` so the summary says which half of the suite the machine declined. + +`Channel` (`helpers/traffic.py`) is the one place that knows whether TLS is in the way. The old suite branched on it at every traffic check, about twenty times. + +`helpers/quicserver.py`'s `h3_is_blocked` counts **only a closed connection** as a block, because that is what a refusal on that layer does; a malformed request, an unreachable port and a timeout are failures with nothing to do with a filter, and catching them too let "the filter blocked it" pass for reasons that are not a block. It found one immediately: `authority()` brackets an IPv6 literal, and without that every IPv6 exchange was answered `H3_MESSAGE_ERROR` by the far end and read here as a refusal. + +Run the suite **twice in a row** when changing anything about addresses or TLS: state left behind by the first pass only shows up on the second. + +`standalone/` is opt-in because it drives `run.py stop`/`start` itself — the IP allowlist and the boot-time auth mode are read once at startup, so there is no way to test them against a running instance. It restores an unrestricted one at the end, which every other module depends on. + +`tests/integration/test_grpc.py` and `tests/integration/test_http_versions.py` are the two that say whether HTTP/2 is really filtered. The second is **the** test of the whole rendering: one filter file, unchanged, in front of an HTTP/1.1 service, an HTTP/2 one and an HTTP/3 one, blocking the same request on all three — it can only pass because each version is shown to the chain as the same HTTP/1.1. `grpcio` is an optional dependency alongside `aioquic`, and everything needing it skips with a reason, because a gRPC service is the honest way to stand up a real HTTP/2 service with an HPACK-compressed path and a trailer-section status. + +The Rust datapath has its own suite, which needs no running instance but does need the production toolchain (vectorscan and Python 3.14 — the filter library imports `compression.zstd`): +```bash +docker build --target compiler -t firegex-compiler . +docker run --rm -v "$PWD/backend:/execute" -w /execute/proxysrc firegex-compiler cargo test +cd backend/proxysrc && cargo test # on a host with libhs + fgex installed +cargo test --release --test throughput -- --ignored --nocapture # the matcher benchmark +``` +Mount `backend/`, not `backend/proxysrc/`: `tests/pyworker.rs` launches `../modules/services/pyworker.py`. That file imports `firegex` and the library imports `brotli`, neither of which the compiler stage installs — without them on `PYTHONPATH` eight pyworker tests fail with a `ModuleNotFoundError` buried in captured output and read as real failures. The runtime image has both (`/usr/local/lib/python3.14/site-packages` and `/usr/local/lib64/...`), and the repo's own `fgex-lib` has to come *first* on the path or a stale image's `firegex` wins. + +The suite runs against an instance with authentication turned off as well: `login()` reads a 403 there as "there is nothing to log in to" rather than as a failure, since every request is already allowed. `integration/test_auth.py` is the exception and skips itself — it is about passwords, and those endpoints answer 403 in that mode by design. Note `BearerSession.post` copies the header dict rather than writing into it: the form `Content-Type` belongs to the login request alone, and it used to stick, which went unnoticed only because a successful login replaces the whole dict a moment later. + +CI (`.github/workflows/release.yml`) runs **only on `release: published`**, not on pushes or pull requests — nothing validates a PR automatically. Its `run_tests` job (`ubuntu-latest` + `ubuntu-24.04-arm`) does `python3 run.py start -P testpassword` then `cd tests && ./run_tests.sh`, and gates every publish job below it. The tests' dependencies are pure python; `iperf3`, matplotlib and numpy belong to `bench/requirements.txt` and CI installs none of them. + +## Architecture + +### Three tiers +React (Vite + Mantine + react-router + TanStack Query + socket.io-client) → FastAPI backend → per-transport datapath binaries that do the actual interception. The backend mostly orchestrates: SQLite-backed config, spawning/talking to those processes, and deploying the built React app. + +### The layers are compared in the interface, not only in the docs +`components/Services/LayerChoice.tsx` puts the trade in front of the operator where the choice is made: a table of what each layer costs, plus the paragraph for the one selected. `transportSummary()` carries the same sentence onto every service row and detail page. Keep them in step with `docs/services.md` — the point is that an operator never has to read documentation to find out that one layer reassembles in the kernel and the other has a kernel-guaranteed fail-open. There used to be a `losesClientAddress()` beside them, marking the one combination that gave the client's address up; source transparency is unconditional on every layer now, including UDP, so it answered `false` for every input and the badge it gated could not render. Do not reintroduce it without a combination that actually loses something. + +### The two layers (`backend/modules/services/`) +The central idea, and the thing to preserve when changing anything here: **how traffic is intercepted and what is done with it are independent choices.** + +- **Network layer** (`transports.py`): `nfqueue` — packets queued to userspace, verdict handed back, nothing terminated, kernel fail-open via `NFQA_CFG_F_FAIL_OPEN` plus `bypass` on the nft rule. `proxy` — the connection is terminated and reopened, which buys exact rewriting, kernel reassembly, backpressure and an ordered chain, and costs the kernel backstop (rebuilt by hand in `proxysrc/src/filter.rs`). + + **The proxy is not the slow one, and the docs used to say it was.** Measured: 2.2× NFQUEUE's bulk throughput at one thread and 4.7× at eight, because NFQUEUE pays a userspace round trip *per packet* while the proxy pays once per connection and then the kernel moves the bytes. `NTHREADS` reaches the Rust engine too since 5.0.0 — `main.rs` builds its tokio runtime with that many workers rather than `#[tokio::main]` taking every core, so `run.py --threads` means one thing on both layers instead of one thing on one and nothing on the other. On short connections the two are indistinguishable — that claim cost three contradictory measurements before `tests/bench/connbench.py` started interleaving the layers within each pass instead of measuring one and then the other, which was handing the machine's drift to whichever went second. The trade between the layers is fail-open and transparency, not speed; keep `docs/services.md`, `LayerChoice.tsx` and `transportSummary()` saying that. `external` — a stateless destination rewrite that puts a proxy the operator wrote in the path (this is what `porthijack` used to be); no firegex process is involved, so `check()` refuses any filter, since a filter attached there would be one that never runs. +- **Filter layer**: `regex` (patterns compiled into one hyperscan database) or `pyfilter` (the user's Python). A service holds an ordered list of them. + +`Transport.check()` is where an impossible combination is refused **before** anything starts: more than `MAX_CHAIN_POSITIONS` filters on `nfqueue`, any filter at all on `external`, and the `tls` protocol anywhere but the proxy layer. It is asked twice: at creation with an empty chain, for what the service alone makes impossible, and again whenever a chain is pushed. UDP was never removed from the datapath: `cppregex` runs in block mode for it and `cpproxy` handles it, and the only thing that had gone missing was the choice in the UI. + +**UDP on the proxy layer (`proxysrc/src/udp.rs`).** The TCP path recovers where a connection was headed with `SO_ORIGINAL_DST`, which the kernel implements **for TCP and SCTP only** (`ENOPROTOOPT` otherwise). So UDP is relayed with **one socket per protected address**, each with its upstream fixed — nothing has to be recovered because nothing was lost. The engine prints one `UDP ` line per relay after `PORT`, and `_add_proxy` points each address's rule at its own port. + +**One spelling of an upstream, and one owner of the map.** `udp_relay_key()` in `nftables.py` is the only place the `host:port` / `[host]:port` token is built, and `udp_relay_host()` the only place an interface name is resolved to something a relay can bind — IPv4 first and link-local last, because an interface almost always carries an `fe80::` address it was never configured with and binding there puts the service where nobody dials. Both were written out by hand in four and three places respectively, and the engine's map, the rule that points at a relay and the manager adding an address to a running service all have to agree on the token exactly; when they stop, the symptom is a service whose UDP silently goes nowhere. For the same reason `ProxyTransport.udp_ports` is the *only* map of relays: `start()` returns that dict as `_steer["udp_ports"]`, so it is the same object the rules read, and `ServiceManager.address_added` asks `add_udp_target()` for a relay rather than keeping a second copy of the answer beside it. + +Client address transparency is preserved via `transparent::connect_as_udp` using `IP_TRANSPARENT` with mark `0x1339` and policy routing `table 1339`. New UDP addresses can be added to running services dynamically via `ADD_UDP` over the control channel without restarting the proxy. + +What UDP keeps: per-flow filter state keyed by client address and released after `IDLE` — UDP has no close to observe, so a timeout is the only thing that ends a flow — and both directions inspected. `Verdict::Reject` drops the datagram rather than closing anything, because there is nothing to close. Replies leave through the **listener** socket so conntrack rewrites them to appear from the address the client dialled; sending from the upstream socket would reach a client not expecting that source. The relay's own dial carries `SELF_MARK`, or the redirect rule would catch it and loop. + +`ConnectionMeta.l4` carries the transport to the filter, so the library's stream and HTTP models key off what the engine says rather than a hardcoded `True` in the worker. It is an enum and not a boolean because QUIC broke the question in two: a QUIC stream is ordered and reliable, so every model built on a stream applies to it, and it rides on UDP, so every rule that matches the wire matches UDP — one flag answering both got one of them wrong whichever way it was set. The worker is sent both words (`is_tcp` for the wire, `l4` for what can be built on it) and `RawPacket` exposes `l4` and `is_stream` beside `is_tcp`; `is_stream` is what `models/tcp.py` and `models/http.py` gate on. Absent, `l4` derives from `is_tcp`, because the NFQUEUE binaries carry TCP and UDP and have nothing else to say. + +### A service is a list of addresses +`service_addresses` holds `(ip, port, proto)` rows; the service itself holds the name, transport, protocol and TLS material. One daemon routinely answers on more than one address — a v4 and a v6 one, a public and an internal one — and all of them deserve the same chain. Two services with hand-copied chains is how one of them silently stops being protected. + +Consequences worth keeping: +- **Adding an address to a running service drops nothing.** `ServiceManager.address_added` installs only that address's rules against the datapath already running (`self._steer` holds what to point at). On TCP and TLS the proxy listener is dual-stack from the start, and on UDP new relays are bound dynamically on the fly, so adding any address (IPv4, IPv6, UDP, TCP, TLS) never drops connections or restarts the service. +- **`address_removed` drops the address from `self.srv.addresses` itself** rather than re-reading the table, because the caller takes the rules back *before* deleting the row. Re-reading would put it straight back, and the next restart would re-protect an address that no longer exists — with TLS, by claiming a port for a stream nobody owns, which then refuses the next service that wants it. +- **`proto` is denormalised onto each address** and rewritten when the service's changes. That is what makes `(ip, port, proto)` a usable uniqueness key, so a TCP service and a UDP one can share an address exactly as the kernel allows. +- **The `external` proxy endpoint belongs to the address, not the service**, and a partial unique index forbids two addresses sharing one: the return rule recognises the operator's proxy by address and port to put the original port back, so two addresses behind one endpoint could not be told apart on the way out. +- **The proxy listener is dual-stack whenever the host supports IPv6** (`[::]:0`), and `transparent::unmap` turns the v4-mapped peer and local addresses back into IPv4 before anything else in the engine touches them — conntrack, the source-preserving dial and the metadata a filter reads all want the address the packet actually had. +- **Addresses accept network interface names as well as IPs.** An entry in `service_addresses` can be an IP/CIDR or a network interface name (1-15 chars, e.g. `eth0`, `wg0`, `tun0`, `lo`), checked via `utils.is_interface_name` and marked by `Address.is_interface`. The interface is not a second spelling of an address, and the interface is what the operator is choosing when the address is somebody else's to hand out. The frontend says which of the two it understood *while it is being typed* — `addressKind` in `components/InterfaceInput.tsx` puts it on a badge in the field, because `eth-0` is a perfectly good interface name and a perfectly wrong address, and the two are matched by completely different rules. That rule is written once on the frontend too, as `isInterfaceName`/`isIpAddress` in `js/utils.tsx`, reading the same charset the backend does — it used to be pasted into `AddEditService.tsx` and `AddressList.tsx`, two copies of a rule the picker beside them did not consult at all, which is how the hand-off layer came to offer interface names in the list and refuse them on submit. `InterfaceInput` lists interfaces and addresses as two groups, each interface carrying what it holds right now (one question decides between them: "and what is on it"), and stays a combobox rather than a select because an address a service will answer on need not exist on this host yet. **Link-local is not offered** — every interface carries an `fe80::` it was never configured with, which doubled the list with addresses that would put a service where no client looks for it, the same reason `udp_relay_host()` takes them last; one typed by hand is still accepted, and an interface holding nothing else says "link-local only" rather than listing it. A row is **the badge as a column, and everything else in the one beside it** — value, then the note, wrapping inside that column and never under the badge. Neither of those two is given a fixed share, because which one is long changes with the group (the addresses a link carries, or the address itself): fixing it took the whole width from a full-length IPv6 and left the other wrapping a character per line, a row a screen tall beside an address cut off mid-digit; and three items in one wrapping row put that same address at the container's left edge, under the badge, three lines tall. The list's own width is **computed from the longest value in it**, between a floor and a ceiling, and three simpler rules were wrong first: inherited from the content it collapsed to the field's width, because content allowed to wrap asks for less than it needs to stay on one line; pinned to the widest row any host could produce, every list was that wide beside fields half the size; and a floor alone put the one long address back over three lines. A host answering on `10.60.3.1` gets the floor, one carrying a full-length IPv6 gets the room to show it with its note beside it, and monospace is what makes the arithmetic honest. **The row beside them is laid out by the same arithmetic, and got it wrong the same way.** Its note (`now 10.10.0.3`, `on eth0`) is a flex item with a basis wide enough for a full-length IPv6, because a basis is what decides whether an item shares a line or drops to one of its own — at 80px it shared, and the first row with a couple of tags on it squeezed the note into the gap that was left and broke an IPv6 across three lines mid-digit. A min-width beside that basis is not the fix and was tried: on a phone the column is narrower than any floor worth setting, so the floor pushed the address past the card's edge, and it buys nothing anyway — an item only shares a line when its basis fitted, and on a line of its own it is already as wide as it can be. The row's own container grows to what the buttons leave rather than to its contents, or each row is as wide as its own longest line and two rows of one list wrap opposite ways: the one carrying a full IPv6 kept its note inline while the one saying `now 127.0.0.1` pushed it down. `overflowWrap: anywhere` stays as the last resort, which is what a 375px screen needs and nothing wider should reach. **`AddressList` reads in the same words it was chosen in** — the same badge at the same `BADGE_WIDTH`, then the value, then `on eth0` for an address and `now 10.10.0.3` for an interface — because a row an operator comes back to check, described differently from the list it came out of, is a reason to wonder whether it is the same thing. What is *shown* is `bareAddress`: the backend stores what it parsed, so `127.0.0.1` comes back as `127.0.0.1/32`, a spelling nobody types that reads like a range beside real ones; a host-length prefix is dropped for display and a real one is kept, which is also what tells `addressKind` apart. A real range survives an edit for the same reason — the modal seeded itself by cutting at the first slash, so opening "move this address" on a `/16` and saving narrowed a service to one host of it. Where interfaces are not an answer — the `external` proxy endpoint, and the service address on that layer — they are not listed at all and a name typed there is badged red, rather than offered and then refused on submit. Nftables matches use `meta iifname ` for inbound/prerouting and `meta oifname ` for outbound/postrouting/route. On `proxy` (TCP/TLS), redirect rules match `iifname` and `route_chain` marks return packets with `0x1339` on `oifname`. **The output hook cannot match either of those, and matches the interface's addresses instead.** `iifname` means nothing for a packet this host generates, and `oifname` is the wrong question twice over: `oifname eth0 dport 80` says *we* are calling somebody else's port 80, which is the host's own outbound traffic and none of firegex's business — and a connection from this host to one of its **own** addresses is routed through `lo` anyway, so it would never match. So `_add_proxy` resolves what the interface carries (`interface_addresses`, link-local dropped for the reason `udp_relay_host` takes it last) and writes one `daddr` rule per address. For a while it wrote nothing there at all, on the correct observation that `oifname` would hijack the wrong traffic, and the consequence was a service intercepted for the outside world and silently not for anything on the box — reported by an operator curling their own service and watching a blocked payload sail through. The cost is that an address the interface gains later is not covered *for traffic this host generates* until the service restarts; inbound keeps following the interface, and `udp_relay_host` pays the same price for the same reason. Each of those rules carries the interface's name as a **comment**, because nothing else in it does: `delete()` finds rules by what they match, and without it every one would outlive the service that installed it as a redirect to a port nobody is listening on. On UDP `proxy`, the primary IP assigned to the interface is resolved dynamically via `psutil.net_if_addrs()` to bind the relay. On `nfqueue`, interface traffic is queued natively in prerouting/postrouting without needing an IP. On `external`, interfaces are disallowed because return rewrites require a concrete IP. + +**An address says what is spoken at it and where the service is, and those are two different questions from the one the rules ask.** `service_addresses` carries `edge` — `tcp`, `tls`, `udp` or `quic`, the same words a service speaks minus `http`, which is the one that means "ask the addresses" — and `target_port`, which is where the traffic goes when that is not the port it arrived on. The two are scoped by different things, and confusing the two scopes is what the first version of this did. `edge` is only ever *chosen* on an `http` service, because that is the only protocol whose addresses are not all the same; elsewhere it is the service's own and a value disagreeing with it is refused, while `proto` (what the kernel matches) stays `L4.l4_of(edge)`. `target_port` is a question for the **layer**: publishing is being reachable one more way than the service is, and what that takes is something that *dials*. The proxy terminates the connection and opens the one to the service, so it can open it on another port, whatever the protocol — a plain `tcp` service published on a second port is exactly as real as an HTTPS one, and `_address_target()` was keyed on `http` only because `http` was the protocol the case was first noticed on. NFQUEUE hands the kernel a verdict on packets already on their way and opens nothing; `external` gives the traffic to the operator's own proxy, which decides for itself where the service is. On those two it is **refused** rather than stored, because a port nothing reads would tell an operator their service is published while it is merely intercepted. **And every one of them is editable after the fact.** `edit_address` rewrites `ip_int`, `port`, `proto`, `edge`, `target_port` and `upstream` — it took the first two and the proxy endpoint alone for a while, which made the rest add-only and left an operator who had set one while creating the service with no way back to it but deleting the address. The edit already takes that address's rules back and reinstalls them, which is the whole cost of changing any of it, and only that address stops being steered meanwhile. Two properties hold it together: everything the row will say is worked out **before** the old rules come back, so a refusal changes nothing; and a field the caller did not mention **keeps what the row had**, because this is also the endpoint that merely moves an address, and a move sending only an address and a port would otherwise quietly unpublish it. Taking one off is a value (`0`, `same`), not an absence. The interface asks for all of it in one place, `AddressOptions.tsx`, used by the ⚙ on each row of the creation form and by the modal on the service's page: the capability rules and the wording were written twice before, which is how the form came to offer what the backend then refused, and how one setting came to be described in two sets of words on two screens. The wording rule there took two corrections to settle. It began as field names — "The service is on port", "The service behind speaks" — which were true and left the operator to work out the effect. Rewritten as pure effect, "Send it to another port" was **read backwards**: with a port at each end, an operator with a cleartext service on `:80` who wanted it reached over TLS on `:443` typed `80` as the address and `443` here, and got a service intercepted where it already listened and published nowhere. So the label names the **fact the operator supplies** and the description says **what firegex does with it**, naming both ends: "The service is on another port" over "Clients arrive on :443 — firegex dials the service on :80". And because the misreading is really about the model rather than the sentence, the empty state says where a second way in comes from: another address, not another port in this box. + +`target_port` is what makes firegex a **publisher** rather than only a transparent interceptor, on the addresses that use it: one daemon answering HTTP/1.1 in the clear on `:80` is reached over TLS on `:443` and over HTTP/3 on a UDP port, all three pointed back at `:80`, with one chain over them. It is per address and not per service because that is the shape of the thing — the service has one place it listens and several places it is published. + +**`resolve_target()` and `service_at()` are the two halves and must not be merged again.** The first is where the world knocks, which is what a *rule matches*; the second is where the service is, which is what the engine *dials*. They were one function for as long as those were one answer, and making it return the target was tried first: the rules then redirected the service's own port twice and left the published port unprotected, which is a service that looks configured and answers nothing. Everything that matches asks the first (`FiregexTables.add`, `Rule.matches`); everything that dials asks the second (`_udp_targets`, the relay key in `add`, `_published`). + +The TCP side needs telling because one listener fronts every address and recovers each connection's destination from conntrack: `FGEX_PROXY_TARGETS` at startup and `PUBLISH`/`WITHDRAW` on the control channel while it runs, both carrying `
|[=]`. UDP and QUIC need none of it — a relay's upstream is fixed when it is bound, so `ADD_UDP` already answered both questions. **Only `tls` is sent as an edge**, because only it is a promise the engine keeps (a cleartext client on that port is refused); "in the clear" is the permissive default that still terminates a client bringing TLS, and making it strict was tried and broke every existing `http` service, whose single TCP address has always carried both and decided per connection. + +**The service form asks three questions and the address row asks one, and everything else is behind a button.** It had grown into a wall: a five-answer protocol picker, a service-wide upstream control, a layer picker greyed out to one option, and two ports plus an edge on every address row — all of it in front of an operator who wanted a TCP service on one port. The shape now is **is it encrypted → which protocol → its certificate**, then the addresses, then the layer; and `AddressRow.tsx` puts a service published elsewhere, a service answering in the clear, and the hand-off endpoint behind one ⚙ per address, offering only what applies to the service being configured. What is set comes back as a **tag under the address that opens the panel it came from** — folded is not hidden, and an option an operator cannot see they set is one that will surprise them at the worst moment. **The layer picker is not shown at all for a service that is decrypted**, because decrypting means terminating and one layer does that: a control with two disabled options and one answer is a question nobody is being asked. It is replaced by the sentence, not dropped, since the layer is a real property of the service — and an effect pins `transport` to `proxy`, because a form that hides a control still submits its value. + +**An ordered chain works on every layer, and gets there differently.** The proxy walks a list inside one process. NFQUEUE cannot — its binaries fuse transport and filter — so there a chain is a chain of *processes*: one binary per filter, each with its own queue and its own pair of base chains, at `-307 + position` (prerouting) and `107 + position` (postrouting). A packet a filter accepts carries on to the next base chain in the same hook, so **filter order is base-chain priority order**. This was established by experiment, not assumed: two `cppregex` processes at adjacent priorities, where the second refuses traffic the first passed. The cost is a process and a reassembly pass per filter, which is why `MAX_CHAIN_POSITIONS` exists at all. + +Changing the *shape* of an NFQUEUE chain (adding, removing, reordering) cannot be pushed: the shape is the arrangement of processes and rules. `NfqueueTransport.reload` raises `ChainShapeChanged` and `ServiceManager.update_chain` rebuilds, logging that it did. Editing a pattern inside an existing filter still costs nobody their connection. + +### One engine per concern, never two +Two implementations of the same concept is the mistake this module was restructured to remove, and it is worth restating because it keeps being tempting: + +- **There is one HTTP/1.1 rendering, and every terminated HTTP protocol is shown it.** `proxysrc/src/http1.rs` holds it; `h2.rs` and `h3.rs` are adapters to hyperium's `h2` and `h3` crates and nothing more. Two copies of the rendering would be two things a filter can be shown, and the day they drift the symptom is a pattern that matches over one version and not another with nothing to say so. For the same reason neither protocol is reimplemented here: HPACK, QPACK, multiplexing and flow control are the crates' problem. +- **Regex matching is hyperscan, everywhere — there is no second regex engine in the tree.** `binsrc/regex/` on the NFQUEUE side, `proxysrc/src/hyperscan.rs` on the proxy side, a binding rather than a reimplementation. Adding `regex` back to `Cargo.toml` would undo this. +- **A pattern blocks. There is no second action, and `rewrite` is not coming back.** It existed, and was withdrawn because it could not keep the promise it made: rewriting scanned one chunk at a time, since bytes already forwarded cannot be taken back, so a match straddling two chunks was never rewritten. Measured — a pattern sent whole arrived redacted, the same pattern split across two TCP segments arrived intact, and blocking (which scans in stream mode) caught both. The miss was not the problem; its silence was — no block, no log, no counter, just a rule the operator trusted that had quietly not applied. The removal has to be complete to mean anything, and it was half-done for a while: an `action` column that was gone from the datapath still had a `rewritten` field travelling from `debug.rs` through the backend model to a panel in `RegexDebugger.tsx` that could never render, a single-member `Action` enum in `firegex.regex`, and a `replace_with` on the frontend `Regex` type. If any of those spellings reappears, it is residue, not a feature. +- **A pattern is validated against the mode it will run in**, which is streaming, because that is how a pattern follows a connection across chunk boundaries. hyperscan does not accept quite the same patterns in every mode (`a{1,1000}b` streams but does not block-scan), and judging by the wrong one refuses rules that work — the same class of lie as a tester that disagrees with the engine. Block-mode scanning survives for one job only: reporting *where* a match starts, so the tester can highlight it. `debug.rs` reports `unscannable` for patterns that are valid, will run, and cannot be highlighted. +- **The Python API is `firegex.pyfilters`, everywhere.** `cpproxy` embeds it; `modules/services/pyworker.py` runs it out of process. An earlier version of that worker defined a look-alike `@pyfilter` taking `(data, direction)`, which meant a filter written from the docs loaded on one transport and failed on the other. +- **A filter file's protocol is read off the file, never declared.** The library already decides when to call a filter from what its parameters are annotated with, so `infer_proto` in `firegex/pyfilters/internals` reads the same table back the other way: asking for an `HttpRequest` makes a file an HTTP filter, asking only for a `RawPacket` makes one run on anything, and asking for two *different* application protocols is refused naming both functions. `type_annotations_associations` is the single place a protocol is defined; a second list of names would be one more thing to keep in step, and the day it fell behind the symptom would be a filter that silently never runs. The backend stores the detected value on the row so the interface can show it, and `PUT .../code` is what recomputes it. Passing one in from outside is exactly what used to make an HTTP filter on a TCP service a 500: the *service's* protocol was sent to the datapath, so the filter was refused for asking for an `HttpRequest`, which is the only thing an HTTP filter does. +- **A filter file's functions run in the order the file defines them**, and the first to answer anything but ACCEPT ends the packet. `@pyfilter` sets a mark on the function and hands it back; `collect_pyfilters` reads a module's globals, which are in definition order. What that replaced was a *set of names* on a module-global attribute of the decorator, so the order was whatever hashing produced — which decided which function got credited with a block and whether an earlier `UNSTABLE_MANGLE` survived, with nothing in the file able to tell you which. The `position` column on `pyfilters` carries the same order to the interface, which used to list them `ORDER BY name`. Three consequences of that registry are also gone with it: `get_filters_info` had to clear it before and after (and reached it by `exec`ing import strings), two compilations could race, and a function registered from somewhere the module globals could not reach was registered and then not found. +- **A pyfilter file's functions are switched on and off individually, and the code is never edited to do it.** `pyfilters` holds one row per `@pyfilter` the code defines, reconciled with the file on every `PUT .../code`: functions that have gone are dropped, new ones arrive active, survivors keep the operator's `active` and `blocked`. Only the enabled names reach `__firegex_pyfilter_enabled`, which is already what decides whether the library ever calls a function. Three states are distinct all the way down (`Option>` in the spec, `*` / `=a,b` / `=` on the worker's argv): **absent** means every function the file defines — the case of a filter whose list was never derived, where defaulting to "none" would silently stop every filter running — while an **empty list** means the operator switched them all off. `ChainLink.enabled_functions` returns `None` for no rows and `[]` for all-inactive, which is the same distinction. +- **gRPC is a body format inside HTTP, not a protocol beside it, and that is what keeps it free.** `GrpcMessage`/`GrpcRequest`/`GrpcResponse` live under `"http"` in `type_annotations_associations` and nowhere else. The alternative — a `"grpc"` value in `filters.proto`'s CHECK and in `PROTO` — would have been a schema change, and changing that dict **deletes and recreates `db/services.db`**, so the price of a model would have been every service on the instance. Everything *around* a call already arrives through `HttpRequest`: the method is the path, the metadata are headers, the status is a trailer section. What did not was the body, which is a sequence of `[flag][4-byte length][protobuf]`. +- **The parser learned to release body chunks, and only the gRPC models turn it on.** A server-streaming or bidirectional RPC has a body that does not finish until the stream closes, so a model shown only finished bodies is called once, at the end — a log entry rather than a block. `release_body_chunks` on `InternalCallbackHandler` hands the message over as its body grows; it is off by default, and each gRPC model has its own `_parser_class()`, so `HttpRequest` keeps its contract exactly (twice: once with the headers, once with the whole message). Two details are load-bearing: the partial body is **undecoded**, because `content-encoding` is undone once at message-complete and half a gzip stream is not half a body (the models decline while one is in the way, which for real gRPC never happens — it compresses per message instead); and the cursor lives on the `InternalHTTPMessage`, not beside it, because a keep-alive connection carries many messages through one parser and a per-connection cursor would start the second one part-way in. Separate parser keys are not tidiness either: taking a frame *consumes* it, so two models sharing one parser would leave the second silently never running. +- **`RawPacket` is the only model that survives a datagram**, and that fact is stated once, as `DATAGRAM_SAFE_MODELS` in `routers/services.py`. Every other model declines to be built when `is_tcp` is false, so a filter asking for one on a UDP service would sit in the chain and never be called. `--check` reports the models a file asks for, saving refuses the combination naming them, and `Transport.check()` catches the case where the service was switched to UDP afterwards. Python filters *do* work on UDP on both layers: `cpproxy` gained `filter_action_udp` (no stream follower, no ack fixing — a datagram is complete in itself), and its per-flow contexts are bounded by `enforce_limit` because a datagram has no close to observe and a spoofed-source flood would otherwise accumulate one set of module globals per forged address. `handle_next_packet` used to throw for anything but TCP, with a message claiming UDP was supported. +- **A filter file is validated by the process that will run it.** `pyworker.py --check ` builds the module exactly as the worker does, compiles it, and prints one JSON object: `{ok, proto, filters}` or an error carrying **type, message, line, column, the source line and the traceback**. `validate_code()` runs it in a **subprocess with a timeout** — user code used to be `exec`ed inside the backend's own event loop by `detect_proto`, so a `while True:` at module level would have taken the interface down. `POST .../check` answers 200 even when the answer is no: a refusal is the result, and the editor wants a position to mark, not a string to print. Saving goes through the same check, so the operator is never handed "the filter code did not load: worker exited" and left to find the traceback in the service log. + - `_where_in_source` finds the line three ways: a `SyntaxError`'s own position; a traceback frame belonging to the file (`` counts as theirs — the library execs the source to find its filters, and everything the library runs from has a real path); otherwise any identifier in the message looked back up among the file's `def`s, **last definition wins**, because that is the one Python is left holding. + - `Filters` compiles once under the operator's file name instead of re-parsing per connection, so runtime tracebacks in the log name their file and their line too. + - A filter is called directly. `context_call` used to write the arguments into the module globals, `eval` a call expression built from them, and delete the three keys again — per filter, per packet — which is exactly `func(*args)`, because the operator's code was `exec`ed into those globals and so the function's `__globals__` *is* that dict. + - The default for `FGEX_STREAM_MAX_SIZE` is 1 MiB, as documented. It read `1*8e20` under a comment saying "1MB", so `FGEX_FULL_STREAM_ACTION` never fired unless a size was set by hand, and the float it produced would have been refused by the setter beside it. +- **The editor's hints are introspected, never written down.** `modules/services/pyapi.py` reads the models out of `type_annotations_associations` and their members out of the classes — including `writable`, from `property.fset`, which is how the interface can say that the payload is the only thing a filter may assign to. `GET /api/services/pyfilter-api` serves it and `pyfilterHints.ts` feeds Monaco's completion and hover from it. A second, hand-written description would be wrong the first time a model gained a property, and autocompletion that lies is worse than none. The route is declared **before** `/{service_id}`, or that one matches it and looks for a service by that name. +- **A filter that raises says so the same way on both layers.** `_on_exception` writes one sentence — the traffic was forwarded unfiltered — and it has to be reachable from either. NFQUEUE gets there directly: the interpreter is embedded in the binary, which sends `EXCEPTION`. The proxy cannot, because `pyworker.py` catches the exception in a separate process and answers with an ordinary ACCEPT, so nothing crosses the frame boundary that the engine could tell apart from a filter agreeing. It marks stderr with `EXCEPTION_MARK` instead and `Transport._pump_stderr` turns that back into the same call. The traceback is printed *beside* it, not instead of it: the traceback says which filter and which line, and the sentence says what happened to the connection — for a while the proxy printed only the first, which tells an operator a filter broke and not that traffic went through unchecked. The marker is spelled in both files because the worker is launched as a script and imports nothing from the backend, which is what keeps the user's code out of this process's imports; `tests/integration/test_resilience.py` asks for the sentence on both layers and so keeps them in step. +- **A block names the function, not just the file.** Both layers report `/`: the NFQUEUE binaries already reported the function name, and the proxy worker now returns it in the verdict frame (`[verdict][name len][name][payload]`). `ServiceManager._on_block` splits that one token in one place. A file with ten functions blocking 900 connections is not an answer to "which of them is doing it". +- **Nothing below the application layer crosses into a filter.** `RawPacket` exposes addresses, ports, family and direction as read-only metadata, and `data` — the payload — as the only writable thing; `mangled_data` comes back, never a packet. That boundary exists because the two layers cannot honestly offer the same thing underneath it: NFQUEUE has a real header whose rewriting desynchronises the stream, while the proxy terminated the connection and writes its own. The old model papered over that by handing the proxy a literal `FAKE:IP:TCP:HEADERS:` prefix as `raw_packet`, so the same filter did different things depending on where it was attached. On the proxy the metadata is *better* than a header would be — the engine knows both ends and sends them in a `KIND_OPEN` frame (`ConnectionMeta` → `Filter::connection_opened`), once per connection rather than on every chunk. +- **The order the operator set is the order that runs.** `parse_ruleset` groups patterns by their `filter` id and keeps the positions the backend sent. An earlier version grouped every block, then the Python — which turned the chain into a set, so a filter placed after another could run before it, and nothing said so. Within one filter the operator expressed no order, so blocking goes before rewriting there; across filters, never reorder. `tests/rules.rs` pins both. +- **The regex tester runs the engine.** `POST /api/services/debug-regex` shells out to `fgex-proxy --debug-regex`, so what it says is what will happen. A `re`-based approximation would bless patterns hyperscan rejects. + +### Per-module pattern +Each module (`services`, `firewall`) follows the same shape: +- `backend/routers/.py` — FastAPI endpoints + its own SQLite DB, via `utils.sqlite.SQLite`. +- `backend/modules//models.py` — plain model classes, `from_dict`. +- `backend/modules//firewall.py` — `FirewallManager`/`ServiceManager` pair: the manager tracks one `ServiceManager` per DB row, which starts/stops that one service. +- `backend/modules//nftables.py` — builds/tears down the module's nftables rules; extends `utils.NFTableManager` (a `Singleton`), so there is exactly one rule-set owner per module. + +**Every nftables object firegex creates is named `fgex_`, and lives in a table firegex owns.** The services module holds `table inet fgex` with `fgex_queue_in_`, `fgex_queue_out_`, `fgex_hijack_*`, `fgex_divert`, `fgex_nat`, `fgex_nat_out`, `fgex_route`; the firewall module holds `fgex_filter` and `fgex_mangle` per family, with base chains `fgex_input`/`fgex_forward`/`fgex_output`/`fgex_prerouting`/`fgex_postrouting` jumping to `fgex_rules_in`/`_out`/`_fwd`. One `grep fgex_` over the ruleset is the whole footprint. + +**An interface rule says where traffic *arrived*, not where it was going, and that is deliberate.** `iifname docker0 tcp dport 443` matches every container's **outbound** HTTPS as well, because that traffic arrives on `docker0` too, and a `redirect` then takes it into the engine, which terminates it with this service's certificate — measured: `pip install` inside any container failed with `CERTIFICATE_VERIFY_FAILED: hostname mismatch` while such a service was running, and worked the moment it stopped. Pinning the destination to the addresses the interface carries was written, tested and **reverted**, because it removes the one case interfaces exist for: an operator names a link when **the address is not theirs to know**, which includes firegex in front of a service on another machine reached through it, and a rule pinned to this host's own addresses silently stops protecting that. Protecting a link protects what crosses it. Naming one that carries other people's traffic is a choice, and the answer to not wanting it is a narrower address, not a narrower rule — `docs/services.md` says so where the choice is made, because it is surprising the first time. Do not re-narrow this without a way to tell the two deployments apart. + +**An interface stands for the addresses it carries, and every place that needs an address has to resolve it — three of them gave up instead.** `udp_relay_host()` got this right from the start; the others did not, and each failure was silent in its own way. `_add_proxy` wrote no output-hook rule at all, so a service protected on `lo` filtered the outside world and not a client on the box. `_published()` and `ServiceManager.address_added` skipped the announcement, because the engine keys that map on what `SO_ORIGINAL_DST` hands back and an interface name is not an address — so an HTTPS edge declared on `lo:443` was never announced, the engine met the connection knowing nothing, terminated it as an ordinary one and dialled `:443` where nothing listens. HTTPS on that address simply did not answer while the same address written `127.0.0.1:443` worked, which is how it was found and is as confusing a symptom as this module has produced. All three now go through `interface_addresses()`, which returns the concrete addresses (link-local dropped) and hands back a single-element list for an address, so the two spellings take one path. The cost is the one `udp_relay_host` already pays: the answer is resolved when the rules are installed, so an address the interface gains later is not covered until the service restarts. When adding anything that needs *an address* rather than *a match*, the question to ask is what it does when handed `eth0` — and the answer is never "skip it". + +**The proxy layer's redirect runs before anybody else's destination rewrite, and that is not tuning.** `NAT_PRIORITY` is -120: below `dstnat` (-100), above conntrack (-200). Docker and podman publish a container's port by appending their own chain to `nat PREROUTING` and `nat OUTPUT`, which `iptables-nft` registers at exactly `dstnat` — the priority these chains used to share. Two base chains at one priority in two tables are ordered by neither of them, and it was measured going the wrong way: the DNAT rewrote the destination to the container's address, firegex's rule (which matches the *published* address) stopped matching, its counter stayed at zero, and a request carrying a blocked pattern was answered by the container. The service still read `ACTIVE`. Running first, the published port is captured before anyone can rewrite it — the property the NFQUEUE layer always had for free by living at `raw`, three hundred below all of this, which is why that layer filtered a containerised service correctly the whole time. Nothing downstream is lost: the engine's own dial carries `SELF_MARK` and is skipped by the rule it would otherwise match, so the container runtime's DNAT still does its job on the way out. `tests/integration/test_addresses.py::test_a_container_runtimes_rewrite_does_not_take_the_traffic` stages a competing rewrite at `dstnat` and asks for the block; it was confirmed to fail with the old priority before being kept. A CTF service is almost always in a container, so this is not an edge case — it is the deployment. + +The firewall module is the one that had to move to get there, and the reason is worth keeping. It used to write into the tables called `filter` and `mangle` — the ones `iptables-nft` claims — adding base chains named INPUT and FORWARD beside iptables' own and driving their policy. That broke two things, both reproduced before being fixed: + +- **`iptables` stopped working on the host.** `iptables-nft` refuses an entire table as soon as one rule in it cannot be expressed in iptables' own format, and it stores conntrack matches as an `xt` blob where nft writes a native `ct state`. Bisecting the rules firegex writes, `ct state` was the only one that tripped it — which is to say `allow_established` and `drop_invalid`, on in any sane configuration. From then on `iptables -L` and `iptables-save` answer ``table `filter' is incompatible, use 'nft' tool``, on a box where Docker, ufw or the organisers' scripts are the ones asking. +- **Stopping firegex opened the host.** `reset()` re-added `filter INPUT` with `policy accept`, and in nftables `add chain` on an existing chain *updates* its policy. That chain was the administrator's: a host carrying default-deny had it silently removed when the firegex firewall was switched off. + +A base chain carries its own policy wherever it lives, so default-deny is expressed exactly as before and a `drop` is terminal whichever table's chain reached it — nothing about what gets blocked changed. What changed is that firegex now adds a layer instead of taking one over: an `accept` in `fgex_rules_in` means "firegex does not block this", never "nothing else will", which is all it ever honestly meant. `reset()` is now deleting two tables per family (`add` before `delete`, because a batch is atomic and deleting a table that is not there fails all of it). + +`Rule.table` is still `filter` or `mangle` in the database, the API and the interface — it names the hook a rule is evaluated at. `NFT_TABLES` maps it to the table it lands in; the two were one string only because that string used to be somebody else's table.<> + +A refused edit must change nothing: `_apply_chain(service_id, undo)` in `routers/services.py` rolls the write back when the transport rejects the resulting chain. Without it an operator is told their filter was rejected and then finds it in the list, and the next start fails for the reason they thought they had avoided. + +Order is the load-bearing detail in `services/firewall.py`, because getting it wrong is not an error anybody sees — it is a window during which traffic goes unfiltered. Starting: the capture interface if the service decrypts (the engine opens its socket to it at startup, so one that appears afterwards is one this process will not find), then the datapath, then the nft rules that point at it. Stopping is the exact reverse. + +### Block statistics (`modules/services/stats.py`) +A cumulative counter per filter, per pattern and per `@pyfilter` function lives in the database; the bucketed history answers the other question — *when* did this start. Blocks are counted per filter into one-minute buckets, kept for `HISTORY_BUCKETS`, and flushed at most every `FLUSH_INTERVAL` seconds. Both properties are load-bearing for the same reason the live log's are: a service refusing thousands of connections a minute must not be able to take the backend down through its own bookkeeping, which is what one row per block would do. There is deliberately **no background task** — flushing happens on the next write once the interval has passed, and always on a read — so nothing has to be started, cancelled or waited for at shutdown. The frontend draws it as hand-written SVG (`StatsPanel.tsx`) rather than pulling in a charting library several times the size of the page it sits on. + +`share` is computed in the backend, not the browser, so a chart and the table beside it cannot disagree about what a rule accounts for. + +**History is stored per *rule*, not per filter, and that is what makes a time range mean one thing.** `block_history` keys on the token the datapath reported — a pattern id, `/`, or a filter id — so `GET .../stats?range_from=&range_to=` can answer the chart, the per-filter totals, the per-pattern totals and the shares over the *same* window. Storing only per filter would have left the chart honouring the range and the tables beside it reporting all of time. A filter's total is the **sum of its rules**, not a lookup of its own id: that token appears only when nothing named itself. + +`bucket_width` takes either a `target` bar count (the default, which is what makes the bar widen with the range) or an explicit `want` in seconds from the operator, and both are floored at `MAX_STEPS` bars — a step fine enough to melt the chart is widened, never refused, and `bucket_seconds` reports what was used rather than what was asked. A range never grows the number of buckets, only their width (`bucket_width`): everything is an aggregate of the one-minute rows, so twelve hours costs the same rows as one and is drawn in the same handful of bars. Ranges are clamped to **two** floors — `retention_start()` and the service's `filtering_since` — and the served range is reported back, so the interface can say what it actually showed. `filtering_since` is written by `_note_filtering()`, which is called from `start_service` and from `_apply_chain` (the two places the two halves of "running, with an active filter" change) and once per service at `startup()`, which is what covers services restored from a backup or predating the column. It is written once and never moved: a service stopped and started again has a real gap in its chart, and moving the mark would hide it. Note the clamp happens *after* the caller's ends are put in order, and the `filtering_since` floor applies **only to a window that reaches into the service's life**: a window that ended before it began is answered exactly as asked and comes back empty, because it is. Dragging its start forward instead would have reported the current minute's blocks under a range an hour old — buckets are minute-aligned, so a zero-width range is not an empty one. `all_time` is reported alongside `total` so a narrow window showing nothing is not read as a contradiction against the cards. + +`StatsCharts.tsx` draws bars, lines, stacked area and 100%-share from the same series — four views, one dataset, still hand-written SVG for the reason the first chart was. Hover is tracked once on the container and turned into a step index, not per shape: a line and an area have nothing under the cursor to attach a handler to, and one handler means the four charts cannot disagree about which step is being pointed at. The readout itself is HTML positioned over the plot, for the same reason the axis labels are — `preserveAspectRatio="none"` stretches rectangles correctly and distorts glyphs. `clock()` is exported from there and used by the axis labels, so a bar's tick and its readout cannot disagree about what a step is called. + +**Editing a pattern keeps the row, resets its counters, and hands its history to the filter.** `PUT .../regexes/{id}` validates the *resulting* row through the engine (changing the action alone can invalidate a stored pattern, since blocking compiles for streaming and rewriting for block scanning), and when the pattern text itself changed it resets `blocked` and calls `stats.disown_rule` — counts belong to the text that produced them, so the new text must not inherit them. That call happens **after** `_apply_chain` accepts, never before: a refused edit has to change nothing, and history moved ahead of the verdict cannot be put back by `undo()`. The rule's *pending* batch moves with it, or the last few blocks of the old pattern would be flushed under the new one. + +What `disown_rule` must not do is **delete** those rows, which is what it used to do. Those connections were refused, by that filter, in those minutes, and the chart is answering *when this service was under attack* — a question a later typo fix has nothing to say about. Deleting took a bite out of the timeline on every correction, and on a filter holding one pattern it emptied the chart. So the rows are re-keyed to the filter's own id, already the token the datapath reports when nothing finer named itself: `window()` and `totals_by_filter()` group by `filter_id` and do not change at all, while `totals()` groups by `rule_id` and correctly credits the new text with nothing. + +The cost is that some blocks in a range then belong to no rule still present — which was already true of a *deleted* pattern, whose rows are left behind on purpose. The stats route closes that by adding one **residual** entry per filter for the difference between what the filter was credited with and what its current rules account for (`residual: true`, prose rather than a pattern, so `StatsPanel` prints it as text and not in a `Code` box). Without it the per-pattern list quietly adds up to less than the chart above it, and the operator is left hunting for the difference. Note `check_pattern` coerces `case_sensitive`: a caller checking a row it read back from SQLite hands over `1`, and the engine's request format refuses an integer where a boolean belongs. + +**Traffic totals are reported in the unit each layer can honestly produce, and the two are never divided by one another.** This is the part to preserve: + +- **Packets and bytes** come from `counter` statements on the intercept rules — free, kernel-side, and unskewable because no userspace filter is involved in producing them. `FiregexTables.traffic()` reads them back, from **one inbound chain per transport**: the nfqueue layer installs a rule pair per filter position all seeing the same packets, so summing them would multiply the answer by the length of the chain. +- **Proxy services report no packet count at all.** Their rule lives in a `nat` chain, and conntrack translates a connection once — every packet after the first is handled without the rule being walked again. That counter is *new connections wearing a packet label*, so it is dropped rather than shown. +- **Connections** come from the engine's periodic `STATS seen=… refused=…` line. They live on `ProxyStats`, not on `ChainStats`: a chain is replaced wholesale on every ruleset push, and counters that reset on a rule edit would make the ratio jump for a reason nobody could see. Because both numbers are connections, `refused_share` there is exact. +- NFQUEUE has no connection to count, so it reports `null` for the share rather than dividing refused connections by a packet count. **The asymmetry is the difference between the layers showing through; do not "fix" it by inventing a denominator.** The engine's counters arrive on a timer, so a test that reads them has to wait for them. + +### The live log (`modules/services/logs.py`) +A bounded ring per service plus a coalescing flush over a `log` Socket.IO event (distinct from `update`, which only invalidates queries). Two properties are load-bearing and easy to lose: the ring is capped, and the flush is on a timer with a per-batch cap that reports a dropped count rather than carrying it — a service refusing thousands of connections a minute must not be able to take the backend or the browser down through its own logging. + +Sources: block attribution (resolved to the filter name and pattern, one query per block, on a path that only runs when something was already refused), the user's `print()`, and the datapath's own stderr. That last one is why `pyworker.py` reassigns `sys.stdout = sys.stderr` before running user code: stdout is the length-prefixed protocol channel, and a `print()` landing in the middle of a frame gets the worker killed on every packet with nothing to explain it. `tests/pyworker.rs` has a regression test for exactly that. + +### The proxy datapath (`backend/proxysrc/`) +A Rust binary, built in the Dockerfile's `compiler` stage rather than a `rust` image on purpose: it links the same `libhs` the C++ binaries do, and a binary built against one distro's vectorscan and run against another's is a bad afternoon. + +- `filter.rs` — the chain, and everything that keeps a filter from taking the datapath down: `catch_unwind`, a deadline on `spawn_blocking`, and a filter that misbehaves losing its say rather than the traffic being held. `panic = "unwind"` in `Cargo.toml` is load-bearing, not tuning. +- Filters that keep state do so **per connection, per direction** (`ChainSessions`), because one client's bytes deciding another client's verdict is both a false positive and a way to smuggle a pattern past a filter by splitting it across two connections. +- `pyworker.rs` frames carry a connection id and a close frame, so the Python side can honour the documented promise that each stream gets its own module globals. +- Source preservation is unconditional: the engine always dials the service from the client's address. `ct status dnat` is what separates the engine answering a client from the service answering the engine, which is why redirect (not tproxy) works even when the service is on this host. + +### The `external` hand-off +Three nft chains, not two: prerouting for traffic from outside, **output for traffic this host generates** (which never reaches prerouting — the same reason the proxy transport has `fgex_nat_out`), and postrouting to undo the rewrite on the way back. A `mangle` target is one host, so `one_address()` strips the prefix that service addresses carry; + +Note for tests: Linux refuses a loopback connection to a port with no listener *without emitting a packet*, so no rule can rescue it. A hand-off test has to keep the real service listening and tell the two apart by their replies, not by which one answers. + +### Holding up under load + +**A service carries at most `max_connections` at once, and the operator says what happens to the rest.** It exists because the alternative was measured, not argued: silent connections cost two descriptors each — one from the client, one to the service, since the upstream is dialled on accept — and ~505 of them from one host exhausted the container's 1024 and took **every** service down. UDP was cheaper still: 600 datagrams from 600 forged sources in 0.03 s took 400 descriptors and held them a minute after the sender left. The same cap on the same shape already existed on the NFQUEUE side (`MAX_UDP_FLOWS` in `pyproxy.cpp`) and had never been carried to the Rust engine; TCP had none anywhere. + +Say what it buys, and no more. **A cap does not save the service being attacked** — it cannot tell a connection that is silent because it is an attack from one that is silent because the client is slow, so an attacker who fills the limit still fills it. What it buys is that one service's attacker stops being everyone's: measured, 400 silent connections against a limited service left the service beside it answering. Do not let the interface or the docs imply otherwise. + +- One number for both halves, because TCP connections and UDP flows spend the same descriptors and two limits for that would be two things to keep in step. `0` is no limit, kept as a value rather than an absence, because "unlimited" is a choice. +- `over_limit_forwards` is the same trade as fail-open at a different moment, and off by default for the same reason: a filter that quietly stops filtering is the worse surprise. +- Folded into the form's one **Advanced settings** accordion, not dropped: these are settings an operator reaches for once — after something went wrong, or before a competition — and never again, and left in the open they are more things to read past on every service. One fold rather than one per topic, and its summary is built from what *differs from the defaults* (`changed`), so an untuned service says "Everything at its default" and a tuned one says exactly how. Folded is not hidden. What is inside depends on the layer, because the settings belong to different ones: the connection limits are the proxy's, fail-open is NFQUEUE's, and the hand-off layer has neither, so it gets no accordion at all. +- **Proxy layer only**, and absent from the form elsewhere. NFQUEUE hands the kernel a verdict on packets in flight; it accepts nothing and dials nothing. The form also resets it to `0` when the layer changes, because a stored setting that does nothing is the trap this whole file keeps arguing against. +- `Slot` is a guard, not a decrement at the end of `handle_connection`: that function has a dozen ways out — a refused chain, a failed dial, a timed-out handshake, a panic — and a counter that leaks on any one of them is a limit that tightens until it refuses everything. +- The accept loop **backs off** after a failed accept. That one is not a policy question: out of descriptors, the loop spun on `accept` → `EMFILE` → print → `accept`, measured at ~40% of a core with four clients knocking, writing a log line per attempt into the pipe the backend reads. + +**`first_byte_timeout` is what the cap cannot be.** A cap contains a phantom flood; a deadline ends it — measured, with the limit full and the attacker still holding every socket, clients got back in once it passed. Two properties are load-bearing and easy to break: + +- **Either direction counts.** The flag is set by whichever pump moves a byte first, so a service that greets its client (SMTP, SSH, most game protocols) satisfies it with its banner. Requiring the *client* to speak would hang every server-speaks-first protocol, and the failure would look like firegex breaking them at random. +- **Only until the first byte, never again.** A connection that has spoken and gone quiet is a session, and sessions think. This is not an idle timeout; making it one would close long-lived connections that are legitimately waiting. + +The flag is set *before* the chain runs, because a chunk a filter goes on to refuse is still a connection that said something. The watchdog fires through the same `stop_tx` a refusal uses, so both directions shut down cleanly rather than being dropped — a dropped peer reads a truncation, and this connection has done nothing to deserve one. Zero is off, and there is no default worth picking: how long a legitimate client may stay silent is a fact about the service, not about firegex. + +**The trace is deliberately in two places.** `_on_over_limit` writes `over_limit_hits`/`_first`/`_last` to the database *and* logs at warning level, rate-limited by `OVER_LIMIT_QUIET`. The log answers "why are clients failing right now" and is a bounded ring, so a burst at three in the morning is gone by breakfast; the columns answer "did we ever hit the wall", which is asked the next day. The engine's counter is cumulative *per process*, so the transport hands up only the **increase** — passing the absolute number would reset the durable count on every restart of the service. + +### TLS + +**The two controls that constrain each other read one rule.** `carries(transport, proto)` in `components/Services/utils.ts` is the only place the constraint lives: the protocol picker hides what the chosen layer cannot carry, the layer picker disables what cannot carry the chosen protocol, and `Transport.check()` refuses it on the server. Written twice, the two halves drift and the form ends up offering a combination the backend then refuses — being told no after filling in a form is a worse way to learn it than not being offered the option. Both pickers say no the same way: the option stays where it is, **disabled**, with a sentence underneath saying why and what would make it available. An option that vanishes leaves nothing to ask about — the operator is left wondering whether it exists at all or whether they misremembered seeing it — and two controls side by side that refuse in two different ways are a confusion of their own. + +**`ProtocolChoice.tsx` is the other half of that pair, and it exists for a confusion that was reported rather than guessed at.** The picker used to be a bare row of five buttons headed "Transport protocol" — five alternatives on one axis, which they are not. An operator with a plain HTTP/1.1 service in the clear had nothing to tell them whether `TCP` or `HTTP` was meant for them (both are true sentences about their service), and picking `HTTP` then asked for a certificate for something with nothing encrypted about it. It is now two questions, the first being the one that actually divides them and the one the certificate hangs off — **is the traffic encrypted?** — with the second asked *inside* that answer and offering only protocols of that kind: `tcp`/`udp` in the clear, `tls`/`quic`/`https` encrypted. Every option is a protocol under its own name, which matters: an intermediate version asked "on what transport?" and computed the value from the pair, showing the name only in a badge underneath, and that reads as the form having decided something on your behalf. `http` is labelled **HTTPS** in the interface — `protoLabel()` in `components/Services/utils.ts`, the one place a protocol is named, used by the picker, the service row and the detail page alike, because being offered `HTTPS` and then shown `HTTP` on the row reads as two different settings. It is still `http` in the database, the API and the docs; the rename is only about where it is chosen, since beside TLS and QUIC an option called "HTTP" reads as the cleartext web protocol. Two more things are load-bearing: the lock on the encrypted answer, which says before the click which branch will want a certificate, and the certificate fields sitting directly under the choice that asks for them rather than below the advanced settings — cause and request a screenful apart is most of why the request looked unmotivated. **And the inline text is one clause per option, deliberately.** What is needed while choosing is which option is yours; what a protocol costs and what filters attach to it is documentation, and it lives in the popover and in `docs/services.md`. Three successive versions of this component grew paragraphs explaining the trade inline and each one had to be cut back — the same lesson `LayerChoice` already records one section above. + +**TLS is one of `L4`'s values, beside `tcp`, `udp` and `quic` — not a boolean on top of one.** It is what the service speaks, and as a flag it made three combinations expressible that are not real: TLS on a UDP service, on NFQUEUE, and on a hand-off, each caught by a check written by hand. Two of those are now unrepresentable, and the third (`NfqueueTransport.check()`) is asked at **creation** as well as at start — `add_service` calls `transports.build_class(...).check(srv, [])` with an empty chain, which catches what the *service* makes impossible while leaving what the *chain* makes impossible to the moment there is one. A row that is created happily and then refuses to start every time is a trap. + +The second answer that split off from it: an address carries the transport **the kernel matches on** (`L4.l4_of`), not the service's own, because `tls` is TCP on the wire and `quic` is UDP. Without that, `(ip, port, proto)` would let a TCP service and a TLS one claim one `ip:port` between them, which is the same port twice. `l4_proto()` in `nftables.py` is gone with it — the address already holds the answer it used to compute. + +It belongs to the **proxy** layer: decrypting means terminating the connection, and terminating is what that layer does. That is not a lost feature so much as an honest one — TLS on NFQUEUE worked by putting nginx in front to terminate and re-encrypt, so the connection *was* terminated and the layer's one distinguishing property was already gone. + +**The engine terminates it, and a TLS service therefore occupies no port at all.** `proxysrc/src/tls.rs` decrypts from the client and re-encrypts towards the service inside the process that filters, so `resolve_target()` points the rules at the address the world dials, exactly as for a plain service. `ProxyTransport._tls_env()` hands the engine `FGEX_PROXY_TLS_CERT`/`_KEY` — **paths**, because a certificate on a command line or in an environment dump is a certificate in a log — written `0600` under `TLS_DIR` and removed after the engine is gone, never before. + +What that replaced: nginx terminated on one derived loopback port and re-encrypted from a second, with the filters attached to the leg in between. Protecting one address cost a pair of ports produced by hashing `ip:port` — *chosen* ports, which could collide with something real — plus a generated config, a `tls_streams` table keyed by address id, an nft chain of its own, and a failure mode where one unloadable certificate stopped nginx and took **every** TLS service on the instance down with it. All of that is gone: `modules/tls/` no longer exists, and the blast radius is now structural, since each service carries its certificate into its own engine process. + +**ALPN is mirrored, and the ordering that makes it possible is the design.** The client's handshake is *started* and held at the ClientHello (`tokio_rustls::LazyConfigAcceptor`), because that message carries the protocols the client will speak; the upstream handshake goes next offering exactly that list, and whatever the service picked is the single protocol the client is then told (`tls::with_alpn` / `tls::answering_with`, both cloning the config per connection — the list comes from the client, so there is no fixed set to precompute, and a config clone is nothing beside a handshake). Answering something the service did not say is how a proxy breaks a protocol it was only carrying: a client told `h2` against an HTTP/1.1 service sends frames nothing can read. When the service picks nothing the client is told nothing — no agreement is invented. `tests/integration/test_tls.py` pins all five cases, including both mismatches and a client that offers only one protocol. Before this, ALPN was dropped in both directions and every client silently fell back to HTTP/1.1. + +**Mirroring ALPN means `h2` can be agreed, and `h2.rs` renders it.** For a while nothing did, and the symptom is worth keeping because it is the failure this whole file is arranged against: HTTP/2 compresses its headers with HPACK exactly as HTTP/3 does with QPACK, so on an `h2` connection a pattern written against a request line matched nothing and a filter asking for an `HttpRequest` was handed the `PRI * HTTP/2.0` preface — parsed as one fictitious request — and then frames it could not read, so it was never called again. No block, no log, no counter. Only a body travelled in the clear, and since gRPC *is* HTTP/2, gRPC could not be filtered at all. The engine named it at `warn` once per process (`warned_h2`) and the docs said to turn HTTP/2 off on the service; both are gone, because the real fix is done. See the HTTP/2 section below. + +What cannot be carried, and is a limit rather than a missing setting: **TLS 1.0 and 1.1**, which rustls does not implement at all; the cipher list, which is fixed and forward-secret only; and **client certificates**, since `with_no_client_auth()` is the only honest answer when the certificate a client would present is one only the service can check. Say these are impossible rather than unconfigured — an operator who needs TLS 1.0 needs the `external` layer, not a flag. + +The key must be one `ring` will sign with — RSA ≥ 2048 bits, ECDSA, or Ed25519. nginx could be told to take a 1024-bit key with `@SECLEVEL=1`; rustls has no equivalent, so a smaller key is refused when the service starts. That refusal is worth having land well: `_pump_stderr` keeps the engine's last `[fatal]` line and `_died_because` puts it in the exception, because "the proxy engine did not report a listening port" is the symptom and the engine already printed the reason. + +**What the service behind speaks is its own question, and it belongs to the *address*.** `upstream` on `service_addresses` — `same`, `tcp`, `tls` — because that is where it has an answer: one daemon reached over TLS on one port and in the clear on another is re-encrypted for the first and handed the plaintext for the second, and what leaves is a property of the way in. It was a service column first, which forced one answer on every edge of an `http` service. It reaches the engine **with the address**: `FGEX_PROXY_TARGETS` and `PUBLISH` for a TCP one (`
||[=]`), `FGEX_PROXY_UDP` and `ADD_UDP` for a relay — never as a variable of its own, since there is no longer a per-process answer. `Onward` in `proxy.rs` is the one spelling of the three values (a second enum, `UpstreamProtocol`, existed in `quic.rs` for an hour and is exactly the drift this file forbids); `TlsSetup.upstream` is built whenever the engine terminates anything and *used* per address. + +`same` is what every address did before there was a choice: a connection this engine terminated goes back out encrypted, one it never terminated was never encrypted to begin with, so a plain TCP service is untouched by this and always was. The other two say the service speaks HTTP/1.1 — in the clear or under its own TLS — whatever arrived at the front. + +On the TCP edge that is nothing but a config: `build_tls` hands back no upstream client configuration and the dial is an ordinary socket. **On the QUIC edge it is the one place in this engine where what leaves is not the version that arrived**, and it was the point of the whole exercise: most of the web is a service that speaks HTTP/1.1 and never will speak QUIC, so firegex terminates HTTP/3 in front of it. What makes that honest rather than a translation invented here is that the exchange is *already* rendered as HTTP/1.1 for the chain — `judge` sees those bytes, `firegex0` carries those bytes, and now the service receives those same bytes. Three things are load-bearing: + +- **`http1.rs` grew the sending mirror of `Incoming`** — `Outbound`, `OutboundBody`, `Answer` — and `h3.rs`'s `exchange`/`carry_request`/`carry_response` are generic over it. The alternative was a second copy of the orchestration for the HTTP/1.1 case, which is the drift the `Incoming` trait already exists to prevent one level down. `h1up.rs` implements them over hyper's HTTP/1.1 client (client+http1 features only): the request direction is free because the rendering already built it, and the *answer* is what needed a crate, since HTTP/1.1 is the one version this engine has never parsed — an HTTP/1.1 connection elsewhere is a byte pump. Writing a response parser beside hyper's is what "one engine per concern" forbids. +- **The request target has to be reduced to origin form.** HTTP/2 and HTTP/3 carry it split across `:scheme`, `:authority` and `:path`, which `http::Uri` holds as one absolute URI; written to a socket unchanged that is `GET https://host/path`, the *proxy* form, which an origin server may refuse and which every service reading the path back gets wrong. `h1up` reduces it and moves the authority to `host` — the same decision `render_request` already makes for the chain, so what the filters saw and what the service receives stay the same message. The tests caught this immediately; it is not theoretical. +- **One connection per exchange**, because a QUIC stream is one request and HTTP/1.1 has no multiplexing. Pooling would put two clients' requests on one socket and with it the thing `ChainSessions` exists to prevent, at a layer nobody would think to look at. Its body is a channel into hyper, and `finish()` **drops the sender** — waiting on the channel instead (`Sender::closed()`) waits for the service to answer a request this end has not finished sending, which is a deadlock the chunked test found at once. + +One combination is refused, and it is impossible rather than unimplemented: a QUIC edge carrying something that is **not HTTP/3**, whose opaque stream bytes have no HTTP/1.1 form. That is a fact about the **ALPN**, not about the protocol — `upstream_refusal()` in `modules/services/models.py` keys on `quic_alpn()` and so refuses exactly the instances that set `FGEX_PROXY_QUIC_ALPN` to something else, on `quic` and `http` services alike. Keyed on the protocol instead, as it was first written, it refused every QUIC service and allowed every HTTPS one, which is a rule with no reason behind it. The same function is asked from all three places that would otherwise drift: at creation (against the addresses in the form), when an address is added, and in `ProxyTransport.check`. + +**The decrypted traffic is written to an interface of its own, and those are reconstructed packets.** Terminating in-process means the plaintext is never a packet, so `proxysrc/src/capture.rs` frames it as the TCP stream it was and sends it to `firegex0` over `AF_PACKET`: one interface carrying every decrypted service's plaintext — TLS and QUIC both, see the QUIC section for what a stream costs there — and nothing else, which is what a capture tool wants pointed at it. The bytes are real — what the filters saw and what was forwarded — but the framing is invented, because the framing on the wire was encrypted. Sequence numbers start at zero, there are no retransmissions, and the segmentation is the engine's read sizes; a SYN and a FIN are emitted so Wireshark has a stream to follow rather than orphan segments. Say so wherever it is offered: a capture from here is not evidence of what was on the wire. + +Three things keep it from being able to hurt anything. `modules/services/mirror.py` creates the device in `FirewallManager.init()`, before any engine starts (a socket opened before the interface exists finds nothing), and `release_capture()` removes it from the process's `shutdown()` hook and nowhere else — deliberately not from `close()`, which `reset()` also calls, or the device would vanish and reappear underneath whatever was capturing from it. It is the **instance's** device, not each TLS service's: it used to be created and released per service, on the argument that an interface present while nothing is decrypting is one somebody points a capture at and watches stay empty — but a capture tool is attached once, for a round, and Zeek and Suricata exit rather than wait when the interface under them disappears, which restarting a single TLS service was enough to cause. Watching it stay empty is the better failure. `release_capture_if_idle()` is gone; do not add a per-service release back. Every failure is silent and local — no interface, no socket, a failed send — and the traffic goes on exactly as it would have. And the `AF_PACKET` socket needs `CAP_NET_RAW`, which `docker-entrypoint.sh` asks for **and starts without**: it tries the full capability set, falls back to the set without `cap_net_raw` while saying what that costs, and only then runs as root. A host that will not grant it loses the capture, not the firewall. + +Note for tests: a client now dials the service's own address, because there is no second port to be pointed at. The old note about the DNAT skipping loopback belonged to nginx's public-port redirect and went with it. + +### QUIC + +**QUIC is terminated, and unlike TLS there was never a second option.** TLS over TCP can be carried past unopened — the bytes are framed by a transport the kernel understands, so filtering the ciphertext is a thing an operator can choose. Past its Initial packet QUIC encrypts the frames, the stream boundaries and the packet number along with the payload, so a packet queued to NFQUEUE is a datagram of noise. `NfqueueTransport.check()` and `ExternalTransport.check()` refuse it saying that, rather than the sentence they say about TLS. + +It is UDP to the kernel, and that is the whole of what the backend had to learn: `L4.l4_of(quic) == udp`, so the addresses, the uniqueness key, the nft rules, `udp_relay_key`/`udp_relay_host`, `ProxyTransport.udp_ports` and `ADD_UDP` are the code that was already there. What changes is what binds the port — `FGEX_PROXY_QUIC=1` beside the certificate makes the engine build `quic::QuicManager` instead of `udp::UdpManager`, and `relays::Relays` is the two-armed enum the control channel and the startup list both go through. `Service.decrypts` (once `terminates_tls`) is what gates the certificate, on both protocols. + +**The ALPN is asked of the service, which is the one thing that could not be kept from the TLS path.** There the ClientHello is held at `LazyConfigAcceptor`; in QUIC it arrives inside an encrypted Initial whose processing *is* the handshake, and quinn offers nothing to hold it at. So the order is reversed: `dial_upstream` goes first with the candidate list, and `QuicSetup::answering_with` advertises to the client exactly the one protocol the service chose. The invariant survives — the client is never told something the service did not say — and what is lost is knowing in advance whether the client would take it, which shows up as a failed handshake with a log line naming the protocol. The list is `FGEX_PROXY_QUIC_ALPN` (default `h3`), read from the instance's environment like the filter deadline beside it; a per-service list would be a column and nothing has wanted one. + +**One stream is one connection to the chain** (`Carrier::open`), with its own `ChainSessions` and its own module globals, released when the stream ends. A filter's state follows a stream of bytes from start to end, which is what a QUIC stream is and what a QUIC connection is not; on HTTP/3 it also lands where an operator expects, one request to a stream. The raw path reuses `proxy.rs`'s pumps unchanged — `spawn_pumps`, `pump` and `join_pumps` are `pub(crate)` for exactly this, and `spawn_pumps` takes the `spoken` flag rather than making one, because the first-byte deadline is the *connection's* question and a flag per stream would never answer it for a connection that opened a hundred and spoke on none. + +**A refusal closes the connection, not the stream.** Resetting one stream would leave the client free to ask again on the next, which is not what a block means anywhere else; it is what keep-alive already does on HTTP/1.1. Both ends are closed with the same reason through `Carrier::refuse`. + +**HTTP/3 is terminated too, and rendered to the chain as the HTTP/1.1 it would have been** (`proxysrc/src/h3.rs`, on hyperium's `h3` — a binding, not a second implementation). This is the point of the whole exercise: HTTP/3 puts the method, path and headers in a QPACK-compressed HEADERS frame, so a filter shown the stream would be shown a compression format and every pattern written for the TCP service beside it would silently stop matching. Rendering it means `HttpRequest`, `HttpResponse` and a hyperscan pattern mean one thing on both. The same bargain the capture interface makes — the bytes are real, the framing is reconstructed — and the details are load-bearing: the request line says `HTTP/1.1`, `Host` comes from `:authority`, a declared length is passed through, and **the head is held back until it is known whether a body is coming**, so a bodiless request is never shown a `Transfer-Encoding: chunked` it never carried. A filter looking for that header is looking for smuggling, and one invented by the proxy in front of it is the worst possible answer; `tests/h3.rs` pins both halves of that. What goes on to the service is HTTP/3 again, re-encoded — the rendering is a view, never a translation the service sees. + +Two things about that hold, both of which were wrong first and are pinned now. **It is bounded** (`HEAD_HOLD`, 100 ms), because holding it indefinitely deadlocks every exchange the service is supposed to speak first in — a bidirectional RPC sends a head and waits to be answered, and a proxy that will not forward the head until a body arrives is waiting for something that is waiting for it. `head_framing` is what makes the hold cheap enough to bound: a declared `content-length` frames the body by itself and waits for nothing at all, a message with no body has already ended its stream, and what is left over is a sender holding the stream open — which is a message of undeclared length, which is what chunked means, so the timeout renders rather than refuses. **And a trailer section is always shown to the chain**, which is what decides the framing of a message carrying one and no body: in HTTP/1.1 trailers belong to a chunked message and nowhere else, so that message is rendered chunked with an empty body — the shape of a gRPC trailers-only answer, which used to reach the far side with nothing having read it. The one rendering that cannot carry trailers is a declared length, whose body ends where the header said; there `carried()` **drops** them with a log line, because forwarding a piece of a message no filter saw is the thing nothing else here does. `Incoming` exists so both directions decide all of this in one place rather than in two copies that can drift. + +**The plaintext is written to `firegex0`, one TCP stream per QUIC stream.** A QUIC stream is ordered, reliable and starts at zero, which is exactly what a `Tap` rebuilds; what it has not got is an identity a capture tool can use, because every stream of one connection shares the connection's four-tuple and writing them out as they are would interleave a hundred of them into one conversation whose bytes decode as nothing. So `Tap::open_stream` gives each a **synthetic client port** from a cycling counter — a counter and not a hash of the stream id, since ids restart at zero on every connection and two clients' first streams would land in one conversation. That is one more piece of invented framing on top of the invented framing this file already is, and it has to be said wherever the capture is offered, which `AddressList`'s panel now does: the port names a stream, not a socket. On HTTP/3 what is written is the **rendering** — the same HTTP/1.1 the filters were shown, since `judge` is the funnel every rendered byte already passes through — which is the only way `firegex0` can keep the promise it makes everywhere else, that what is on it is what the filters saw. It is not what left this process (that went out as HTTP/3 again), and the two cannot disagree because they are the same bytes. Written only once the chain has agreed, as the TCP pumps do it: a refused piece is one the traffic never got past this process, so the conversation ends where it did. + +**QUIC datagrams are carried, and one connection's are one flow.** `L4::QuicDatagram` is its own value rather than `Quic` with a flag, because the two answer the stream question differently and `is_stream` is what every model built on a stream turns on — a datagram gets `RawPacket` and nothing else, which is the same answer the plain UDP relay gives. `Carrier::carry_datagrams` opens the flow **on the first datagram**, not on the connection, so a connection that negotiates the extension and never uses it costs no filter state; both directions share the one id, and the sessions persist across datagrams exactly as the UDP relay's do — a pattern split over two datagrams sent a millisecond apart is otherwise a way straight through, and `tests/quic.rs` pins it. A refused datagram is **dropped, and the connection carries on**: that is a block meaning something different from what it means one function up, and the difference belongs to the traffic rather than to a preference — a stream is a conversation that can be ended, a datagram is complete by the time it is judged. `DATAGRAM_BUFFER` bounds what one connection may have waiting, because unbounded it is memory bought in this process by whoever sends fastest. + +**They are not carried on HTTP/3, and the handshake says so rather than the peer finding out.** An HTTP/3 datagram names the stream it belongs to, and the request streams this proxy opens towards the service are not the ones the client opened, so the name would point somewhere else on the far side; `QuicSetup::build` therefore gives the `h3` accept configuration a transport that does not advertise the extension, and the upstream edge — dialled before the service has said what it speaks — is decided by the candidate list instead. **WebTransport is not carried for a further reason and is a bigger question than its datagrams**: its streams are bound to a session by a varint every one of them carries, which breaks both of the rules the rest of this rests on — "one stream is one connection to the chain" would hand a filter a routing header it does not know to skip, and the HTTP/1.1 rendering has nothing to say for "this stream belongs to that request". It needs a session model, not a switch. + +Two things are refused rather than left unconfigured, and should be said as limits: **0-RTT** (`max_early_data_size = 0`), because early data is replayable and a refused request cannot be un-delivered; and a **Retry for every unvalidated client**, so that nothing is opened towards the service on the word of a source address nobody has checked. `over_limit_forwards` is honoured but is not free here: a QUIC connection past the limit is still terminated, decrypted and re-encrypted, because there is no such thing as forwarding a QUIC connection unopened. + +### HTTP/2, and one rendering for every version + +**`src/http1.rs` is the rendering, and `h2.rs`/`h3.rs` are what is left of each protocol once it is taken out.** The HTTP/1.1 view a filter is shown — `Framing`, `HEAD_HOLD`, `head_framing`, `carried`, `render_request`/`render_response`, `push_body`/`close_body`, `judge` — lived inside `h3.rs` and had nothing QUIC about it. Copying it into `h2.rs` would have been two renderings that can drift, which is the argument the `Incoming` trait already makes one level down, and the day they drift the symptom is a pattern that matches over HTTP/3 and not over HTTP/2, silently. So it lives in one place and both protocols are adapters to somebody else's parser: hyperium's `h3`, and `h2` for HTTP/2 — a binding, never a second implementation, for the reason stated under "one engine per concern". + +**An HTTP/2 client can be put in front of a service that speaks HTTP/1.1, and for a while it could not.** `h2.rs` takes its upstream as an `Outbound` — `H2Upstream` where the service speaks HTTP/2, `h1up::H1Upstream` where the address said `upstream = plaintext` — so the rendering the chain is shown is also what the service receives, exactly as on the QUIC edge. Before that, `h2::carry` opened an HTTP/2 client handshake and nothing else, which is why the ALPN could only ever be *mirrored*: with a cleartext service there was no upstream handshake to ask, so the client was told nothing and fell back to HTTP/1.1. The asymmetry is what gave it away — the same cleartext service was reachable over HTTP/3 and not over HTTP/2, one rendering underneath, opposite answers. `proxy.rs` now answers `h2` itself when the address says the service is cleartext, **and only while the chain will render it**: a bypassed chain is a byte pump, and a byte pump between an HTTP/2 client and an HTTP/1.1 service carries frames nothing at the far end can read. + +Two things the trait had to grow for it, both because HTTP/2 states on the head what HTTP/1.1 frames for itself. `Outbound::open` takes **`ends_it`**: a head that is the whole message has to go out as one, or a trailers-only gRPC answer becomes an answer with a body, which is the thing `head_is_the_message` exists to prevent — HTTP/1.1 just closes the body early and HTTP/3 ignores it. And `Incoming` grew **`release`** and **`ended`**: flow control is HTTP/2's alone, and releasing the window only *after* a piece has been forwarded is what turns the far end's window into backpressure on the near one, so it could not stay inside a loop that no longer knows which protocol it is reading. + +**Terminating HTTP/2 was never forced the way terminating QUIC is.** The bytes are framed by a transport the kernel understands, so firegex could have gone on forwarding them. What forces it is the promise: a filter written once has to work everywhere, and there is no way to keep that over HPACK without decompressing it, which is what parsing HTTP/2 means. + +Three things differ from h3 and each is a way to get this wrong: + +- **Flow control is ours to run.** `h2` hands received bytes over and waits for them to be released. Releasing on read means no backpressure and this process buying memory for whichever side is faster; never releasing stalls the stream after the first window. So capacity is released **after the piece has been forwarded**, and outbound writes reserve capacity before writing rather than queueing in the connection. +- **The end of a message rides on its last frame.** A head that *is* the whole message has to be sent with `end_of_stream` on the head; sending it open and closing with an empty DATA frame turns a **trailers-only** answer into one with a body — which is exactly a gRPC status-only reply, and gRPC clients refuse the mangled version. `head_is_the_message` is the one place that is decided, and it asks the sender (`RecvStream::is_end_stream`) rather than inferring, which is also what keeps a `content-length: 0` request headers-only through the proxy. +- **One stream is one connection to the chain**, as on h3 — and unlike HTTP/1.1, where a keep-alive connection carries many requests through one set of filter state. HTTP/2 interleaves its streams, and sharing state between two of them is "one client's bytes deciding another client's verdict", which is what `ChainSessions` exists to prevent and also a way to smuggle a pattern past a filter by splitting it across two streams. Say this in the docs: it is a real difference an operator writing a stateful filter can be caught by. + +**Server push is refused in the handshake, not by dropping what arrives** (`enable_push(false)` towards the service) — the same courtesy the QUIC edge pays about datagrams, and for the same reason: a peer that learns from SETTINGS is better off than one that watches its pushes vanish. **Extended CONNECT is not advertised**, and a plain `CONNECT` stream is reset with a log line rather than rendered: a tunnel is opaque bytes with no HTTP/1.1 message in it, and inventing one would be the fabrication everything here avoids. Both are the WebTransport answer from the QUIC section, arriving at the same place. + +**A refusal closes the connection, not the stream**, with `Reason::CANCEL` — deliberately not `REFUSED_STREAM`, which tells a client the request was never acted on and may be retried, and a block a client retries past is not a block. The GOAWAY is polled onto the wire before the socket goes (`GOAWAY_GRACE`), for the same reason the byte pumps shut down rather than abort. + +**A bypassed chain is not terminated.** Rendering for a chain with nothing to say is work paid for no answer, so a service with no filters, one whose filters have all been disabled, and a connection admitted past the limit with `over_limit_forwards` all take the byte pump. The cost is that a connection admitted while the chain was empty stays a byte pump for its whole life even if a filter is pushed a moment later — the same trade `over_limit_forwards` itself makes. Note what this does to tests: `tests/h2.rs` spells its no-op chain `LIVE` rather than `""`, because an empty one passed every assertion while exercising none of this, which is the class of false green that found itself immediately (a trailer section that should have been dropped was carried, because nothing had been terminated). + +**`h2c` — HTTP/2 in the clear — is recognised from the connection preface, not from a setting.** Most gRPC is deployed without TLS behind a load balancer, so leaving it out would have made "gRPC is filtered" half true. A prior-knowledge client opens with a fixed 24 bytes that no other protocol begins with, and `sniff()` in `proxy.rs` **peeks** rather than reads — nothing is consumed, so a connection that turns out to be something else is handed to the byte pumps exactly as it arrived and there is no buffer to replay. Two properties are load-bearing: it **races the service**, because a protocol where the server speaks first (SMTP, SSH, most game protocols) would otherwise be held waiting for a client that is correctly waiting for a banner, and firegex would look like it broke them at random; and it is **bounded** (`SNIFF_WAIT`), so two silent peers cannot hold the two descriptors open. The same peek answers the TLS question for an `http` service, which is why there is one `Opening` and not two sniffs. + +What is *not* carried and is a limit rather than an unconfigured setting: the **HTTP/1.1 `Upgrade: h2c`** handshake. The upgrade request itself is filtered as the HTTP/1.1 request it is, but if the service answers `101` everything after it is HTTP/2 nobody rendered. RFC 9113 deprecated the mechanism and nginx dropped it in 1.25.1, so in practice no service accepts it; say so rather than implying it is handled. + +### A service can speak every version of HTTP at once (`http`) + +`L4.HTTP` is the odd value in that enum and deliberately so: the other four say what is on the wire, and this one says what the *service* is, letting each address say how it is reached — `:80` in the clear, `:443` under TLS, `:443/udp` under QUIC. One service, one chain, one certificate. + +It exists because the alternative was measured against the same rule that put several addresses under one service: HTTP/1.1 and HTTP/2 live on `tls` (TCP) and HTTP/3 on `quic` (UDP), so covering the three took two or three services with the same filter chain copied between them by hand — and a chain kept in step by hand is a chain that stops protecting one of them silently. + +**Nothing about the kernel became ambiguous**, because nothing here is asked of the kernel. Each address still carries its own `tcp` or `udp` in `service_addresses.proto`, which is what the rules match and what `(ip, port, proto)` is keyed on; `nftables.py` needed no change at all, because it was already asking the address. What changed is that the questions which used to be asked of the *service* — "does this need a UDP relay", "does the TCP listener need to accept IPv6" — now go through `Service.carries(l4)` and `Service.udp_addresses`, since `Service.l4` no longer has one answer. `L4.l4_of("http")` returns `tcp`, which is the right default for an address that did not say and never a claim about all of them. + +**What the connection turns out to be is decided per connection, from what the client sent.** `TlsSetup.optional` is the whole of the mechanism: the TCP listener terminates TLS only for the connections that start a handshake, so the cleartext port and the encrypted one are the same listener, the same chain and the same certificate. The upstream leg mirrors it — a connection that arrived in the clear is forwarded in the clear, one that arrived under TLS is re-encrypted — because the service is being dialled on the port the client chose, and that port's edge is the one the client has just demonstrated. This is the ALPN mirroring rule applied one layer down: firegex carries what the two ends are doing rather than deciding it for them. + +Two consequences worth keeping: + +- **TLS on TCP and QUIC on UDP now live in one engine process**, which they could not before: `main.rs` built one or the other on the reasoning that a service speaks one or the other. `FGEX_PROXY_TLS` says so explicitly now, and the old implicit rule (a certificate means terminate, unless this is a QUIC service) still applies to everything that does not. +- **A certificate is required**, even for the cleartext addresses beside it. A service that only answers in the clear is a `tcp` one — where HTTP/1.1 is filtered as it always was and HTTP/2 is now recognised from its preface — so the TLS and HTTP/3 edges are the whole of what `http` adds, and neither exists without one. + +`http` is **proxy-layer only**, refused at creation by `NfqueueTransport.check` and `ExternalTransport.check` with its own sentence rather than the TLS one or the QUIC one: the reason is neither, it is that an `http` service is reached over TLS *and* QUIC *and* in the clear and that layer can read none of it — not even the cleartext edge, because HTTP/2 in the clear is HPACK and rendering it means terminating the connection. + +### Authentication (`backend/app.py`) +Every module router is mounted onto `api = APIRouter(prefix="/api", dependencies=[Depends(is_loggined)])`, so a new file in `routers/` is authenticated by construction. The whole auth surface is four places: that dependency, the Socket.IO `connect` handler (reads the token out of `auth`), and the three routes declared on `app` directly rather than on `api` — `/api/status`, `/api/login`, `/api/set-password`. Adding an unauthenticated route means declaring it on `app`, which should stay a deliberate, rare act. + +`--unsafe-disable-auth` (env `UNSAFE_DISABLE_AUTH`) hands access control to a reverse proxy: `check_login` returns `True`, the socket.io handler skips its check, and `APP_STATUS()` reports `run` even with no password stored. The password endpoints (`/api/login`, `/api/set-password`, `/api/change-password`) answer `403` in this mode on purpose — otherwise an anonymous caller could plant a credential that keeps working once authentication is turned back on. `run.py config --password` still works, writing the hash straight into the DB from the host, and an explicit `-P` at first start is kept so that re-enabling authentication lands on a known password instead of the open initial-setup state. + +**The flag is state, not a constant.** It lives in `keys_values["auth_disabled"]` and every check reads it (`auth_disabled()`); the environment is only the seed, written once per boot in `startup_main`. It used to be a module-level constant read at import, which is why `run.py config --password` could print "it will take effect immediately" at a process that had already decided every caller was an administrator — the hash landed in the database and nothing ever asked for it. `run.py`'s `put_runtime_settings()`/`get_runtime_setting()` are the one way in and out of a running instance's `keys_values` (Docker exec or the standalone rootfs, same function), used by both `--password` and `--[no-]unsafe-disable-auth`, and `config --show` says when the running instance disagrees with the persisted config. **`config --password` re-enables authentication when it was off**, live and persisted: setting a password is asking for one to be asked for, and a stored-but-unchecked password is worse than none because it reads like one. It stands down when the same command line said otherwise — an explicit `--unsafe-disable-auth`, or `--keep-auth-disabled` for an instance deliberately held open behind a proxy. Note the password block runs *after* the auth block in `handle_config_command`, which is why the auth block's "there is no password set" warning checks whether one is about to be set. + +`POST /api/auth-mode` turns it off and back on at runtime. **The two directions are not symmetrical, and that asymmetry is the design:** off is an administrator's deployment choice, but on, asked while it is off, would let any passer-by set a password and keep the real operator out for good — a lasting foothold, unlike the vandalism the mode already permits. So re-enabling takes a token this instance signed *before* it was turned off. Nothing new is signed while it is off (login and set-password both 403), so holding one is exactly that proof, and the operator who turned it off still has theirs. `token_is_valid()` is split out of `check_login` for this single caller: it needs the question without the "authentication is off, so yes" that `check_login` answers first. Either direction lasts only as long as the process, because the environment is what firegex comes up with — `run.py config` writes both the live database and `.firegex-conf.json`. + +Frontend note: a 401 used to call `window.location.reload()`. A token left over from a database that has since been recreated (a new `secret`) 401s on the first query, the page reloads itself, and what comes back is whatever `APP_STATUS()` calls for — "choose a password", if none is set. That is the "sometimes a password screen appears on refresh" report. The token is now dropped and the app re-renders from the next `/api/status`, and no `Authorization: Bearer null` is sent when there is no token at all. + +Independently of that, `IPFilterMiddleware` restricts access by CIDR (`--allowed-ips`, optionally reading the client IP from `--proxy-ip-header`). It **fails closed**: a missing, unparseable, or non-matching IP is denied. Note the header variant trusts a client-supplied value, so it only holds if the proxy overwrites it. + +### SQLite wrapper (`backend/utils/sqlite.py`) +Each DB file's schema is versioned by hashing the Python schema dict; on mismatch the **entire file is deleted and recreated** (not migrated) — any schema dict change wipes that DB file on next startup, including `db/firegex.db`'s `keys_values` table (password/JWT secret). `dump()`/`load()` back the export/import backup feature; `dump()` deliberately excludes `password`, `secret` **and `auth_disabled`** from exported `keys_values` rows, so `/api/import` explicitly re-preserves all three around the import. The third one is the newest and the easiest to lose: it is not a credential, it is *whether credentials are asked for*, which belongs to the deployment and not to the configuration being restored — a backup able to carry it is a backup able to turn a firewall's authentication off. `auth_disabled` is also put back **after** `startup_main()`, not before like the other two, because `startup_main` seeds it from the environment: right when a process is starting, wrong during an import where nothing restarted and the environment did not change. `tests/integration/test_backup.py` turns authentication off *around* the import on purpose — asserting it is still on afterwards passes either way on a container whose environment agrees, and the bug only shows where the running state and the boot-time environment differ. + +### Real-time frontend updates +One Socket.IO event, `"update"`, whose payload is a list of strings treated as a react-query `queryKey` prefix (e.g. `["services"]`, `["firewall"]`). `App.tsx` does `queryClient.invalidateQueries({ queryKey: data })` — prefix matching invalidates any query whose key starts with that array. Convention: each module's mutating endpoints emit the same tag string that module's frontend queries use as the first `queryKey` element. `staleTime: Infinity` is set globally, so this event is the **only** automatic refresh mechanism — a missing/mismatched tag means silent staleness for other clients/tabs. + +### Frontend structure +`pages//` (routes) + `components//` (ServiceRow, AddEditService, `utils.ts` with query key + API wrapper) per module, mirroring the backend split. Detail drill-down lives at `pages//ServiceDetails.tsx` under a nested `:srv` route (see `App.tsx`'s `PageRouting`). Theme tokens are CSS variables in `src/index.css`; Mantine `Card` defaults are set globally in `index.tsx`. Prefer Mantine `Group`/`Stack` with `gap` over the legacy `Box className="center-flex"` utility classes still present in older parts of the codebase. + +### `fgex-lib/` +Separate, independently-published PyPI package (`pip install fgex`/`firegex`) — the library users write Python filters against (`@pyfilter`, `RawPacket`/`HttpRequest`/etc.), plus the `fgex` CLI, a local `proxysim` for testing filters, and `firegex.regex` for testing *pattern* rulesets, without a real firegex instance. + +The library module is **`firegex.pyfilters`**, and the CLI command is `fgex pyfilters`. Both were `nfproxy` when that was one of several modules. A compatibility alias carried the old spellings through 4.x and was **removed in 5.0.0**, which is a breaking change and the reason the version is a major one: `from firegex.nfproxy import pyfilter` is the first line of every filter written before it, including the ones already sitting in a running firegex's database. + +Two things about that removal are worth keeping. The directory has to go, not just its contents — a `firegex/nfproxy/` left behind with only a `__pycache__` in it is a *namespace package*, so the import succeeds and hands back something empty, and `from firegex.nfproxy import pyfilter` then fails with an `ImportError` about a name rather than a `ModuleNotFoundError` about a module. And the failure is made to explain itself where the operator meets it: `pyworker.py --check` recognises this one import and says what to write instead, because the alternative is a filter that stops loading at the first start after an upgrade, which at a competition is the start of a round. + +Do not reintroduce the alias as a file of re-exports if it ever comes back. `type_annotations_associations` maps model **classes** to protocols, and both the library and the datapath decide when to call a filter by looking an annotation up in that table, so a shim building second copies of `HttpRequest` would leave a filter annotating a class the table has never heard of — importing cleanly and then never being called. The old one aliased submodules into `sys.modules` for exactly that reason: one class per model, two spellings of one package. + +**Deploying is `run.py`, and only `run.py`.** The package used to carry a `fgex deploy` passthrough that exec'd a copy of it; it is gone, along with the build-time copy `setup.py` made. The package is for writing and testing filters — `firegex.pyfilters`, `firegex.regex`, the two simulators — and starting a firegex is a separate job done from a clone. Do not reintroduce a deployment command here: the reason the old one was a verbatim passthrough was that any second description of `run.py`'s argument surface drifts, and not shipping one at all is the same argument taken one step further. + +**The HTTP parser ships as source, and nobody upstream will bump it.** `fgex-lib/llhttp/` carries llhttp itself (`lib/`, verbatim from a `release/*` tag) and our binding beside it, because `pyllhttp` is archived, every binding with prebuilt wheels fails to load in the subinterpreters `cpproxy` runs filters in, and a ctypes binding against the system `libllhttp` would make `pip install firegex` assume a package installed beforehand. `PROVENANCE.md` there is what makes that maintainable: which tag, which checksums, what is ours and what is theirs, and how to move it. Each release changes what the parser *accepts* — going from 9.2.1 to 9.4.3, an empty `Transfer-Encoding` stopped being accepted and tabs around `Content-Length` started being — so `tests/unit/test_http_parsing.py` pins both through the public models: a change there is a decision about what filters are shown, never a test to fix. A parser more lenient than the service behind it is how a filter gets bypassed while still looking switched on. **The binding names itself and nothing else**: `pyllhttp` is where it was forked from, not something it uses, and the error path that imported it at runtime raised `SystemError: ... returned NULL without setting an exception` on every malformed request anywhere that archived package was not installed — which is everywhere except the machines it was written on. `wheel_smoke.py` is the check that could have caught it, and the reason it is a file rather than a `-c` string: `cibuildwheel` runs it against each wheel in an environment with nothing else in it, parsing something malformed as well as something valid. + +`firegex/regex/` binds `libhs` through ctypes rather than depending on a wheel: no build step, nothing to go stale, and by construction the same library the product loads. It is a third binding to one engine, not a third engine — the distinction that matters is that there is still only one thing deciding what matches. Where libhs is absent the tools raise `HyperscanMissing` and stop; adding an `re` fallback would reintroduce exactly the lie the in-app tester was built to avoid. `tests/unit/test_regex_library.py` pins the agreement: validity in the mode a pattern will run in, direction, the shared file format, and that a ruleset asking to `rewrite` is refused rather than quietly treated as a block. Not part of the backend/frontend build. `fgex-lib/README.md` is a short index (PyPI-facing) linking out to `docs/*.md`; it isn't rendered anywhere itself. + +### Module documentation (`docs/*.md`) +Each module's user-facing docs (`docs/services.md`, `docs/pyfilter.md`, `docs/firewall.md`) are single markdown sources, each imported directly with Vite's `?raw` suffix by a matching `Docs.tsx` component (e.g. `components/Services/ServicesDocs.tsx`) and rendered through the shared `components/MarkdownDocs.tsx` (React Markdown + `remark-gfm`, code fences routed through `CodeHighlight`, relative `.md` cross-links rewritten to the equivalent GitHub blob URL since they'd otherwise resolve against the app's own origin). `DocsButton.tsx`'s `DocType` enum wires each module to its component. Every language a fence uses has to be in `index.tsx`'s `createHighlighter({ langs })` — Shiki throws for one it was not given, and that throw is uncaught, so it takes the whole page down rather than the block it could not colour. Editing a module's docs means editing its one `docs/*.md` file — never the `*Docs.tsx` component directly. Reading these files across the `frontend/`↔repo-root boundary requires `vite.config.ts`'s `server.fs.allow: ['..']` (dev) and the `Dockerfile` frontend build stage's `COPY ./docs/*.md /docs/` (prod/standalone rootfs) to stay in sync with any path changes. diff --git a/CLAUDE.md b/CLAUDE.md index f5d92dd..94d7f62 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,302 +1,12 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +**Read [AGENTS.md](AGENTS.md).** It holds everything: what this is, the commands, the +architecture, and the decisions worth not undoing. -## What this is +Nothing is repeated here on purpose. This file existed as a full copy of that one for a +while, and by the time anybody noticed the two had drifted — the copy still described an +interface rule that had been rewritten and a protocol picker that had been redesigned — +with nothing to say which was current. One document, one place to edit it. -Firegex is a firewall built for CTF Attack-Defense competitions: it sits in front of a vulnerable service and blocks/mangles malicious traffic before it reaches it. Behind a single FastAPI backend and React frontend it ships **services** (a network layer plus an ordered chain of filters) and plain nftables rules (`firewall`). That is the whole product surface: `nfregex`, `nfproxy`, `tls` and `porthijack` were folded into `services` as choices rather than modules. - -## Commands - -### Running the whole app (Docker or standalone) -```bash -python3 run.py # build+start (Docker if available, else standalone chroot on Linux) -python3 run.py --standalone # force standalone mode (no Docker) -python3 run.py stop [--clear] # stop (optionally wipe the DB volume/rootfs) -python3 run.py restart # recreate the container WITHOUT rebuilding (stale image if source changed) -python3 run.py status -python3 run.py config --password [newpass] # change the running instance's password; omit the value to be prompted interactively -python3 run.py config --show # print the persisted settings (port, host, socket dir, auth) -python3 run.py start --unsafe-disable-auth # no authentication at all — see "Authentication" below -``` -`run.py` has zero external dependencies (stdlib only) by design. `restart` runs `docker compose up -d --force-recreate`, so it re-reads the generated compose file and re-applies env vars (`ALLOWED_IPS`, `PROXY_IP_HEADER`, `UNSAFE_DISABLE_AUTH`) — a plain `docker compose restart` would keep the old container environment and silently ignore those. It still does **not** rebuild the image; after a source change use `stop` then the bare start command to force `docker compose up -d --build`. - -Most `start`/`restart` flags are persisted to `.firegex-conf.json` and become the default for later runs, so a bare `python3 run.py start` inherits whatever was last passed. - -### Frontend (`frontend/`) -```bash -bun install # bun.lock is canonical -bun run dev # vite dev server -bun run build # tsc && vite build -npx tsc --noEmit -p . # typecheck only (no separate lint script is configured) -``` - -### Backend (`backend/`) -Needs Linux with `NET_ADMIN` (nftables/nfqueue) and root, so it's normally only run via `run.py` (Docker or standalone). Direct iteration on a Linux box/VM: -```bash -cd backend && DEBUG=1 python3 app.py DEBUG -``` - -### Tests (`tests/`) -**One suite, one runner: `pytest`.** Everything drives a **live, already-running** firegex (`http://127.0.0.1:4444` by default), root/Linux, except `unit/` which needs nothing running at all. - -```bash -cd tests -./run_tests.sh # installs deps, waits for the instance, runs pytest -./run_tests.sh --set-pass # against one still in its initial-setup state -pytest unit # needs no instance -pytest -k udp # one subject -pytest --layer proxy --no-ipv6 -x # narrow, stop at the first failure -pytest standalone # restarts the instance; opt-in, see below -``` - -| Directory | Needs an instance | What it holds | -|---|---|---| -| `unit/` | no | `firegex.regex` against libhs, the `firegex.pyfilters` models, the address→nftables translation | -| `integration/` | yes | the product, parametrised over the combinations | -| `standalone/` | yes, and restarts it | settings read once at process startup | -| `helpers/` | — | the API client, stand-in services, certificates, the traffic `Channel` | -| `bench/` | yes | measurement, not testing; `pytest` does not collect it | - -**The combinations are parameters, not a shell loop.** `integration/conftest.py` holds `Layer(transport, ipv6, tls)` and the fixtures that fan out over it — `inspecting_layer`, `filtering_layer`, `proxy_layer`, `tls_layer`, `external_layer`, `any_layer`. Running both network layers with the same filters is the point: a filter that only works on one of them is exactly the failure the unified model exists to prevent. This was a shell script invoking one 1100-line program eight times, which meant the first failure ended the run and the other seven combinations were never reached, and a host without IPv6 on loopback failed rather than skipped. - -Three things that have to stay true of this layout: - -- **`helpers/`, never `utils/`.** The backend's own package is called `utils`, and this directory's used to be too — so whichever imported first won, and `unit/test_nftables_addresses.py` (which imports the backend) and everything importing `helpers.firegexapi` could not be collected in one pytest process. That name collision is why the suite *could not* be unified before. -- **The interception canary.** It is perfectly possible to reach the API of an instance whose datapath protects a different machine — a container without host networking, a VM with the port forwarded. Every call returns `ok` and not one packet goes near any of it, which without a check reads as dozens of "the filter did not block" failures rather than as a suite pointed somewhere it cannot reach. `datapath_in_our_path` asks once, plainly, and everything needing interception is skipped with the reason. -- **Skip with a reason rather than fail.** A laptop with no IPv6 loopback, a non-root run, an instance elsewhere — none of those are firegex bugs, and `-ra` is in `addopts` so the summary says which half of the suite the machine declined. - -`Channel` (`helpers/traffic.py`) is the one place that knows whether TLS is in the way. The old suite branched on it at every traffic check, about twenty times. - -Run the suite **twice in a row** when changing anything about addresses or TLS: state left behind by the first pass only shows up on the second. - -`standalone/` is opt-in because it drives `run.py stop`/`start` itself — the IP allowlist and the boot-time auth mode are read once at startup, so there is no way to test them against a running instance. It restores an unrestricted one at the end, which every other module depends on. - -The Rust datapath has its own suite, which needs no running instance but does need the production toolchain (vectorscan and Python 3.14 — the filter library imports `compression.zstd`): -```bash -docker build --target compiler -t firegex-compiler . # then run cargo test inside it -cd backend/proxysrc && cargo test # on a host with libhs + fgex installed -cargo test --release --test throughput -- --ignored --nocapture # the matcher benchmark -``` - -The suite runs against an instance with authentication turned off as well: `login()` reads a 403 there as "there is nothing to log in to" rather than as a failure, since every request is already allowed. `integration/test_auth.py` is the exception and skips itself — it is about passwords, and those endpoints answer 403 in that mode by design. Note `BearerSession.post` copies the header dict rather than writing into it: the form `Content-Type` belongs to the login request alone, and it used to stick, which went unnoticed only because a successful login replaces the whole dict a moment later. - -CI (`.github/workflows/release.yml`) runs **only on `release: published`**, not on pushes or pull requests — nothing validates a PR automatically. Its `run_tests` job (`ubuntu-latest` + `ubuntu-24.04-arm`) does `python3 run.py start -P testpassword` then `cd tests && ./run_tests.sh`, and gates every publish job below it. The tests' dependencies are pure python; `iperf3`, matplotlib and numpy belong to `bench/requirements.txt` and CI installs none of them. - -## Architecture - -### Three tiers -React (Vite + Mantine + react-router + TanStack Query + socket.io-client) → FastAPI backend → per-transport datapath binaries that do the actual interception. The backend mostly orchestrates: SQLite-backed config, spawning/talking to those processes, and deploying the built React app. - -### The layers are compared in the interface, not only in the docs -`components/Services/LayerChoice.tsx` puts the trade in front of the operator where the choice is made: a table of what each layer costs, plus the paragraph for the one selected. `transportSummary()` carries the same sentence onto every service row and detail page. Keep them in step with `docs/services.md` — the point is that an operator never has to read documentation to find out that one layer reassembles in the kernel and the other has a kernel-guaranteed fail-open. There used to be a `losesClientAddress()` beside them, marking the one combination that gave the client's address up; source transparency is unconditional on every layer now, including UDP, so it answered `false` for every input and the badge it gated could not render. Do not reintroduce it without a combination that actually loses something. - -### The two layers (`backend/modules/services/`) -The central idea, and the thing to preserve when changing anything here: **how traffic is intercepted and what is done with it are independent choices.** - -- **Network layer** (`transports.py`): `nfqueue` — packets queued to userspace, verdict handed back, nothing terminated, kernel fail-open via `NFQA_CFG_F_FAIL_OPEN` plus `bypass` on the nft rule. `proxy` — the connection is terminated and reopened, which buys exact rewriting, kernel reassembly, backpressure and an ordered chain, and costs the kernel backstop (rebuilt by hand in `proxysrc/src/filter.rs`). - - **The proxy is not the slow one, and the docs used to say it was.** Measured: 2.2× NFQUEUE's bulk throughput at one thread and 4.7× at eight, because NFQUEUE pays a userspace round trip *per packet* while the proxy pays once per connection and then the kernel moves the bytes. `NTHREADS` reaches the Rust engine too since 5.0.0 — `main.rs` builds its tokio runtime with that many workers rather than `#[tokio::main]` taking every core, so `run.py --threads` means one thing on both layers instead of one thing on one and nothing on the other. On short connections the two are indistinguishable — that claim cost three contradictory measurements before `tests/bench/connbench.py` started interleaving the layers within each pass instead of measuring one and then the other, which was handing the machine's drift to whichever went second. The trade between the layers is fail-open and transparency, not speed; keep `docs/services.md`, `LayerChoice.tsx` and `transportSummary()` saying that. `external` — a stateless destination rewrite that puts a proxy the operator wrote in the path (this is what `porthijack` used to be); no firegex process is involved, so `check()` refuses any filter, since a filter attached there would be one that never runs. -- **Filter layer**: `regex` (patterns compiled into one hyperscan database) or `pyfilter` (the user's Python). A service holds an ordered list of them. - -`Transport.check()` is where an impossible combination is refused **before** anything starts: more than `MAX_CHAIN_POSITIONS` filters on `nfqueue`, any filter at all on `external`, and the `tls` protocol anywhere but the proxy layer. It is asked twice: at creation with an empty chain, for what the service alone makes impossible, and again whenever a chain is pushed. UDP was never removed from the datapath: `cppregex` runs in block mode for it and `cpproxy` handles it, and the only thing that had gone missing was the choice in the UI. - -**UDP on the proxy layer (`proxysrc/src/udp.rs`).** The TCP path recovers where a connection was headed with `SO_ORIGINAL_DST`, which the kernel implements **for TCP and SCTP only** (`ENOPROTOOPT` otherwise). So UDP is relayed with **one socket per protected address**, each with its upstream fixed — nothing has to be recovered because nothing was lost. The engine prints one `UDP ` line per relay after `PORT`, and `_add_proxy` points each address's rule at its own port. - -**One spelling of an upstream, and one owner of the map.** `udp_relay_key()` in `nftables.py` is the only place the `host:port` / `[host]:port` token is built, and `udp_relay_host()` the only place an interface name is resolved to something a relay can bind — IPv4 first and link-local last, because an interface almost always carries an `fe80::` address it was never configured with and binding there puts the service where nobody dials. Both were written out by hand in four and three places respectively, and the engine's map, the rule that points at a relay and the manager adding an address to a running service all have to agree on the token exactly; when they stop, the symptom is a service whose UDP silently goes nowhere. For the same reason `ProxyTransport.udp_ports` is the *only* map of relays: `start()` returns that dict as `_steer["udp_ports"]`, so it is the same object the rules read, and `ServiceManager.address_added` asks `add_udp_target()` for a relay rather than keeping a second copy of the answer beside it. - -Client address transparency is preserved via `transparent::connect_as_udp` using `IP_TRANSPARENT` with mark `0x1339` and policy routing `table 1339`. New UDP addresses can be added to running services dynamically via `ADD_UDP` over the control channel without restarting the proxy. - -What UDP keeps: per-flow filter state keyed by client address and released after `IDLE` — UDP has no close to observe, so a timeout is the only thing that ends a flow — and both directions inspected. `Verdict::Reject` drops the datagram rather than closing anything, because there is nothing to close. Replies leave through the **listener** socket so conntrack rewrites them to appear from the address the client dialled; sending from the upstream socket would reach a client not expecting that source. The relay's own dial carries `SELF_MARK`, or the redirect rule would catch it and loop. - -`ConnectionMeta.tcp` carries the L4 protocol to the filter, so the library's stream and HTTP models — which only apply to TCP — key off what the engine says rather than a hardcoded `True` in the worker. - -### A service is a list of addresses -`service_addresses` holds `(ip, port, proto)` rows; the service itself holds the name, transport, protocol and TLS material. One daemon routinely answers on more than one address — a v4 and a v6 one, a public and an internal one — and all of them deserve the same chain. Two services with hand-copied chains is how one of them silently stops being protected. - -Consequences worth keeping: -- **Adding an address to a running service drops nothing.** `ServiceManager.address_added` installs only that address's rules against the datapath already running (`self._steer` holds what to point at). On TCP and TLS the proxy listener is dual-stack from the start, and on UDP new relays are bound dynamically on the fly, so adding any address (IPv4, IPv6, UDP, TCP, TLS) never drops connections or restarts the service. -- **`address_removed` drops the address from `self.srv.addresses` itself** rather than re-reading the table, because the caller takes the rules back *before* deleting the row. Re-reading would put it straight back, and the next restart would re-protect an address that no longer exists — with TLS, by claiming a port for a stream nobody owns, which then refuses the next service that wants it. -- **`proto` is denormalised onto each address** and rewritten when the service's changes. That is what makes `(ip, port, proto)` a usable uniqueness key, so a TCP service and a UDP one can share an address exactly as the kernel allows. -- **The `external` proxy endpoint belongs to the address, not the service**, and a partial unique index forbids two addresses sharing one: the return rule recognises the operator's proxy by address and port to put the original port back, so two addresses behind one endpoint could not be told apart on the way out. -- **The proxy listener is dual-stack whenever the host supports IPv6** (`[::]:0`), and `transparent::unmap` turns the v4-mapped peer and local addresses back into IPv4 before anything else in the engine touches them — conntrack, the source-preserving dial and the metadata a filter reads all want the address the packet actually had. -- **Addresses accept network interface names as well as IPs.** An entry in `service_addresses` can be an IP/CIDR or a network interface name (1-15 chars, e.g. `eth0`, `wg0`, `tun0`, `lo`), checked via `utils.is_interface_name` and marked by `Address.is_interface`. The interface is not a second spelling of an address, and the interface is what the operator is choosing when the address is somebody else's to hand out. The frontend says which of the two it understood *while it is being typed* — `addressKind` in `components/InterfaceInput.tsx` puts it on a badge in the field, because `eth-0` is a perfectly good interface name and a perfectly wrong address, and the two are matched by completely different rules. That rule is written once on the frontend too, as `isInterfaceName`/`isIpAddress` in `js/utils.tsx`, reading the same charset the backend does — it used to be pasted into `AddEditService.tsx` and `AddressList.tsx`, two copies of a rule the picker beside them did not consult at all, which is how the hand-off layer came to offer interface names in the list and refuse them on submit. `InterfaceInput` lists interfaces and addresses as two groups, each interface carrying what it holds right now (one question decides between them: "and what is on it"), and stays a combobox rather than a select because an address a service will answer on need not exist on this host yet. **Link-local is not offered** — every interface carries an `fe80::` it was never configured with, which doubled the list with addresses that would put a service where no client looks for it, the same reason `udp_relay_host()` takes them last; one typed by hand is still accepted, and an interface holding nothing else says "link-local only" rather than listing it. A row is **the badge as a column, and everything else in the one beside it** — value, then the note, wrapping inside that column and never under the badge. Neither of those two is given a fixed share, because which one is long changes with the group (the addresses a link carries, or the address itself): fixing it took the whole width from a full-length IPv6 and left the other wrapping a character per line, a row a screen tall beside an address cut off mid-digit; and three items in one wrapping row put that same address at the container's left edge, under the badge, three lines tall. The list's own width is **computed from the longest value in it**, between a floor and a ceiling, and three simpler rules were wrong first: inherited from the content it collapsed to the field's width, because content allowed to wrap asks for less than it needs to stay on one line; pinned to the widest row any host could produce, every list was that wide beside fields half the size; and a floor alone put the one long address back over three lines. A host answering on `10.60.3.1` gets the floor, one carrying a full-length IPv6 gets the room to show it with its note beside it, and monospace is what makes the arithmetic honest. **`AddressList` reads in the same words it was chosen in** — the same badge at the same `BADGE_WIDTH`, then the value, then `on eth0` for an address and `now 10.10.0.3` for an interface — because a row an operator comes back to check, described differently from the list it came out of, is a reason to wonder whether it is the same thing. What is *shown* is `bareAddress`: the backend stores what it parsed, so `127.0.0.1` comes back as `127.0.0.1/32`, a spelling nobody types that reads like a range beside real ones; a host-length prefix is dropped for display and a real one is kept, which is also what tells `addressKind` apart. A real range survives an edit for the same reason — the modal seeded itself by cutting at the first slash, so opening "move this address" on a `/16` and saving narrowed a service to one host of it. Where interfaces are not an answer — the `external` proxy endpoint, and the service address on that layer — they are not listed at all and a name typed there is badged red, rather than offered and then refused on submit. Nftables matches use `meta iifname ` for inbound/prerouting and `meta oifname ` for outbound/postrouting/route. On `proxy` (TCP/TLS), redirect rules match `iifname` and `route_chain` marks return packets with `0x1339` on `oifname`, skipping `nat_output_chain` to avoid hijacking unrelated outbound host traffic. On UDP `proxy`, the primary IP assigned to the interface is resolved dynamically via `psutil.net_if_addrs()` to bind the relay. On `nfqueue`, interface traffic is queued natively in prerouting/postrouting without needing an IP. On `external`, interfaces are disallowed because return rewrites require a concrete IP. - -**An ordered chain works on every layer, and gets there differently.** The proxy walks a list inside one process. NFQUEUE cannot — its binaries fuse transport and filter — so there a chain is a chain of *processes*: one binary per filter, each with its own queue and its own pair of base chains, at `-307 + position` (prerouting) and `107 + position` (postrouting). A packet a filter accepts carries on to the next base chain in the same hook, so **filter order is base-chain priority order**. This was established by experiment, not assumed: two `cppregex` processes at adjacent priorities, where the second refuses traffic the first passed. The cost is a process and a reassembly pass per filter, which is why `MAX_CHAIN_POSITIONS` exists at all. - -Changing the *shape* of an NFQUEUE chain (adding, removing, reordering) cannot be pushed: the shape is the arrangement of processes and rules. `NfqueueTransport.reload` raises `ChainShapeChanged` and `ServiceManager.update_chain` rebuilds, logging that it did. Editing a pattern inside an existing filter still costs nobody their connection. - -### One engine per concern, never two -Two implementations of the same concept is the mistake this module was restructured to remove, and it is worth restating because it keeps being tempting: - -- **Regex matching is hyperscan, everywhere — there is no second regex engine in the tree.** `binsrc/regex/` on the NFQUEUE side, `proxysrc/src/hyperscan.rs` on the proxy side, a binding rather than a reimplementation. Adding `regex` back to `Cargo.toml` would undo this. -- **A pattern blocks. There is no second action, and `rewrite` is not coming back.** It existed, and was withdrawn because it could not keep the promise it made: rewriting scanned one chunk at a time, since bytes already forwarded cannot be taken back, so a match straddling two chunks was never rewritten. Measured — a pattern sent whole arrived redacted, the same pattern split across two TCP segments arrived intact, and blocking (which scans in stream mode) caught both. The miss was not the problem; its silence was — no block, no log, no counter, just a rule the operator trusted that had quietly not applied. The removal has to be complete to mean anything, and it was half-done for a while: an `action` column that was gone from the datapath still had a `rewritten` field travelling from `debug.rs` through the backend model to a panel in `RegexDebugger.tsx` that could never render, a single-member `Action` enum in `firegex.regex`, and a `replace_with` on the frontend `Regex` type. If any of those spellings reappears, it is residue, not a feature. -- **A pattern is validated against the mode it will run in**, which is streaming, because that is how a pattern follows a connection across chunk boundaries. hyperscan does not accept quite the same patterns in every mode (`a{1,1000}b` streams but does not block-scan), and judging by the wrong one refuses rules that work — the same class of lie as a tester that disagrees with the engine. Block-mode scanning survives for one job only: reporting *where* a match starts, so the tester can highlight it. `debug.rs` reports `unscannable` for patterns that are valid, will run, and cannot be highlighted. -- **The Python API is `firegex.pyfilters`, everywhere.** `cpproxy` embeds it; `modules/services/pyworker.py` runs it out of process. An earlier version of that worker defined a look-alike `@pyfilter` taking `(data, direction)`, which meant a filter written from the docs loaded on one transport and failed on the other. -- **A filter file's protocol is read off the file, never declared.** The library already decides when to call a filter from what its parameters are annotated with, so `infer_proto` in `firegex/pyfilters/internals` reads the same table back the other way: asking for an `HttpRequest` makes a file an HTTP filter, asking only for a `RawPacket` makes one run on anything, and asking for two *different* application protocols is refused naming both functions. `type_annotations_associations` is the single place a protocol is defined; a second list of names would be one more thing to keep in step, and the day it fell behind the symptom would be a filter that silently never runs. The backend stores the detected value on the row so the interface can show it, and `PUT .../code` is what recomputes it. Passing one in from outside is exactly what used to make an HTTP filter on a TCP service a 500: the *service's* protocol was sent to the datapath, so the filter was refused for asking for an `HttpRequest`, which is the only thing an HTTP filter does. -- **A filter file's functions run in the order the file defines them**, and the first to answer anything but ACCEPT ends the packet. `@pyfilter` sets a mark on the function and hands it back; `collect_pyfilters` reads a module's globals, which are in definition order. What that replaced was a *set of names* on a module-global attribute of the decorator, so the order was whatever hashing produced — which decided which function got credited with a block and whether an earlier `UNSTABLE_MANGLE` survived, with nothing in the file able to tell you which. The `position` column on `pyfilters` carries the same order to the interface, which used to list them `ORDER BY name`. Three consequences of that registry are also gone with it: `get_filters_info` had to clear it before and after (and reached it by `exec`ing import strings), two compilations could race, and a function registered from somewhere the module globals could not reach was registered and then not found. -- **A pyfilter file's functions are switched on and off individually, and the code is never edited to do it.** `pyfilters` holds one row per `@pyfilter` the code defines, reconciled with the file on every `PUT .../code`: functions that have gone are dropped, new ones arrive active, survivors keep the operator's `active` and `blocked`. Only the enabled names reach `__firegex_pyfilter_enabled`, which is already what decides whether the library ever calls a function. Three states are distinct all the way down (`Option>` in the spec, `*` / `=a,b` / `=` on the worker's argv): **absent** means every function the file defines — the case of a filter whose list was never derived, where defaulting to "none" would silently stop every filter running — while an **empty list** means the operator switched them all off. `ChainLink.enabled_functions` returns `None` for no rows and `[]` for all-inactive, which is the same distinction. -- **`RawPacket` is the only model that survives a datagram**, and that fact is stated once, as `DATAGRAM_SAFE_MODELS` in `routers/services.py`. Every other model declines to be built when `is_tcp` is false, so a filter asking for one on a UDP service would sit in the chain and never be called. `--check` reports the models a file asks for, saving refuses the combination naming them, and `Transport.check()` catches the case where the service was switched to UDP afterwards. Python filters *do* work on UDP on both layers: `cpproxy` gained `filter_action_udp` (no stream follower, no ack fixing — a datagram is complete in itself), and its per-flow contexts are bounded by `enforce_limit` because a datagram has no close to observe and a spoofed-source flood would otherwise accumulate one set of module globals per forged address. `handle_next_packet` used to throw for anything but TCP, with a message claiming UDP was supported. -- **A filter file is validated by the process that will run it.** `pyworker.py --check ` builds the module exactly as the worker does, compiles it, and prints one JSON object: `{ok, proto, filters}` or an error carrying **type, message, line, column, the source line and the traceback**. `validate_code()` runs it in a **subprocess with a timeout** — user code used to be `exec`ed inside the backend's own event loop by `detect_proto`, so a `while True:` at module level would have taken the interface down. `POST .../check` answers 200 even when the answer is no: a refusal is the result, and the editor wants a position to mark, not a string to print. Saving goes through the same check, so the operator is never handed "the filter code did not load: worker exited" and left to find the traceback in the service log. - - `_where_in_source` finds the line three ways: a `SyntaxError`'s own position; a traceback frame belonging to the file (`` counts as theirs — the library execs the source to find its filters, and everything the library runs from has a real path); otherwise any identifier in the message looked back up among the file's `def`s, **last definition wins**, because that is the one Python is left holding. - - `Filters` compiles once under the operator's file name instead of re-parsing per connection, so runtime tracebacks in the log name their file and their line too. - - A filter is called directly. `context_call` used to write the arguments into the module globals, `eval` a call expression built from them, and delete the three keys again — per filter, per packet — which is exactly `func(*args)`, because the operator's code was `exec`ed into those globals and so the function's `__globals__` *is* that dict. - - The default for `FGEX_STREAM_MAX_SIZE` is 1 MiB, as documented. It read `1*8e20` under a comment saying "1MB", so `FGEX_FULL_STREAM_ACTION` never fired unless a size was set by hand, and the float it produced would have been refused by the setter beside it. -- **The editor's hints are introspected, never written down.** `modules/services/pyapi.py` reads the models out of `type_annotations_associations` and their members out of the classes — including `writable`, from `property.fset`, which is how the interface can say that the payload is the only thing a filter may assign to. `GET /api/services/pyfilter-api` serves it and `pyfilterHints.ts` feeds Monaco's completion and hover from it. A second, hand-written description would be wrong the first time a model gained a property, and autocompletion that lies is worse than none. The route is declared **before** `/{service_id}`, or that one matches it and looks for a service by that name. -- **A filter that raises says so the same way on both layers.** `_on_exception` writes one sentence — the traffic was forwarded unfiltered — and it has to be reachable from either. NFQUEUE gets there directly: the interpreter is embedded in the binary, which sends `EXCEPTION`. The proxy cannot, because `pyworker.py` catches the exception in a separate process and answers with an ordinary ACCEPT, so nothing crosses the frame boundary that the engine could tell apart from a filter agreeing. It marks stderr with `EXCEPTION_MARK` instead and `Transport._pump_stderr` turns that back into the same call. The traceback is printed *beside* it, not instead of it: the traceback says which filter and which line, and the sentence says what happened to the connection — for a while the proxy printed only the first, which tells an operator a filter broke and not that traffic went through unchecked. The marker is spelled in both files because the worker is launched as a script and imports nothing from the backend, which is what keeps the user's code out of this process's imports; `tests/integration/test_resilience.py` asks for the sentence on both layers and so keeps them in step. -- **A block names the function, not just the file.** Both layers report `/`: the NFQUEUE binaries already reported the function name, and the proxy worker now returns it in the verdict frame (`[verdict][name len][name][payload]`). `ServiceManager._on_block` splits that one token in one place. A file with ten functions blocking 900 connections is not an answer to "which of them is doing it". -- **Nothing below the application layer crosses into a filter.** `RawPacket` exposes addresses, ports, family and direction as read-only metadata, and `data` — the payload — as the only writable thing; `mangled_data` comes back, never a packet. That boundary exists because the two layers cannot honestly offer the same thing underneath it: NFQUEUE has a real header whose rewriting desynchronises the stream, while the proxy terminated the connection and writes its own. The old model papered over that by handing the proxy a literal `FAKE:IP:TCP:HEADERS:` prefix as `raw_packet`, so the same filter did different things depending on where it was attached. On the proxy the metadata is *better* than a header would be — the engine knows both ends and sends them in a `KIND_OPEN` frame (`ConnectionMeta` → `Filter::connection_opened`), once per connection rather than on every chunk. -- **The order the operator set is the order that runs.** `parse_ruleset` groups patterns by their `filter` id and keeps the positions the backend sent. An earlier version grouped every block, then the Python — which turned the chain into a set, so a filter placed after another could run before it, and nothing said so. Within one filter the operator expressed no order, so blocking goes before rewriting there; across filters, never reorder. `tests/rules.rs` pins both. -- **The regex tester runs the engine.** `POST /api/services/debug-regex` shells out to `fgex-proxy --debug-regex`, so what it says is what will happen. A `re`-based approximation would bless patterns hyperscan rejects. - -### Per-module pattern -Each module (`services`, `firewall`) follows the same shape: -- `backend/routers/.py` — FastAPI endpoints + its own SQLite DB, via `utils.sqlite.SQLite`. -- `backend/modules//models.py` — plain model classes, `from_dict`. -- `backend/modules//firewall.py` — `FirewallManager`/`ServiceManager` pair: the manager tracks one `ServiceManager` per DB row, which starts/stops that one service. -- `backend/modules//nftables.py` — builds/tears down the module's nftables rules; extends `utils.NFTableManager` (a `Singleton`), so there is exactly one rule-set owner per module. - -**Every nftables object firegex creates is named `fgex_`, and lives in a table firegex owns.** The services module holds `table inet fgex` with `fgex_queue_in_`, `fgex_queue_out_`, `fgex_hijack_*`, `fgex_divert`, `fgex_nat`, `fgex_nat_out`, `fgex_route`; the firewall module holds `fgex_filter` and `fgex_mangle` per family, with base chains `fgex_input`/`fgex_forward`/`fgex_output`/`fgex_prerouting`/`fgex_postrouting` jumping to `fgex_rules_in`/`_out`/`_fwd`. One `grep fgex_` over the ruleset is the whole footprint. - -The firewall module is the one that had to move to get there, and the reason is worth keeping. It used to write into the tables called `filter` and `mangle` — the ones `iptables-nft` claims — adding base chains named INPUT and FORWARD beside iptables' own and driving their policy. That broke two things, both reproduced before being fixed: - -- **`iptables` stopped working on the host.** `iptables-nft` refuses an entire table as soon as one rule in it cannot be expressed in iptables' own format, and it stores conntrack matches as an `xt` blob where nft writes a native `ct state`. Bisecting the rules firegex writes, `ct state` was the only one that tripped it — which is to say `allow_established` and `drop_invalid`, on in any sane configuration. From then on `iptables -L` and `iptables-save` answer ``table `filter' is incompatible, use 'nft' tool``, on a box where Docker, ufw or the organisers' scripts are the ones asking. -- **Stopping firegex opened the host.** `reset()` re-added `filter INPUT` with `policy accept`, and in nftables `add chain` on an existing chain *updates* its policy. That chain was the administrator's: a host carrying default-deny had it silently removed when the firegex firewall was switched off. - -A base chain carries its own policy wherever it lives, so default-deny is expressed exactly as before and a `drop` is terminal whichever table's chain reached it — nothing about what gets blocked changed. What changed is that firegex now adds a layer instead of taking one over: an `accept` in `fgex_rules_in` means "firegex does not block this", never "nothing else will", which is all it ever honestly meant. `reset()` is now deleting two tables per family (`add` before `delete`, because a batch is atomic and deleting a table that is not there fails all of it). - -`Rule.table` is still `filter` or `mangle` in the database, the API and the interface — it names the hook a rule is evaluated at. `NFT_TABLES` maps it to the table it lands in; the two were one string only because that string used to be somebody else's table.<> - -A refused edit must change nothing: `_apply_chain(service_id, undo)` in `routers/services.py` rolls the write back when the transport rejects the resulting chain. Without it an operator is told their filter was rejected and then finds it in the list, and the next start fails for the reason they thought they had avoided. - -Order is the load-bearing detail in `services/firewall.py`, because getting it wrong is not an error anybody sees — it is a window during which traffic goes unfiltered. Starting: the capture interface if the service decrypts (the engine opens its socket to it at startup, so one that appears afterwards is one this process will not find), then the datapath, then the nft rules that point at it. Stopping is the exact reverse. - -### Block statistics (`modules/services/stats.py`) -A cumulative counter per filter, per pattern and per `@pyfilter` function lives in the database; the bucketed history answers the other question — *when* did this start. Blocks are counted per filter into one-minute buckets, kept for `HISTORY_BUCKETS`, and flushed at most every `FLUSH_INTERVAL` seconds. Both properties are load-bearing for the same reason the live log's are: a service refusing thousands of connections a minute must not be able to take the backend down through its own bookkeeping, which is what one row per block would do. There is deliberately **no background task** — flushing happens on the next write once the interval has passed, and always on a read — so nothing has to be started, cancelled or waited for at shutdown. The frontend draws it as hand-written SVG (`StatsPanel.tsx`) rather than pulling in a charting library several times the size of the page it sits on. - -`share` is computed in the backend, not the browser, so a chart and the table beside it cannot disagree about what a rule accounts for. - -**History is stored per *rule*, not per filter, and that is what makes a time range mean one thing.** `block_history` keys on the token the datapath reported — a pattern id, `/`, or a filter id — so `GET .../stats?range_from=&range_to=` can answer the chart, the per-filter totals, the per-pattern totals and the shares over the *same* window. Storing only per filter would have left the chart honouring the range and the tables beside it reporting all of time. A filter's total is the **sum of its rules**, not a lookup of its own id: that token appears only when nothing named itself. - -`bucket_width` takes either a `target` bar count (the default, which is what makes the bar widen with the range) or an explicit `want` in seconds from the operator, and both are floored at `MAX_STEPS` bars — a step fine enough to melt the chart is widened, never refused, and `bucket_seconds` reports what was used rather than what was asked. A range never grows the number of buckets, only their width (`bucket_width`): everything is an aggregate of the one-minute rows, so twelve hours costs the same rows as one and is drawn in the same handful of bars. Ranges are clamped to **two** floors — `retention_start()` and the service's `filtering_since` — and the served range is reported back, so the interface can say what it actually showed. `filtering_since` is written by `_note_filtering()`, which is called from `start_service` and from `_apply_chain` (the two places the two halves of "running, with an active filter" change) and once per service at `startup()`, which is what covers services restored from a backup or predating the column. It is written once and never moved: a service stopped and started again has a real gap in its chart, and moving the mark would hide it. Note the clamp happens *after* the caller's ends are put in order, and the `filtering_since` floor applies **only to a window that reaches into the service's life**: a window that ended before it began is answered exactly as asked and comes back empty, because it is. Dragging its start forward instead would have reported the current minute's blocks under a range an hour old — buckets are minute-aligned, so a zero-width range is not an empty one. `all_time` is reported alongside `total` so a narrow window showing nothing is not read as a contradiction against the cards. - -`StatsCharts.tsx` draws bars, lines, stacked area and 100%-share from the same series — four views, one dataset, still hand-written SVG for the reason the first chart was. Hover is tracked once on the container and turned into a step index, not per shape: a line and an area have nothing under the cursor to attach a handler to, and one handler means the four charts cannot disagree about which step is being pointed at. The readout itself is HTML positioned over the plot, for the same reason the axis labels are — `preserveAspectRatio="none"` stretches rectangles correctly and distorts glyphs. `clock()` is exported from there and used by the axis labels, so a bar's tick and its readout cannot disagree about what a step is called. - -**Editing a pattern keeps the row, resets its counters, and hands its history to the filter.** `PUT .../regexes/{id}` validates the *resulting* row through the engine (changing the action alone can invalidate a stored pattern, since blocking compiles for streaming and rewriting for block scanning), and when the pattern text itself changed it resets `blocked` and calls `stats.disown_rule` — counts belong to the text that produced them, so the new text must not inherit them. That call happens **after** `_apply_chain` accepts, never before: a refused edit has to change nothing, and history moved ahead of the verdict cannot be put back by `undo()`. The rule's *pending* batch moves with it, or the last few blocks of the old pattern would be flushed under the new one. - -What `disown_rule` must not do is **delete** those rows, which is what it used to do. Those connections were refused, by that filter, in those minutes, and the chart is answering *when this service was under attack* — a question a later typo fix has nothing to say about. Deleting took a bite out of the timeline on every correction, and on a filter holding one pattern it emptied the chart. So the rows are re-keyed to the filter's own id, already the token the datapath reports when nothing finer named itself: `window()` and `totals_by_filter()` group by `filter_id` and do not change at all, while `totals()` groups by `rule_id` and correctly credits the new text with nothing. - -The cost is that some blocks in a range then belong to no rule still present — which was already true of a *deleted* pattern, whose rows are left behind on purpose. The stats route closes that by adding one **residual** entry per filter for the difference between what the filter was credited with and what its current rules account for (`residual: true`, prose rather than a pattern, so `StatsPanel` prints it as text and not in a `Code` box). Without it the per-pattern list quietly adds up to less than the chart above it, and the operator is left hunting for the difference. Note `check_pattern` coerces `case_sensitive`: a caller checking a row it read back from SQLite hands over `1`, and the engine's request format refuses an integer where a boolean belongs. - -**Traffic totals are reported in the unit each layer can honestly produce, and the two are never divided by one another.** This is the part to preserve: - -- **Packets and bytes** come from `counter` statements on the intercept rules — free, kernel-side, and unskewable because no userspace filter is involved in producing them. `FiregexTables.traffic()` reads them back, from **one inbound chain per transport**: the nfqueue layer installs a rule pair per filter position all seeing the same packets, so summing them would multiply the answer by the length of the chain. -- **Proxy services report no packet count at all.** Their rule lives in a `nat` chain, and conntrack translates a connection once — every packet after the first is handled without the rule being walked again. That counter is *new connections wearing a packet label*, so it is dropped rather than shown. -- **Connections** come from the engine's periodic `STATS seen=… refused=…` line. They live on `ProxyStats`, not on `ChainStats`: a chain is replaced wholesale on every ruleset push, and counters that reset on a rule edit would make the ratio jump for a reason nobody could see. Because both numbers are connections, `refused_share` there is exact. -- NFQUEUE has no connection to count, so it reports `null` for the share rather than dividing refused connections by a packet count. **The asymmetry is the difference between the layers showing through; do not "fix" it by inventing a denominator.** The engine's counters arrive on a timer, so a test that reads them has to wait for them. - -### The live log (`modules/services/logs.py`) -A bounded ring per service plus a coalescing flush over a `log` Socket.IO event (distinct from `update`, which only invalidates queries). Two properties are load-bearing and easy to lose: the ring is capped, and the flush is on a timer with a per-batch cap that reports a dropped count rather than carrying it — a service refusing thousands of connections a minute must not be able to take the backend or the browser down through its own logging. - -Sources: block attribution (resolved to the filter name and pattern, one query per block, on a path that only runs when something was already refused), the user's `print()`, and the datapath's own stderr. That last one is why `pyworker.py` reassigns `sys.stdout = sys.stderr` before running user code: stdout is the length-prefixed protocol channel, and a `print()` landing in the middle of a frame gets the worker killed on every packet with nothing to explain it. `tests/pyworker.rs` has a regression test for exactly that. - -### The proxy datapath (`backend/proxysrc/`) -A Rust binary, built in the Dockerfile's `compiler` stage rather than a `rust` image on purpose: it links the same `libhs` the C++ binaries do, and a binary built against one distro's vectorscan and run against another's is a bad afternoon. - -- `filter.rs` — the chain, and everything that keeps a filter from taking the datapath down: `catch_unwind`, a deadline on `spawn_blocking`, and a filter that misbehaves losing its say rather than the traffic being held. `panic = "unwind"` in `Cargo.toml` is load-bearing, not tuning. -- Filters that keep state do so **per connection, per direction** (`ChainSessions`), because one client's bytes deciding another client's verdict is both a false positive and a way to smuggle a pattern past a filter by splitting it across two connections. -- `pyworker.rs` frames carry a connection id and a close frame, so the Python side can honour the documented promise that each stream gets its own module globals. -- Source preservation is unconditional: the engine always dials the service from the client's address. `ct status dnat` is what separates the engine answering a client from the service answering the engine, which is why redirect (not tproxy) works even when the service is on this host. - -### The `external` hand-off -Three nft chains, not two: prerouting for traffic from outside, **output for traffic this host generates** (which never reaches prerouting — the same reason the proxy transport has `fgex_nat_out`), and postrouting to undo the rewrite on the way back. A `mangle` target is one host, so `one_address()` strips the prefix that service addresses carry; - -Note for tests: Linux refuses a loopback connection to a port with no listener *without emitting a packet*, so no rule can rescue it. A hand-off test has to keep the real service listening and tell the two apart by their replies, not by which one answers. - -### Holding up under load - -**A service carries at most `max_connections` at once, and the operator says what happens to the rest.** It exists because the alternative was measured, not argued: silent connections cost two descriptors each — one from the client, one to the service, since the upstream is dialled on accept — and ~505 of them from one host exhausted the container's 1024 and took **every** service down. UDP was cheaper still: 600 datagrams from 600 forged sources in 0.03 s took 400 descriptors and held them a minute after the sender left. The same cap on the same shape already existed on the NFQUEUE side (`MAX_UDP_FLOWS` in `pyproxy.cpp`) and had never been carried to the Rust engine; TCP had none anywhere. - -Say what it buys, and no more. **A cap does not save the service being attacked** — it cannot tell a connection that is silent because it is an attack from one that is silent because the client is slow, so an attacker who fills the limit still fills it. What it buys is that one service's attacker stops being everyone's: measured, 400 silent connections against a limited service left the service beside it answering. Do not let the interface or the docs imply otherwise. - -- One number for both halves, because TCP connections and UDP flows spend the same descriptors and two limits for that would be two things to keep in step. `0` is no limit, kept as a value rather than an absence, because "unlimited" is a choice. -- `over_limit_forwards` is the same trade as fail-open at a different moment, and off by default for the same reason: a filter that quietly stops filtering is the worse surprise. -- Folded into the form's one **Advanced settings** accordion, not dropped: these are settings an operator reaches for once — after something went wrong, or before a competition — and never again, and left in the open they are more things to read past on every service. One fold rather than one per topic, and its summary is built from what *differs from the defaults* (`changed`), so an untuned service says "Everything at its default" and a tuned one says exactly how. Folded is not hidden. What is inside depends on the layer, because the settings belong to different ones: the connection limits are the proxy's, fail-open is NFQUEUE's, and the hand-off layer has neither, so it gets no accordion at all. -- **Proxy layer only**, and absent from the form elsewhere. NFQUEUE hands the kernel a verdict on packets in flight; it accepts nothing and dials nothing. The form also resets it to `0` when the layer changes, because a stored setting that does nothing is the trap this whole file keeps arguing against. -- `Slot` is a guard, not a decrement at the end of `handle_connection`: that function has a dozen ways out — a refused chain, a failed dial, a timed-out handshake, a panic — and a counter that leaks on any one of them is a limit that tightens until it refuses everything. -- The accept loop **backs off** after a failed accept. That one is not a policy question: out of descriptors, the loop spun on `accept` → `EMFILE` → print → `accept`, measured at ~40% of a core with four clients knocking, writing a log line per attempt into the pipe the backend reads. - -**`first_byte_timeout` is what the cap cannot be.** A cap contains a phantom flood; a deadline ends it — measured, with the limit full and the attacker still holding every socket, clients got back in once it passed. Two properties are load-bearing and easy to break: - -- **Either direction counts.** The flag is set by whichever pump moves a byte first, so a service that greets its client (SMTP, SSH, most game protocols) satisfies it with its banner. Requiring the *client* to speak would hang every server-speaks-first protocol, and the failure would look like firegex breaking them at random. -- **Only until the first byte, never again.** A connection that has spoken and gone quiet is a session, and sessions think. This is not an idle timeout; making it one would close long-lived connections that are legitimately waiting. - -The flag is set *before* the chain runs, because a chunk a filter goes on to refuse is still a connection that said something. The watchdog fires through the same `stop_tx` a refusal uses, so both directions shut down cleanly rather than being dropped — a dropped peer reads a truncation, and this connection has done nothing to deserve one. Zero is off, and there is no default worth picking: how long a legitimate client may stay silent is a fact about the service, not about firegex. - -**The trace is deliberately in two places.** `_on_over_limit` writes `over_limit_hits`/`_first`/`_last` to the database *and* logs at warning level, rate-limited by `OVER_LIMIT_QUIET`. The log answers "why are clients failing right now" and is a bounded ring, so a burst at three in the morning is gone by breakfast; the columns answer "did we ever hit the wall", which is asked the next day. The engine's counter is cumulative *per process*, so the transport hands up only the **increase** — passing the absolute number would reset the durable count on every restart of the service. - -### TLS - -**The two controls that constrain each other read one rule.** `carries(transport, proto)` in `components/Services/utils.ts` is the only place the constraint lives: the protocol picker hides what the chosen layer cannot carry, the layer picker disables what cannot carry the chosen protocol, and `Transport.check()` refuses it on the server. Written twice, the two halves drift and the form ends up offering a combination the backend then refuses — being told no after filling in a form is a worse way to learn it than not being offered the option. Both pickers say no the same way: the option stays where it is, **disabled**, with a sentence underneath saying why and what would make it available. An option that vanishes leaves nothing to ask about — the operator is left wondering whether it exists at all or whether they misremembered seeing it — and two controls side by side that refuse in two different ways are a confusion of their own. - -**TLS is one of `L4`'s values, beside `tcp` and `udp` — not a boolean on top of one.** It is what the service speaks, and as a flag it made three combinations expressible that are not real: TLS on a UDP service, on NFQUEUE, and on a hand-off, each caught by a check written by hand. Two of those are now unrepresentable, and the third (`NfqueueTransport.check()`) is asked at **creation** as well as at start — `add_service` calls `transports.build_class(...).check(srv, [])` with an empty chain, which catches what the *service* makes impossible while leaving what the *chain* makes impossible to the moment there is one. A row that is created happily and then refuses to start every time is a trap. - -The second answer that split off from it: an address carries the transport **the kernel matches on** (`L4.l4_of`), not the service's own, because `tls` is TCP on the wire. Without that, `(ip, port, proto)` would let a TCP service and a TLS one claim one `ip:port` between them, which is the same port twice. `l4_proto()` in `nftables.py` is gone with it — the address already holds the answer it used to compute. - -It belongs to the **proxy** layer: decrypting means terminating the connection, and terminating is what that layer does. That is not a lost feature so much as an honest one — TLS on NFQUEUE worked by putting nginx in front to terminate and re-encrypt, so the connection *was* terminated and the layer's one distinguishing property was already gone. - -**The engine terminates it, and a TLS service therefore occupies no port at all.** `proxysrc/src/tls.rs` decrypts from the client and re-encrypts towards the service inside the process that filters, so `resolve_target()` points the rules at the address the world dials, exactly as for a plain service. `ProxyTransport._tls_env()` hands the engine `FGEX_PROXY_TLS_CERT`/`_KEY` — **paths**, because a certificate on a command line or in an environment dump is a certificate in a log — written `0600` under `TLS_DIR` and removed after the engine is gone, never before. - -What that replaced: nginx terminated on one derived loopback port and re-encrypted from a second, with the filters attached to the leg in between. Protecting one address cost a pair of ports produced by hashing `ip:port` — *chosen* ports, which could collide with something real — plus a generated config, a `tls_streams` table keyed by address id, an nft chain of its own, and a failure mode where one unloadable certificate stopped nginx and took **every** TLS service on the instance down with it. All of that is gone: `modules/tls/` no longer exists, and the blast radius is now structural, since each service carries its certificate into its own engine process. - -**ALPN is mirrored, and the ordering that makes it possible is the design.** The client's handshake is *started* and held at the ClientHello (`tokio_rustls::LazyConfigAcceptor`), because that message carries the protocols the client will speak; the upstream handshake goes next offering exactly that list, and whatever the service picked is the single protocol the client is then told (`tls::with_alpn` / `tls::answering_with`, both cloning the config per connection — the list comes from the client, so there is no fixed set to precompute, and a config clone is nothing beside a handshake). Answering something the service did not say is how a proxy breaks a protocol it was only carrying: a client told `h2` against an HTTP/1.1 service sends frames nothing can read. When the service picks nothing the client is told nothing — no agreement is invented. `tests/integration/test_tls.py` pins all five cases, including both mismatches and a client that offers only one protocol. Before this, ALPN was dropped in both directions and every client silently fell back to HTTP/1.1. - -What cannot be carried, and is a limit rather than a missing setting: **TLS 1.0 and 1.1**, which rustls does not implement at all; the cipher list, which is fixed and forward-secret only; and **client certificates**, since `with_no_client_auth()` is the only honest answer when the certificate a client would present is one only the service can check. Say these are impossible rather than unconfigured — an operator who needs TLS 1.0 needs the `external` layer, not a flag. - -The key must be one `ring` will sign with — RSA ≥ 2048 bits, ECDSA, or Ed25519. nginx could be told to take a 1024-bit key with `@SECLEVEL=1`; rustls has no equivalent, so a smaller key is refused when the service starts. That refusal is worth having land well: `_pump_stderr` keeps the engine's last `[fatal]` line and `_died_because` puts it in the exception, because "the proxy engine did not report a listening port" is the symptom and the engine already printed the reason. - -**The decrypted traffic is written to an interface of its own, and those are reconstructed packets.** Terminating in-process means the plaintext is never a packet, so `proxysrc/src/capture.rs` frames it as the TCP stream it was and sends it to `firegex0` over `AF_PACKET`: one interface carrying every TLS service's plaintext and nothing else, which is what a capture tool wants pointed at it. The bytes are real — what the filters saw and what was forwarded — but the framing is invented, because the framing on the wire was encrypted. Sequence numbers start at zero, there are no retransmissions, and the segmentation is the engine's read sizes; a SYN and a FIN are emitted so Wireshark has a stream to follow rather than orphan segments. Say so wherever it is offered: a capture from here is not evidence of what was on the wire. - -Three things keep it from being able to hurt anything. `modules/services/mirror.py` creates the device in `FirewallManager.init()`, before any engine starts (a socket opened before the interface exists finds nothing), and `release_capture()` removes it from the process's `shutdown()` hook and nowhere else — deliberately not from `close()`, which `reset()` also calls, or the device would vanish and reappear underneath whatever was capturing from it. It is the **instance's** device, not each TLS service's: it used to be created and released per service, on the argument that an interface present while nothing is decrypting is one somebody points a capture at and watches stay empty — but a capture tool is attached once, for a round, and Zeek and Suricata exit rather than wait when the interface under them disappears, which restarting a single TLS service was enough to cause. Watching it stay empty is the better failure. `release_capture_if_idle()` is gone; do not add a per-service release back. Every failure is silent and local — no interface, no socket, a failed send — and the traffic goes on exactly as it would have. And the `AF_PACKET` socket needs `CAP_NET_RAW`, which `docker-entrypoint.sh` asks for **and starts without**: it tries the full capability set, falls back to the set without `cap_net_raw` while saying what that costs, and only then runs as root. A host that will not grant it loses the capture, not the firewall. - -Note for tests: a client now dials the service's own address, because there is no second port to be pointed at. The old note about the DNAT skipping loopback belonged to nginx's public-port redirect and went with it. - -### Authentication (`backend/app.py`) -Every module router is mounted onto `api = APIRouter(prefix="/api", dependencies=[Depends(is_loggined)])`, so a new file in `routers/` is authenticated by construction. The whole auth surface is four places: that dependency, the Socket.IO `connect` handler (reads the token out of `auth`), and the three routes declared on `app` directly rather than on `api` — `/api/status`, `/api/login`, `/api/set-password`. Adding an unauthenticated route means declaring it on `app`, which should stay a deliberate, rare act. - -`--unsafe-disable-auth` (env `UNSAFE_DISABLE_AUTH`) hands access control to a reverse proxy: `check_login` returns `True`, the socket.io handler skips its check, and `APP_STATUS()` reports `run` even with no password stored. The password endpoints (`/api/login`, `/api/set-password`, `/api/change-password`) answer `403` in this mode on purpose — otherwise an anonymous caller could plant a credential that keeps working once authentication is turned back on. `run.py config --password` still works, writing the hash straight into the DB from the host, and an explicit `-P` at first start is kept so that re-enabling authentication lands on a known password instead of the open initial-setup state. - -**The flag is state, not a constant.** It lives in `keys_values["auth_disabled"]` and every check reads it (`auth_disabled()`); the environment is only the seed, written once per boot in `startup_main`. It used to be a module-level constant read at import, which is why `run.py config --password` could print "it will take effect immediately" at a process that had already decided every caller was an administrator — the hash landed in the database and nothing ever asked for it. `run.py`'s `put_runtime_settings()`/`get_runtime_setting()` are the one way in and out of a running instance's `keys_values` (Docker exec or the standalone rootfs, same function), used by both `--password` and `--[no-]unsafe-disable-auth`, and `config --show` says when the running instance disagrees with the persisted config. **`config --password` re-enables authentication when it was off**, live and persisted: setting a password is asking for one to be asked for, and a stored-but-unchecked password is worse than none because it reads like one. It stands down when the same command line said otherwise — an explicit `--unsafe-disable-auth`, or `--keep-auth-disabled` for an instance deliberately held open behind a proxy. Note the password block runs *after* the auth block in `handle_config_command`, which is why the auth block's "there is no password set" warning checks whether one is about to be set. - -`POST /api/auth-mode` turns it off and back on at runtime. **The two directions are not symmetrical, and that asymmetry is the design:** off is an administrator's deployment choice, but on, asked while it is off, would let any passer-by set a password and keep the real operator out for good — a lasting foothold, unlike the vandalism the mode already permits. So re-enabling takes a token this instance signed *before* it was turned off. Nothing new is signed while it is off (login and set-password both 403), so holding one is exactly that proof, and the operator who turned it off still has theirs. `token_is_valid()` is split out of `check_login` for this single caller: it needs the question without the "authentication is off, so yes" that `check_login` answers first. Either direction lasts only as long as the process, because the environment is what firegex comes up with — `run.py config` writes both the live database and `.firegex-conf.json`. - -Frontend note: a 401 used to call `window.location.reload()`. A token left over from a database that has since been recreated (a new `secret`) 401s on the first query, the page reloads itself, and what comes back is whatever `APP_STATUS()` calls for — "choose a password", if none is set. That is the "sometimes a password screen appears on refresh" report. The token is now dropped and the app re-renders from the next `/api/status`, and no `Authorization: Bearer null` is sent when there is no token at all. - -Independently of that, `IPFilterMiddleware` restricts access by CIDR (`--allowed-ips`, optionally reading the client IP from `--proxy-ip-header`). It **fails closed**: a missing, unparseable, or non-matching IP is denied. Note the header variant trusts a client-supplied value, so it only holds if the proxy overwrites it. - -### SQLite wrapper (`backend/utils/sqlite.py`) -Each DB file's schema is versioned by hashing the Python schema dict; on mismatch the **entire file is deleted and recreated** (not migrated) — any schema dict change wipes that DB file on next startup, including `db/firegex.db`'s `keys_values` table (password/JWT secret). `dump()`/`load()` back the export/import backup feature; `dump()` deliberately excludes `password`, `secret` **and `auth_disabled`** from exported `keys_values` rows, so `/api/import` explicitly re-preserves all three around the import. The third one is the newest and the easiest to lose: it is not a credential, it is *whether credentials are asked for*, which belongs to the deployment and not to the configuration being restored — a backup able to carry it is a backup able to turn a firewall's authentication off. `auth_disabled` is also put back **after** `startup_main()`, not before like the other two, because `startup_main` seeds it from the environment: right when a process is starting, wrong during an import where nothing restarted and the environment did not change. `tests/integration/test_backup.py` turns authentication off *around* the import on purpose — asserting it is still on afterwards passes either way on a container whose environment agrees, and the bug only shows where the running state and the boot-time environment differ. - -### Real-time frontend updates -One Socket.IO event, `"update"`, whose payload is a list of strings treated as a react-query `queryKey` prefix (e.g. `["services"]`, `["firewall"]`). `App.tsx` does `queryClient.invalidateQueries({ queryKey: data })` — prefix matching invalidates any query whose key starts with that array. Convention: each module's mutating endpoints emit the same tag string that module's frontend queries use as the first `queryKey` element. `staleTime: Infinity` is set globally, so this event is the **only** automatic refresh mechanism — a missing/mismatched tag means silent staleness for other clients/tabs. - -### Frontend structure -`pages//` (routes) + `components//` (ServiceRow, AddEditService, `utils.ts` with query key + API wrapper) per module, mirroring the backend split. Detail drill-down lives at `pages//ServiceDetails.tsx` under a nested `:srv` route (see `App.tsx`'s `PageRouting`). Theme tokens are CSS variables in `src/index.css`; Mantine `Card` defaults are set globally in `index.tsx`. Prefer Mantine `Group`/`Stack` with `gap` over the legacy `Box className="center-flex"` utility classes still present in older parts of the codebase. - -### `fgex-lib/` -Separate, independently-published PyPI package (`pip install fgex`/`firegex`) — the library users write Python filters against (`@pyfilter`, `RawPacket`/`HttpRequest`/etc.), plus the `fgex` CLI, a local `proxysim` for testing filters, and `firegex.regex` for testing *pattern* rulesets, without a real firegex instance. - -The library module is **`firegex.pyfilters`**, and the CLI command is `fgex pyfilters`. Both were `nfproxy` when that was one of several modules. A compatibility alias carried the old spellings through 4.x and was **removed in 5.0.0**, which is a breaking change and the reason the version is a major one: `from firegex.nfproxy import pyfilter` is the first line of every filter written before it, including the ones already sitting in a running firegex's database. - -Two things about that removal are worth keeping. The directory has to go, not just its contents — a `firegex/nfproxy/` left behind with only a `__pycache__` in it is a *namespace package*, so the import succeeds and hands back something empty, and `from firegex.nfproxy import pyfilter` then fails with an `ImportError` about a name rather than a `ModuleNotFoundError` about a module. And the failure is made to explain itself where the operator meets it: `pyworker.py --check` recognises this one import and says what to write instead, because the alternative is a filter that stops loading at the first start after an upgrade, which at a competition is the start of a round. - -Do not reintroduce the alias as a file of re-exports if it ever comes back. `type_annotations_associations` maps model **classes** to protocols, and both the library and the datapath decide when to call a filter by looking an annotation up in that table, so a shim building second copies of `HttpRequest` would leave a filter annotating a class the table has never heard of — importing cleanly and then never being called. The old one aliased submodules into `sys.modules` for exactly that reason: one class per model, two spellings of one package. - -**Deploying is `run.py`, and only `run.py`.** The package used to carry a `fgex deploy` passthrough that exec'd a copy of it; it is gone, along with the build-time copy `setup.py` made. The package is for writing and testing filters — `firegex.pyfilters`, `firegex.regex`, the two simulators — and starting a firegex is a separate job done from a clone. Do not reintroduce a deployment command here: the reason the old one was a verbatim passthrough was that any second description of `run.py`'s argument surface drifts, and not shipping one at all is the same argument taken one step further. - -**The HTTP parser ships as source, and nobody upstream will bump it.** `fgex-lib/llhttp/` carries llhttp itself (`lib/`, verbatim from a `release/*` tag) and our binding beside it, because `pyllhttp` is archived, every binding with prebuilt wheels fails to load in the subinterpreters `cpproxy` runs filters in, and a ctypes binding against the system `libllhttp` would make `pip install firegex` assume a package installed beforehand. `PROVENANCE.md` there is what makes that maintainable: which tag, which checksums, what is ours and what is theirs, and how to move it. Each release changes what the parser *accepts* — going from 9.2.1 to 9.4.3, an empty `Transfer-Encoding` stopped being accepted and tabs around `Content-Length` started being — so `tests/unit/test_http_parsing.py` pins both through the public models: a change there is a decision about what filters are shown, never a test to fix. A parser more lenient than the service behind it is how a filter gets bypassed while still looking switched on. **The binding names itself and nothing else**: `pyllhttp` is where it was forked from, not something it uses, and the error path that imported it at runtime raised `SystemError: ... returned NULL without setting an exception` on every malformed request anywhere that archived package was not installed — which is everywhere except the machines it was written on. `wheel_smoke.py` is the check that could have caught it, and the reason it is a file rather than a `-c` string: `cibuildwheel` runs it against each wheel in an environment with nothing else in it, parsing something malformed as well as something valid. - -`firegex/regex/` binds `libhs` through ctypes rather than depending on a wheel: no build step, nothing to go stale, and by construction the same library the product loads. It is a third binding to one engine, not a third engine — the distinction that matters is that there is still only one thing deciding what matches. Where libhs is absent the tools raise `HyperscanMissing` and stop; adding an `re` fallback would reintroduce exactly the lie the in-app tester was built to avoid. `tests/unit/test_regex_library.py` pins the agreement: validity in the mode a pattern will run in, direction, the shared file format, and that a ruleset asking to `rewrite` is refused rather than quietly treated as a block. Not part of the backend/frontend build. `fgex-lib/README.md` is a short index (PyPI-facing) linking out to `docs/*.md`; it isn't rendered anywhere itself. - -### Module documentation (`docs/*.md`) -Each module's user-facing docs (`docs/services.md`, `docs/pyfilter.md`, `docs/firewall.md`) are single markdown sources, each imported directly with Vite's `?raw` suffix by a matching `Docs.tsx` component (e.g. `components/Services/ServicesDocs.tsx`) and rendered through the shared `components/MarkdownDocs.tsx` (React Markdown + `remark-gfm`, code fences routed through `CodeHighlight`, relative `.md` cross-links rewritten to the equivalent GitHub blob URL since they'd otherwise resolve against the app's own origin). `DocsButton.tsx`'s `DocType` enum wires each module to its component. Every language a fence uses has to be in `index.tsx`'s `createHighlighter({ langs })` — Shiki throws for one it was not given, and that throw is uncaught, so it takes the whole page down rather than the block it could not colour. Editing a module's docs means editing its one `docs/*.md` file — never the `*Docs.tsx` component directly. Reading these files across the `frontend/`↔repo-root boundary requires `vite.config.ts`'s `server.fs.allow: ['..']` (dev) and the `Dockerfile` frontend build stage's `COPY ./docs/*.md /docs/` (prod/standalone rootfs) to stay in sync with any path changes. +Anything genuinely specific to one agent belongs in that agent's own file. So far nothing +has been. diff --git a/README.md b/README.md index 8f7cbfa..9041a7f 100644 --- a/README.md +++ b/README.md @@ -86,11 +86,15 @@ This is especially useful if you lost the current password: it's applied directl - **[Services](docs/services.md)**: the main module. A service is one protected endpoint described in two independent parts — a **network layer** that says how its traffic is intercepted, and an ordered chain of **filters** that says what happens to it. - A service protects a **list of addresses or network interfaces**, mixing IPv4, IPv6 and network interfaces (`eth0`, `wg0`, etc.) freely: intercept traffic on specific IPs or directly across an entire interface. Adding an address or interface to a running service drops no connections. - - Network layer: **NFQUEUE** (packets lifted to userspace with [nfqueue](https://netfilter.org/projects/libnetfilter_queue/) and [nftables](https://netfilter.org/projects/nftables/), nothing terminated, the kernel keeps forwarding if a filter dies) or **proxy** (the connection is terminated and reopened, which buys kernel reassembly, an unlimited ordered chain of filters, TLS termination and real backpressure — while still dialling your service from the client's own address, so it never stops seeing who is talking to it). + - Network layer: **NFQUEUE** (packets lifted to userspace with [nfqueue](https://netfilter.org/projects/libnetfilter_queue/) and [nftables](https://netfilter.org/projects/nftables/), nothing terminated, the kernel keeps forwarding if a filter dies) or **proxy** (the connection is terminated and reopened, which buys kernel reassembly, an unlimited ordered chain of filters, TLS and QUIC termination and real backpressure — while still dialling your service from the client's own address, so it never stops seeing who is talking to it). - Filters: **regex**, matched by [hyperscan](https://github.com/VectorCamp/vectorscan) — the same engine on either layer, so a pattern means the same thing wherever it runs — or **Python**, your own code written against the [`firegex`](https://pypi.org/project/firegex/) library, with built-in models for HTTP and TCP streams. You never declare which protocol a Python filter speaks: asking for an `HttpRequest` is what makes it an HTTP filter, and a filter reads the addresses and ports as metadata while the payload is the only thing it can change. - A regex **blocks** the connection, and that is deliberately all it does: rewriting had to scan one chunk at a time, so a pattern split across two TCP segments went through unredacted with nothing to say it had — a rule you trust and that quietly does not apply is worse than no rule. To change bytes rather than refuse them, use a Python filter, which sees the reassembled stream. - **TLS** is one of the protocols a service can speak, not a separate thing to create: the proxy engine terminates the connection itself, the filters inspect the plaintext, and it is re-encrypted towards your service — with the client's ALPN mirrored rather than invented, and the decrypted stream written to a `firegex0` interface you can point a capture at. A TLS service occupies no extra port. - Or **your own proxy** as the network layer: firegex rewrites the destination on the way in and changes it back on the way out, so a proxy you wrote yourself sits in the path invisibly. The escape hatch for a protocol nothing built in understands. + - **QUIC**, terminated: it is the fourth protocol a service can speak, and the only one with no alternative — past its first packet QUIC encrypts the frames and the stream boundaries as well as the payload, so nothing that merely forwards can be shown to a filter. Each stream inside reaches the chain as its own connection, and **HTTP/3 is rendered as the HTTP/1.1 it would have been**, so an `HttpRequest` filter or a hyperscan pattern written for a TCP service means exactly the same thing on QUIC without being rewritten. + - **HTTP/2 is rendered the same way**, and with it **gRPC**. HTTP/2 puts the method, the path and the headers in an HPACK-compressed frame, so a firewall that only forwarded it showed a filter a compression format: patterns matched nothing and an `HttpRequest` filter was never called, silently. The engine terminates it now — under TLS, or in the clear from the connection preface — so one pattern and one Python file cover HTTP/1.1, HTTP/2 and HTTP/3. + - **HTTP** is a protocol a service can speak in its own right, meaning *every* version at once: its TCP addresses carry HTTP/1.1 and HTTP/2, encrypted or not as each client chooses, and its UDP addresses carry HTTP/3 — one service, one chain, one certificate, instead of two or three services with the same filters copied between them by hand. + - And a Python filter can read **gRPC messages** rather than the framing they arrive in: one message at a time, as each completes, so a streaming or bidirectional call is filtered while it is still open rather than logged after it ends. - **TCP or UDP**, on every layer — with full source IP transparency on both. On the proxy layer, UDP datagrams are relayed with one dedicated socket per address using transparent IP spoofing (`IP_TRANSPARENT`), with per-flow filter state and zero-downtime dynamic address additions. NFQUEUE filters packets in place with kernel-level fail-open. - **The two inspecting layers are opposite trades, [compared side by side in the interface itself](docs/services.md#network-layers) while you choose**: the proxy owns both halves of a connection, so the kernel reassembles for it and a Python filter can rewrite a payload exactly, at the cost of rebuilding fail-open by hand; NFQUEUE is per packet and keeps forwarding by kernel guarantee if a filter dies, at the cost of userspace reassembly and a process per filter. - **Charts of what each filter has refused** — per filter, per pattern and per `@pyfilter` function, each with **its share of the blocking**. Pick the window (last 15 minutes, hour, 6h, 24h, everything kept, or two exact instants) and the shape (bars, lines, stacked area, or share); **everything on the page counts the same window**, so the chart and the table beside it can never tell different stories. Alongside it, how much traffic arrived, in the unit each layer can honestly report. diff --git a/backend/modules/services/firewall.py b/backend/modules/services/firewall.py index 16d9acd..4fcc29a 100644 --- a/backend/modules/services/firewall.py +++ b/backend/modules/services/firewall.py @@ -24,8 +24,10 @@ from modules.services import mirror from modules.services import transports from modules.services import stats from modules.services.logs import LEVEL, forget as forget_log, log_for -from modules.services.models import KIND, STATUS, Address, Filter, Regex, Service -from modules.services.nftables import FiregexTables, NoRelayAddress, udp_relay_host +from modules.services.models import (KIND, L4, STATUS, UPSTREAM, Address, Filter, + Regex, Service) +from modules.services.nftables import (FiregexTables, NoRelayAddress, + interface_addresses, udp_relay_host) from utils.sqlite import SQLite nft = FiregexTables() @@ -401,14 +403,18 @@ class ServiceManager: reopened. Saying so and rebuilding beats installing a rule that redirects traffic at a socket which will refuse it. """ - was_dual = self.srv.has_ipv6 + was_dual = self.srv.has_ipv6_tcp self.reload_addresses() if not self.active: return + # Asked of the transport the kernel sees, not of what the service speaks: the + # one listener that has a family to get wrong is the TCP one. Anything relayed + # per address — datagrams, and QUIC — binds a socket for the new address when it + # arrives, in whatever family that address is, so there is nothing to reopen. if ( self.srv.transport == transports.TRANSPORT.PROXY - and str(self.srv.proto) != "udp" - and self.srv.has_ipv6 + and self.srv.carries(L4.TCP) + and self.srv.has_ipv6_tcp and not was_dual and not self.transport.is_dual_stack ): @@ -422,19 +428,48 @@ class ServiceManager: if not added: return async with self.lock: - # UDP is relayed by one socket per address, so a new address is a new relay - # — opened on the engine that is already running, before the rule that will - # point traffic at it exists. The engine owns the map of them: `_steer` + # UDP is relayed by one socket per address — and QUIC by one endpoint per + # address, for the same reason — so a new address is a new relay, opened on + # the engine that is already running, before the rule that will point traffic + # at it exists. The engine owns the map of them: `_steer` # carries that same dict, so what is opened here is what the rule finds. if self.srv.transport == transports.TRANSPORT.PROXY: for addr in added: - if str(addr.proto or self.srv.proto) != "udp": + if L4.l4_of(addr.proto or self.srv.proto) != L4.UDP: + # A TCP address is fronted by the one shared listener, which + # recovers where each connection was headed from conntrack — so + # the only thing it can need told is that this address fronts a + # service somewhere else. An address that is simply the service + # needs nothing at all, which is every address until one says so. + moved = addr.target_port and addr.target_port != addr.port + word = ("tls" if (str(self.srv.proto) == L4.HTTP + and str(addr.edge) == L4.TLS) else "any") + onward = UPSTREAM.env(addr.upstream) + if moved or word != "any" or onward != "same": + # One announcement per address, and an interface stands for + # every address it carries: the engine keys this map on what + # `SO_ORIGINAL_DST` hands back, which is never an interface + # name. Skipped instead, an HTTPS edge added on `eth0:443` + # was never announced and the engine dialled :443 instead of + # the service. + for host in interface_addresses(addr.ip_int): + await self.transport.publish( + (host, addr.port), + word, + onward, + (host, addr.target_port) if moved else None, + ) continue try: host = udp_relay_host(addr.ip_int) except NoRelayAddress as e: raise transports.UnsupportedChain(str(e)) from e - await self.transport.add_udp_target(host, addr.port) + # The relay's upstream *is* the answer to the same question: what is + # bound here forwards to where the service is, which is the target + # when there is one and the address itself when there is not. + await self.transport.add_udp_target( + host, addr.target_port or addr.port, UPSTREAM.env(addr.upstream) + ) nft.add(self.srv, added, **self._steer) self.log.add( LEVEL.INFO, @@ -448,6 +483,20 @@ class ServiceManager: async with self.lock: if self.active: nft.delete(self.srv, gone) + # And the engine's own note of where this address fronted, so its + # map cannot come to disagree with the rules that feed it. + for addr in gone: + if (self.srv.transport == transports.TRANSPORT.PROXY + and L4.l4_of(addr.proto or self.srv.proto) == L4.TCP): + # Every address it was announced under, or the engine keeps + # fronting one this service no longer protects. + for host in interface_addresses(addr.ip_int): + try: + await self.transport.withdraw((host, addr.port)) + except Exception as e: + self.log.add( + LEVEL.WARNING, + f"could not withdraw {addr.ip_int}: {e}") if self.active: self.log.add(LEVEL.INFO, f"no longer protecting {gone[0].ip_int}:{gone[0].port}") # Dropped from the list here rather than re-read from the database, because the diff --git a/backend/modules/services/models.py b/backend/modules/services/models.py index 75db77e..c7d0746 100644 --- a/backend/modules/services/models.py +++ b/backend/modules/services/models.py @@ -12,6 +12,7 @@ that can host it, and the service does not have to be recreated to change either """ import base64 +import os import uuid from utils import is_ip_parse @@ -59,13 +60,60 @@ class L4: UDP = "udp" #: TLS over TCP, terminated by the engine. TLS = "tls" + #: QUIC, terminated by the engine. UDP on the wire, TLS 1.3 inside it, and streams + #: inside that — which is the only reason a filter has anything to look at: once past + #: the Initial packet, QUIC encrypts the frames and the stream boundaries along with + #: the payload, so a layer that forwards has nothing to show anybody. + QUIC = "quic" + #: Every version of HTTP, on whichever transport each address carries it. + #: + #: The odd one out, and deliberately so. The other four say what is on the wire; + #: this one says what the *service* is, and lets each address say how it is reached + #: — `:80` in the clear, `:443` under TLS, `:443/udp` under QUIC. One service, one + #: chain, one certificate, and HTTP/1.1, HTTP/2 and HTTP/3 all rendered to the + #: filters as the same HTTP/1.1. + #: + #: It exists because the alternative was two or three services with hand-copied + #: filter chains: HTTP/1.1 and HTTP/2 live on `tls` (TCP) and HTTP/3 on `quic` + #: (UDP), and a chain that has to be kept in step by hand is a chain that stops + #: protecting one of them silently — the same argument that put several addresses + #: under one service in the first place. + #: + #: Nothing about the *kernel* becomes ambiguous, because nothing here is asked of + #: the kernel: each address still carries its own `tcp` or `udp`, which is what the + #: rules match and what `(ip, port, proto)` is keyed on. What the connection turns + #: out to be — TLS or not, HTTP/2 or not — is decided by the engine per connection, + #: from what the client actually sent. + HTTP = "http" - ALL = (TCP, UDP, TLS) + ALL = (TCP, UDP, TLS, QUIC, HTTP) @staticmethod def l4_of(proto: str) -> str: - """The transport a rule has to match to catch this service's traffic.""" - return L4.UDP if str(proto) == L4.UDP else L4.TCP + """The transport a rule has to match to catch this service's traffic. + + Two of the four are the other two wearing a hat: `tls` is TCP on the wire and + `quic` is UDP. That is what keeps `(ip, port, proto)` a key meaning what it says + — without it a TLS service and a TCP one, or a QUIC service and a UDP one, could + each claim one `ip:port` between them, which on the wire is the same port twice. + + `http` answers `tcp` here, which is the right *default* for an address that does + not say otherwise and never the whole answer: an `http` service's addresses each + carry their own, because it is the one protocol whose addresses are not all on + the same transport. Ask `Service.carries` rather than the service's protocol + whenever the question is about the kernel. + """ + return L4.UDP if str(proto) in (L4.UDP, L4.QUIC) else L4.TCP + + @staticmethod + def edges_of(proto: str) -> tuple[str, ...]: + """Which transports an address of this service may be reached on. + + One for every protocol but `http`, which is the point of `http`. + """ + if str(proto) == L4.HTTP: + return (L4.TCP, L4.UDP) + return (L4.l4_of(proto),) class PROTO: @@ -131,6 +179,44 @@ class MODE: ALL = (CLIENT_TO_SERVER, SERVER_TO_CLIENT, BOTH) +class UPSTREAM: + """What the service *behind one address* speaks. + + The engine used to have one answer and never asked: whatever the client spoke, the + service was assumed to speak too — a TLS connection was re-encrypted on the way out, + a QUIC one re-encoded as QUIC. That is right whenever firegex is carrying somebody + else's encryption, and wrong in the case an operator most often has, which is a + service that speaks neither and never will. + + So it is three answers now, and they are about the *service*, not about the client: + + * `same` — it speaks what the client spoke. The default, and what every service did + before there was a choice; + * `tcp` — it answers in the clear. Firegex terminates the client's TLS or QUIC and + forwards the plaintext, which makes firegex the thing that *adds* the encryption + rather than the thing that carries it; + * `tls` — it speaks TLS, whatever the client used to get here. On a TLS service that + is `same` by another name; under QUIC it is the one that matters, because it is + how an HTTPS-only service is reached from an HTTP/3 client. + + Under QUIC the last two are not a pass-through: HTTP/3 has no cleartext form, so what + reaches the service is the HTTP/1.1 the filters were already being shown. That is the + one place in the engine where what leaves is not the version that arrived, and it is + only ever the operator's explicit choice. + """ + + SAME = "same" + TCP = "tcp" + TLS = "tls" + + ALL = (SAME, TCP, TLS) + + @staticmethod + def env(upstream: str) -> str: + """What the engine is told. One variable, because it is one question.""" + return {UPSTREAM.TCP: "plain", UPSTREAM.TLS: "tls"}.get(str(upstream), "same") + + class Address: """One address a service is protected on. @@ -141,10 +227,19 @@ class Address: filter chains in step by hand, and a chain that drifted was a chain that stopped protecting one of them silently. - The transport protocol is carried here as well as on the service, and is always the - service's own. Denormalised on purpose: it is what makes `(ip, port, proto)` a - usable uniqueness key, so a TCP service and a UDP one can share an address the way - the kernel lets them. + The transport protocol is carried here as well as on the service. Denormalised on + purpose: it is what makes `(ip, port, proto)` a usable uniqueness key, so a TCP + service and a UDP one can share an address the way the kernel lets them. It is + derived — `L4.l4_of(edge)` — because `tls` is TCP on the wire and `quic` is UDP, and + what the kernel matches is a smaller question than what the address carries. + + Two things an address says for itself, and both are why it says anything at all: + **what is spoken at it** (`edge`) and **where that traffic goes** (`target_port`). + Firegex was transparent and only transparent — the client dialled the service's own + address and the engine dialled it back as the client — which meant a service could + only be protected where it already listened. One service answering in the clear on + `:80`, reached over TLS on `:443` and over HTTP/3 on a UDP port, is three rows + pointing at one port. """ def __init__( @@ -154,6 +249,9 @@ class Address: ip_int: str, port: int, proto: str = L4.TCP, + edge: str | None = None, + target_port: int | None = None, + upstream: str = UPSTREAM.SAME, proxy_ip: str | None = None, proxy_port: int | None = None, **other, @@ -163,6 +261,27 @@ class Address: self.ip_int = ip_int self.port = port self.proto = proto + #: What clients speak *at this address*: one of `tcp`, `tls`, `udp`, `quic` — + #: the same words a service speaks, minus `http`, which is the one that means + #: "ask the addresses". For every protocol but that one it is the service's own + #: and the operator never sees it; on an `http` service it is the choice, and it + #: is what decides whether this address needs a certificate behind it and which + #: transport the kernel matches. + self.edge = str(edge or proto) + #: Where the traffic that arrives here is actually sent, when that is not the + #: port it arrived on. `None` is the transparent case and the default: firegex + #: dials the address the client dialled, as the client, and the service sees + #: exactly what it would have seen without any of this. + #: + #: Set, this address is a *publication*: one service answered on `:80` in the + #: clear and reached on `:443` under TLS, or on a QUIC port, without moving. The + #: address is where the world knocks; this is where the service is. + self.target_port = int(target_port) if target_port else None + #: What the service *behind this address* speaks. See [`UPSTREAM`]. It belongs to + #: the address rather than to the service because that is where the question has + #: an answer: one daemon is reached over TLS on one port and in the clear on + #: another, and what firegex does on the way out is a property of the way in. + self.upstream = str(upstream or UPSTREAM.SAME) #: Where the operator's own proxy is listening for *this* address. Only for #: `external`, and per address rather than per service: the return rule rewrites #: the source port back to the original one, so two addresses handed to the same @@ -181,6 +300,9 @@ class Address: "ip_int": self.ip_int, "port": self.port, "proto": self.proto, + "edge": self.edge, + "target_port": self.target_port, + "upstream": self.upstream, "proxy_ip": self.proxy_ip, "proxy_port": self.proxy_port, } @@ -197,6 +319,58 @@ class Address: return f"
" +def quic_alpn() -> tuple[str, ...]: + """What this instance offers a QUIC service's clients, in order. + + Instance-wide (`FGEX_PROXY_QUIC_ALPN`) rather than per service, because nothing has + yet wanted a column for it: `h3` is what a QUIC service speaks nine times in ten, and + an instance in front of something else sets it in its own environment the way the + filter deadline beside it is set. Read here as well as where the engine is launched + so that the one question it decides — whether an exchange can be rendered at all — + has a single answer. + """ + raw = os.getenv("FGEX_PROXY_QUIC_ALPN", "h3") + listed = tuple(p.strip() for p in raw.split(",") if p.strip()) + return listed or ("h3",) + + +def upstream_refusal(proto: str, upstream: str, edge_is_udp: bool) -> str | None: + """Why one address cannot be forwarded the way it was told to, or `None`. + + Asked of an **address** rather than of the service, because that is where it is + chosen: what is behind `:443` is the same daemon as what is behind `:80`, but the two + reach it differently — one may be re-encrypted and the other handed the plaintext, and + on an `http` service the two are separate ports with separate answers. Asked from the + three places that would otherwise drift: when an address is created, when one is + added, and in `ProxyTransport.check`. + + One thing is impossible, and it is impossible rather than unimplemented: a QUIC edge + carrying something that is **not HTTP/3**. What lets an HTTP/3 client reach a service + that speaks HTTP/1.1 is that the exchange is already rendered as HTTP/1.1 for the + chain, so sending that rendering on invents nothing. Opaque bytes on a QUIC stream + have no such form — there is no HTTP/1.1 spelling of "some bytes" — so an address + whose clients speak anything else has nothing that could be forwarded. + + Which is a fact about the **ALPN**, not about the protocol: a `quic` service offering + `h3`, which is the default and the usual case, is exactly as renderable as an `http` + one, and an instance that has set `FGEX_PROXY_QUIC_ALPN` to something else is the one + that cannot do this — on both. + """ + if str(upstream) == UPSTREAM.SAME: + return None + if str(proto) not in (L4.TLS, L4.QUIC, L4.HTTP): + return None + alpn = quic_alpn() + if edge_is_udp and "h3" not in alpn: + return ( + f"This instance offers QUIC clients {', '.join(alpn)}, and only HTTP/3 can be " + f"rendered to a service that speaks HTTP/1.1 — anything else on a QUIC stream " + f"is opaque bytes with no HTTP/1.1 form to forward. Let this address be " + f"forwarded as it arrives, or take its HTTP/3 edge away." + ) + return None + + class Service: """One protected endpoint and the transport in front of it.""" @@ -240,7 +414,7 @@ class Service: return cls(**var) @property - def terminates_tls(self) -> bool: + def decrypts(self) -> bool: """Whether the engine decrypts this service's traffic before filtering it. Read off what the service speaks rather than held beside it. Only the proxy layer @@ -248,14 +422,50 @@ class Service: layer does. It was once orthogonal to the layer, which took an nginx in front of NFQUEUE to arrange — so the connection was terminated anyway, and the one property that layer has over the other was already gone. + + Named for what it does rather than for TLS, because two protocols do it now and + QUIC is the one with no alternative: TLS over TCP can at least be carried past + unopened, while a QUIC packet past the handshake has its frames encrypted too. + Both need the same thing from the operator, which is a certificate. + + `http` is in the list because a certificate is the whole of what it adds: a + cleartext HTTP service is already `tcp`, and HTTP/2 in the clear is recognised + there from the connection preface whatever the service is called. What `http` + buys over that is the TLS and the QUIC edges, and neither exists without one. """ - return str(self.proto) == L4.TLS + return str(self.proto) in (L4.TLS, L4.QUIC, L4.HTTP) @property def l4(self) -> str: - """The transport the kernel sees, which is what a rule has to match.""" + """The transport the kernel sees, which is what a rule has to match. + + Only meaningful where a service has one. An `http` service is reached on both, + and everything that used to ask this of such a service asks `carries` instead: + the answer here is its addresses' default, not a claim about all of them. + """ return L4.l4_of(self.proto) + def carries(self, l4: str) -> bool: + """Whether any of this service's addresses is reached over `l4`. + + The question every rule, listener and relay actually has, and the one that + stopped having a single answer when `http` arrived. A service with no addresses + yet is answered from its protocol, so a check made at creation — before there is + an address to look at — still means something. + """ + if not self.addresses: + return l4 in L4.edges_of(self.proto) + return any(L4.l4_of(addr.proto or self.proto) == l4 for addr in self.addresses) + + @property + def udp_addresses(self) -> list: + """The addresses that need a relay of their own: datagrams, and QUIC.""" + return [ + addr + for addr in self.addresses + if L4.l4_of(addr.proto or self.proto) == L4.UDP + ] + @property def has_ipv6(self) -> bool: """Whether any of its addresses is IPv6. @@ -265,6 +475,20 @@ class Service: """ return any(addr.is_ipv6 for addr in self.addresses) + @property + def has_ipv6_tcp(self) -> bool: + """Whether the *TCP* listener needs to be able to accept IPv6. + + The narrower question, and the one worth asking before rebuilding a service: the + only listener that can have its family wrong is the TCP one. Everything relayed + per address — datagrams, and QUIC — binds a socket in the family of the address + when it arrives. On an `http` service the two sit side by side, so an IPv6 HTTP/3 + address must not cost every TCP connection on the service a restart it did not + need. + """ + udp = {addr.id for addr in self.udp_addresses} + return any(addr.is_ipv6 for addr in self.addresses if addr.id not in udp) + def to_dict(self) -> dict: # The certificate goes out, the key never does: it is write-only from the # API's point of view, and a service listing is the easiest place to leak it. diff --git a/backend/modules/services/nftables.py b/backend/modules/services/nftables.py index 18b6d75..f1dda22 100644 --- a/backend/modules/services/nftables.py +++ b/backend/modules/services/nftables.py @@ -48,6 +48,29 @@ QUEUE_MARK_OUTPUT = 0x1338 #: growing except that every position costs two chains whether or not anyone uses it. MAX_CHAIN_POSITIONS = 8 +#: Where the proxy layer's redirect sits in the prerouting and output hooks. +#: +#: **Before `dstnat` (-100), and that is the whole point.** Docker and podman publish a +#: container's port by appending their own chain to `nat PREROUTING`, which `iptables-nft` +#: registers at exactly `dstnat` — the priority this used to share. Two base chains at one +#: priority in two tables are ordered by neither of them, and measured on a host where the +#: DNAT went first: it rewrote the destination to the container's address, firegex's rule +#: (which matches the *published* address) stopped matching, its counter stayed at zero, +#: and traffic carrying a blocked pattern was answered by the container. The service still +#: read `ACTIVE`. Protection that disappears without saying so is the worst failure this +#: module has, and a race nobody arbitrates is not a thing to leave to luck. +#: +#: So the redirect runs first and the published port is captured before anybody can +#: rewrite it — which is the property the NFQUEUE layer already had for free by living at +#: `raw`, three hundred below any of this. The engine's own dial is not caught by the +#: rule it would otherwise match (`SELF_MARK`), so whatever DNAT the container runtime +#: installs still does its job on the way out, exactly as before. +#: +#: The floor is conntrack at -200: a `nat` chain registered before it has no connection to +#: attach a translation to. Anything between the two behaves identically; this one is far +#: enough from both to leave room. +NAT_PRIORITY = -120 + # Shared by the divert rule and the return rule; policy routing keys off it. PROXY_MARK = 0x1339 # Routing table holding the `local` default that makes marked packets be delivered @@ -65,6 +88,13 @@ PROXY_SELF_MARK = 0x133A def resolve_target(srv: Service, addr: Address) -> tuple[str, int] | None: """Where one address's rules should point: at the address itself. + This is the address the **world knocks on**, which is what a rule has to match, and + it is not always where the service is — see [`service_at`]. The two were one answer + for as long as firegex was transparent and only transparent, and making this one + return the other quietly redirected the service's own port twice and left the + published one unprotected, which is a service that looks configured and answers + nothing. + It used to be somewhere else for a TLS service. nginx terminated the connection on one derived loopback port and re-encrypted from a second, and the rules were pointed at the plaintext leg in between — which is why protecting one address cost two ports @@ -79,6 +109,23 @@ def resolve_target(srv: Service, addr: Address) -> tuple[str, int] | None: return addr.ip_int, addr.port +def service_at(srv: Service, addr: Address) -> tuple[str, int] | None: + """Where the service reached through one address actually is. + + The address itself, unless that address is a **publication**: one daemon answering + HTTP/1.1 in the clear on `:80` can be reached over TLS on `:443` and over HTTP/3 on a + UDP port, all three sent back to the port it already listens on. Firegex is then not + transparent on those addresses, which is the point of them and the reason it is said + per address rather than per service. + + Everything that *dials* asks this — the UDP and QUIC relays, whose upstream is fixed + when they are bound, and the TCP listener, which is told through `FGEX_PROXY_TARGETS` + because one listener fronts every address and recovers each connection's destination + from conntrack. Everything that *matches* asks [`resolve_target`]. + """ + return addr.ip_int, (addr.target_port or addr.port) + + def one_address(ip: str) -> str: """Strip any prefix: a rewrite target is one host, never a network. @@ -118,6 +165,28 @@ def udp_relay_host(ip: str) -> str: return ordered[0] +#: Written on the output-hook rules an interface address installs, so they can be found +#: again. There is nothing in the packet that names the interface there — see +#: `interface_addresses` — so the rule says which interface it belongs to itself. +IFACE_COMMENT = "iface " + + +def interface_addresses(ip: str) -> list[str]: + """Every address an interface carries right now, as a rule can match them. + + Link-local is left out for the reason `udp_relay_host` takes it last: an interface + almost always carries an `fe80::` it was never configured with, and nothing dials a + service there. + """ + if is_ip_parse(ip): + return [one_address(ip)] + return [ + one_address(addr.split("%")[0]) + for addr in get_interface_ips(ip) + if not addr.lower().startswith(("fe80:", "169.254.")) + ] + + def udp_relay_key(ip: str, port: int) -> str: """How a UDP relay's upstream is spelled, everywhere it is spelled. @@ -275,10 +344,12 @@ class FiregexTables(NFTableManager): } } }, - _chain(self.nat_chain, "nat", "prerouting", -100), + _chain(self.nat_chain, "nat", "prerouting", NAT_PRIORITY), # Locally generated traffic never reaches the nat prerouting hook, and # a client on this host protecting a service on this host is exactly that. - _chain(self.nat_output_chain, "nat", "output", -100), + # Same priority for the same reason: a container runtime publishes a port + # in `nat OUTPUT` as well, so the race is the same one on this hook. + _chain(self.nat_output_chain, "nat", "output", NAT_PRIORITY), # `route`, so changing the mark forces the packet to be re-routed. _chain(self.route_chain, "route", "output", -150), # Installed once. A packet on the intercepted (redirected) connection @@ -405,7 +476,8 @@ class FiregexTables(NFTableManager): }, ] - def _rule(self, chain: str, expr: list, insert: bool = False) -> dict: + def _rule(self, chain: str, expr: list, insert: bool = False, + comment: str | None = None) -> dict: verb = "insert" if insert else "add" return { verb: { @@ -414,6 +486,7 @@ class FiregexTables(NFTableManager): "table": self.table_name, "chain": chain, "expr": expr, + **({"comment": comment} if comment else {}), } } } @@ -455,7 +528,11 @@ class FiregexTables(NFTableManager): # at that address's port rather than at the one shared TCP listener. port = proxy_port if l4 == "udp": - key = udp_relay_key(udp_relay_host(target_ip), target_port) + # Keyed by where the relay *sends*, which is how the engine reported + # it: a published address and the service behind it are two ports, + # and the relay is filed under the second. + service_ip, service_port = service_at(srv, addr) + key = udp_relay_key(udp_relay_host(service_ip), service_port) port = (udp_ports or {}).get(key) if port is None: raise Exception( @@ -559,21 +636,65 @@ class FiregexTables(NFTableManager): # a service on it. Conntrack NATs a connection once, so a rule in each is not a # double translation. is_iface = not is_ip_parse(ip) + # **The match is the interface, and nothing narrower.** Pinning the destination to + # the addresses the interface carries was tried, to stop a service protecting + # `docker0:443` from also taking every container's outbound HTTPS — which it does, + # measured, terminating it with the service's certificate. It was reverted: an + # interface name exists precisely for the case where **the address is not the + # operator's to know**, which includes firegex sitting in front of a service on + # another machine reached through that link, and pinning to this host's own + # addresses quietly removes that. Protecting an interface protects what crosses + # it; naming a link that carries other people's traffic is a choice, and the + # answer to not wanting it is a narrower address, not a narrower rule. redirect = ( self._not_ours() + self._match(ip, port, family, l4, "daddr", "dport") + [self.COUNTER, {"redirect": {"port": int(proxy_port)}}] ) - cmds = [ - self._rule(self.nat_chain, redirect), + cmds = [self._rule(self.nat_chain, redirect)] + cmds.append( self._rule( self.route_chain, self._match(ip, port, family, l4, "saddr", "sport") + [{"mangle": {"key": {"meta": {"key": "mark"}}, "value": PROXY_MARK}}], - ), - ] + ) + ) if not is_iface: cmds.append(self._rule(self.nat_output_chain, redirect)) + else: + # An interface address has to be caught in the output hook too, and cannot be + # caught the same way. `iifname` means nothing for a packet this host is + # generating, and `oifname` is the wrong question twice over: `oifname eth0 + # dport 80` would mean *we* are calling somebody else's port 80, which is the + # host's own outbound traffic and none of firegex's business — and a + # connection from this host to one of its **own** addresses is routed through + # `lo` anyway, so it would never match. + # + # So the destination is matched instead, against the addresses that interface + # carries. That is exactly "this host calling a service it protects", and it + # is what an operator means when they protect `lo` and then curl `127.0.0.1`. + # Without it the service was intercepted for the outside world and silently + # not for anything on the box, which is the shape of an interception canary + # failing and is how this was reported. + # + # The addresses are resolved now rather than stored, and that is the cost: + # one the interface gains later is not covered *for traffic this host + # generates* until the service restarts. Inbound keeps matching `iifname`, so + # it follows the interface as it always did. `udp_relay_host` pays the same + # price for the same reason. + # + # The rule carries the interface's name as a comment because nothing else in + # it does. `delete()` finds a rule by what it matches, and these match an + # address; without the comment they would be left behind on every stop — a + # redirect to a port nobody is listening on any more. + for addr in interface_addresses(ip): + cmds.append(self._rule( + self.nat_output_chain, + self._not_ours() + + self._match(addr, port, ip_family(addr), l4, "daddr", "dport") + + [self.COUNTER, {"redirect": {"port": int(proxy_port)}}], + comment=f"{IFACE_COMMENT}{ip}", + )) self.cmd(*cmds) # --- reading back --------------------------------------------------------- @@ -611,6 +732,14 @@ class FiregexTables(NFTableManager): ip_int = str(expr[0]["match"]["right"]) else: continue + # An output-hook rule installed for an interface matches one of that + # interface's addresses, so what it matches does not say whose it is. + # The comment does, and reading it back here is what lets the rest of + # this file go on treating the rule as the interface's — including + # `delete()`, which would otherwise never find it. + note = str(rule.get("comment") or "") + if note.startswith(IFACE_COMMENT): + ip_int = note[len(IFACE_COMMENT):] counter = next( (e["counter"] for e in expr if isinstance(e, dict) and "counter" in e), {}, diff --git a/backend/modules/services/pyworker.py b/backend/modules/services/pyworker.py index 5605a02..48d2d5e 100644 --- a/backend/modules/services/pyworker.py +++ b/backend/modules/services/pyworker.py @@ -303,9 +303,14 @@ class Filters: "data": data, "is_input": is_input, "is_ipv6": bool(meta.get("is_ipv6", False)), - # Said by the engine rather than assumed: this worker serves TCP connections - # and UDP flows alike, and the library's stream and HTTP models key off it. + # Said by the engine rather than assumed: this worker serves TCP connections, + # UDP flows and QUIC streams alike. Two words rather than one, because they + # answer different questions — `is_tcp` is what is on the wire, `l4` is what + # the library's stream and HTTP models are built on, and QUIC answers those + # two differently. Absent, `l4` derives from the other: the NFQUEUE engines + # carry TCP and UDP and have nothing else to say. "is_tcp": bool(meta.get("is_tcp", True)), + "l4": meta.get("l4"), "src_ip": client_ip if is_input else server_ip, "src_port": client_port if is_input else server_port, "dst_ip": server_ip if is_input else client_ip, diff --git a/backend/modules/services/transports.py b/backend/modules/services/transports.py index 6be7452..3bfca02 100644 --- a/backend/modules/services/transports.py +++ b/backend/modules/services/transports.py @@ -24,9 +24,11 @@ import os import sys import traceback -from modules.services.models import KIND, L4, PROTO, TRANSPORT, Filter, Regex +from modules.services.models import (KIND, L4, PROTO, TRANSPORT, UPSTREAM, Filter, + Regex, quic_alpn, upstream_refusal) from modules.services.nftables import (MAX_CHAIN_POSITIONS, NoRelayAddress, - resolve_target, udp_relay_host, udp_relay_key) + interface_addresses, one_address, service_at, + udp_relay_host, udp_relay_key) from utils import DEBUG, nicenessify MODULES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") @@ -176,7 +178,8 @@ class Transport: async def reload(self, chain: list[ChainLink]) -> None: raise NotImplementedError - async def add_udp_target(self, ip: str, port: int) -> int: + async def add_udp_target(self, ip: str, port: int, + onward: str = "same") -> int: raise NotImplementedError async def stop(self) -> None: @@ -532,7 +535,20 @@ class NfqueueTransport(Transport): @classmethod def check(cls, srv, chain: list[ChainLink]) -> None: super().check(srv, chain) - if srv.terminates_tls: + if str(srv.proto) == L4.HTTP: + # Not the TLS sentence and not the QUIC one, because the reason is neither: + # an `http` service is reached over TLS and over QUIC *and* in the clear, and + # this layer can decrypt none of it. Even the cleartext edge would be half an + # answer — HTTP/2 in the clear is HPACK, and rendering it means terminating + # the connection, which is the proxy layer's job by definition. + raise UnsupportedChain( + "An `http` service is reached over every version of HTTP at once, which " + "means TLS on one port and QUIC on another — and this layer inspects " + "packets on their way past rather than terminating them, so it can read " + "none of it. Move the service to the proxy layer, or set its protocol to " + "TCP and filter the HTTP/1.1 going by." + ) + if srv.decrypts: # Decrypting means terminating the connection, which is what the proxy layer # does; this one inspects packets on their way past and hands the kernel a # verdict, which is the whole reason it can fail open. It was once offered @@ -540,6 +556,18 @@ class NfqueueTransport(Transport): # filters attached to the plaintext leg in between — so the connection *was* # terminated, the layer's one distinguishing property was already gone, and # it cost two loopback ports chosen by hashing the address. + if str(srv.proto) == L4.QUIC: + # And for QUIC there is not even that: nginx could terminate TLS and hand + # over a TCP stream, but a QUIC connection past its first packet has its + # frames and its stream boundaries encrypted as well as its payload. + # Queued to userspace it is a UDP datagram of noise. + raise UnsupportedChain( + "QUIC encrypts its frames and its stream boundaries, not just the " + "payload, so a packet queued to this layer carries nothing a filter " + "could read. It has to be terminated to be inspected at all: move " + "the service to the proxy layer, or set its protocol to UDP and " + "accept that the rules are matching the encrypted datagrams." + ) raise UnsupportedChain( "A service that speaks TLS has to be decrypted, and decrypting means " "terminating the connection — which is what the proxy layer does and " @@ -636,9 +664,20 @@ class ProxyTransport(Transport): @classmethod def check(cls, srv, chain: list[ChainLink]) -> None: super().check(srv, chain) - if str(srv.proto) == L4.UDP: + # Asked here as well as where each was set, because an address added afterwards + # can be one this instance cannot honour. + for addr in srv.addresses: + refusal = upstream_refusal( + srv.proto, addr.upstream, L4.l4_of(addr.edge) == L4.UDP + ) + if refusal: + raise UnsupportedChain(refusal) + if srv.carries(L4.UDP): from utils import get_interface_ips, is_ip_parse - for addr in srv.addresses: + # Only the addresses that need a relay of their own. On an `http` service + # those are its QUIC ones, sitting beside TCP addresses that need nothing of + # the sort — which is why this asks the address rather than the service. + for addr in srv.udp_addresses: if not is_ip_parse(addr.ip_int): ips = get_interface_ips(addr.ip_int) if not ips: @@ -658,23 +697,79 @@ class ProxyTransport(Transport): #: headed from conntrack, which UDP cannot do — see `udp.rs`. self.udp_ports: dict[str, int] = {} - def _udp_targets(self) -> list[tuple[str, int]]: + #: What an address's edge is called on the wire to the engine. Only `tls` has a name + #: of its own, because only it is a promise: this port is the encrypted one, and a + #: client opening it in the clear is refused. Everything else is "whatever the client + #: turns out to be speaking", which is what one listener fronting both a cleartext + #: port and an encrypted one has always had to do — and refusing TLS at a port nobody + #: promised would be a rule with nothing behind it. + _EDGE_WORD = {L4.TLS: "tls"} + + def _published(self) -> list[str]: + """What this engine is told about its TCP addresses, one entry each. + + `10.0.0.1:443|tls|plain=10.0.0.1:80` reads as: what arrives at the first is + spoken under TLS, the service behind it answers in the clear, and it is at the + second. Every part after the address is optional and an address that needs none + of them is not listed at all — which is every address of every service until an + operator says otherwise, and is what keeps the transparent case exactly as it + was. + + Only TCP: a UDP or QUIC address has a relay of its own, bound to the service it + fronts and terminating what it was built to terminate, so both questions are + already answered by `FGEX_PROXY_UDP`. + + **An interface is resolved to the addresses it carries, one entry each**, rather + than skipped. The engine keys this map on what `SO_ORIGINAL_DST` hands back, + which is an address and never an interface — so there was nothing to key on and + the row was dropped. What that cost: an HTTPS edge declared on `lo:443` was never + announced, so the engine met the connection with no idea it was a TLS edge or + that the service was on `:80`, terminated it as an ordinary one and dialled + `:443`, where nothing listens. HTTPS on that address simply did not answer, and + the same address written `127.0.0.1:443` worked — which is how it was found. + """ + out = [] + for addr in self.srv.addresses: + if L4.l4_of(addr.edge) != L4.TCP: + continue + # Said only where the service is reached in more than one way, because that + # is the only place it is a choice rather than a restatement of the service. + word = (self._EDGE_WORD.get(str(addr.edge), "any") + if str(self.srv.proto) == L4.HTTP else "any") + moved = addr.target_port and addr.target_port != addr.port + onward = UPSTREAM.env(addr.upstream) + # An address with nothing to say is not listed: the engine's answer for one + # it has never heard of is the transparent case, which is what it would be + # told. All three have to be silent for that, and the one that was forgotten + # here — what the service behind speaks — is the one with no other way in. + if word == "any" and not moved and onward == "same": + continue + for host in interface_addresses(addr.ip_int): + entry = f"{udp_relay_key(host, addr.port)}|{word}|{onward}" + if moved: + entry += f"={udp_relay_key(host, addr.target_port)}" + out.append(entry) + return out + + def _udp_targets(self) -> list[tuple[str, int, str]]: """The addresses that each need a relay of their own, resolved. Empty for TCP. Resolved here and nowhere else, so the relay the engine is asked to open, the rule that points at it and the key both are filed under are derived from one answer rather than from three lookups that could each land somewhere different. + + The third item is what the service behind *that relay* speaks, because a relay is + one protected address and the question belongs to the address. """ - if str(self.srv.proto) != L4.UDP: - return [] out = [] - for addr in self.srv.addresses: - target = resolve_target(self.srv, addr) + for addr in self.srv.udp_addresses: + target = service_at(self.srv, addr) if not target: continue target_ip, target_port = target try: - out.append((udp_relay_host(target_ip), target_port)) + out.append((udp_relay_host(target_ip), target_port, + UPSTREAM.env(addr.upstream))) except NoRelayAddress as e: raise UnsupportedChain( f"{e}. Give the service an IP address, or put it on the NFQUEUE " @@ -682,18 +777,25 @@ class ProxyTransport(Transport): ) from e return out - def _tls_env(self) -> dict: + def _crypto_env(self) -> dict: """Hand the engine its certificate, or nothing at all. The engine reads **paths**, not the material: a certificate on a command line or in an environment dump is a certificate in a log somewhere. They are written `0600` under a directory of the service's own, and removed when it stops. - `FGEX_PROXY_TLS_UPSTREAM` is set with them, because a service behind TLS speaks - TLS: nginx used to re-encrypt on the way out with `proxy_ssl on`, and the engine - does the same thing in the same place. + What the service *behind* speaks does not travel here: it is the address's + answer, not the service's, so it rides with the address — on `FGEX_PROXY_TARGETS` + and `PUBLISH` for a TCP one, and on `FGEX_PROXY_UDP` and `ADD_UDP` for a relay. + `same` re-encrypts on the way out the way nginx's `proxy_ssl on` did; the other + two say the service speaks HTTP/1.1, in the clear or under its own TLS. + + QUIC takes the same pair and one more word. It needs no `_UPSTREAM` flag because + there is no unencrypted QUIC to choose between — the handshake is part of the + transport — and it needs `FGEX_PROXY_QUIC` because what binds a protected + address is then an endpoint that terminates rather than a socket that forwards. """ - if not self.srv.terminates_tls: + if not self.srv.decrypts: return {} os.makedirs(TLS_DIR, exist_ok=True) cert_path = os.path.join(TLS_DIR, f"{self.srv.id}.crt") @@ -704,13 +806,42 @@ class ProxyTransport(Transport): fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "w") as handle: handle.write(material or "") - return { + material = { "FGEX_PROXY_TLS_CERT": cert_path, "FGEX_PROXY_TLS_KEY": key_path, - "FGEX_PROXY_TLS_UPSTREAM": "1", } + if str(self.srv.proto) == L4.HTTP: + # Every edge at once, in one process, with one certificate and one chain. + # `TLS_OPTIONAL` is the whole of what makes that possible: the TCP listener + # terminates TLS only for the connections that start a handshake, so the + # cleartext port and the encrypted one are the same listener and the same + # filters. `QUIC` is set whether or not the service has a UDP address yet, + # so that adding one later opens a relay on the engine already running rather + # than needing it restarted. + return { + **material, + "FGEX_PROXY_TLS": "1", + "FGEX_PROXY_TLS_OPTIONAL": "1", + "FGEX_PROXY_QUIC": "1", + "FGEX_PROXY_QUIC_ALPN": ",".join(quic_alpn()), + } + if str(self.srv.proto) == L4.QUIC: + return { + **material, + "FGEX_PROXY_QUIC": "1", + # What the engine offers the service, in this order. It cannot be asked + # of the client the way the TLS path asks: a QUIC ClientHello arrives + # inside an encrypted Initial whose processing *is* the handshake, so + # there is nothing to hold it at and nothing to read before answering. + # `h3` is what a QUIC service speaks nine times in ten; an instance in + # front of something else sets this in its own environment, the same way + # the filter deadline beside it is set. Per service it would be a column, + # and nothing yet has wanted one. + "FGEX_PROXY_QUIC_ALPN": ",".join(quic_alpn()), + } + return material - def _clear_tls_material(self) -> None: + def _clear_crypto_material(self) -> None: for suffix in (".crt", ".key"): try: os.remove(os.path.join(TLS_DIR, f"{self.srv.id}{suffix}")) @@ -743,14 +874,22 @@ class ProxyTransport(Transport): # The same knob the NFQUEUE binaries get. `run.py --threads` used to # reach them and not this engine, so one flag meant two things. "NTHREADS": os.getenv("NTHREADS", "1"), - # UDP gets one relay per address, each with its upstream already - # known: `SO_ORIGINAL_DST` is TCP-only, so there is nothing to - # recover per datagram and nothing to recover it from. The client's - # address is not preserved on that path, which the operator is told - # before choosing it. + # One relay per address, each with its upstream already known: + # `SO_ORIGINAL_DST` is TCP-only, so there is nothing to recover per + # datagram and nothing to recover it from. The same list serves QUIC, + # which is UDP as far as the rules are concerned — what changes is + # what binds the port, and `_crypto_env` is where that is said. "FGEX_PROXY_UDP": ",".join( - udp_relay_key(ip, port) for ip, port in self._udp_targets() + f"{udp_relay_key(ip, port)}|{onward}" + for ip, port, onward in self._udp_targets() ), + # The addresses this service is *published* on rather than + # intercepted at: `dialled=service`, one pair per address that says + # where the service really is. Only TCP needs telling — a UDP or + # QUIC address has a relay of its own whose upstream is already the + # answer, which is the same list above. Empty is the transparent + # case, which is every address that did not say otherwise. + "FGEX_PROXY_TARGETS": ",".join(self._published()), # One number for both halves: TCP connections and UDP flows spend # the same descriptors, and two limits meaning "how much of that may # go" would be two things to keep in step. @@ -759,7 +898,7 @@ class ProxyTransport(Transport): if self.srv.over_limit_forwards else {}), "FGEX_PROXY_FIRST_BYTE_TIMEOUT": str(self.srv.first_byte_timeout), "FGEX_PROXY_FILTER_TIMEOUT_MS": os.getenv("FGEX_PROXY_FILTER_TIMEOUT_MS", "2000"), - **self._tls_env(), + **self._crypto_env(), }, ), ) @@ -868,9 +1007,12 @@ class ProxyTransport(Transport): # message rather than their protection. raise Exception(f"the proxy engine rejected the chain: {detail}") - async def add_udp_target(self, ip: str, port: int) -> int: + async def add_udp_target(self, ip: str, port: int, onward: str = "same") -> int: """Bind a dedicated UDP relay for a new address without restarting the engine. + `onward` is what the service behind *this* relay speaks, because a relay is one + protected address and that question belongs to the address. + Idempotent, and deliberately the only way in: the map of relays is this object's, so a caller that has just added an address asks for the relay and is handed the port, rather than keeping a second map of its own beside this one. @@ -885,7 +1027,7 @@ class ProxyTransport(Transport): if target in self.udp_ports: return self.udp_ports[target] self._ack = asyncio.get_running_loop().create_future() - self.process.stdin.write(f"ADD_UDP {target}\n".encode()) + self.process.stdin.write(f"ADD_UDP {target} {onward}\n".encode()) await self.process.stdin.drain() try: ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT) @@ -902,6 +1044,40 @@ class ProxyTransport(Transport): ) return relay_port + async def publish(self, public: tuple[str, int], edge: str, onward: str, + target: tuple[str, int] | None) -> None: + """Tell the running engine about one TCP address. + + The same thing `FGEX_PROXY_TARGETS` says at startup, said to an engine that is + already carrying traffic — because an address can be added to a running service + and the point of that is that nothing is dropped. + + UDP and QUIC addresses are not here: theirs is a relay with its upstream fixed + when it is bound, so `add_udp_target` has already answered both questions. + """ + where = f" {udp_relay_key(*target)}" if target else "" + await self._command(f"PUBLISH {udp_relay_key(*public)} {edge} {onward}{where}") + + async def withdraw(self, public: tuple[str, int]) -> None: + await self._command(f"WITHDRAW {udp_relay_key(*public)}") + + async def _command(self, line: str) -> None: + """One control line, and the acknowledgement it is owed.""" + if not self.process or self.process.returncode is not None: + raise Exception("the proxy engine is not running") + async with self._cmd_lock: + self._ack = asyncio.get_running_loop().create_future() + self.process.stdin.write(f"{line}\n".encode()) + await self.process.stdin.drain() + try: + ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT) + except asyncio.TimeoutError: + raise Exception(f"the proxy engine did not acknowledge {line.split()[0]}") + finally: + self._ack = None + if not ok: + raise Exception(f"the proxy engine rejected {line}: {detail}") + async def _read_events(self): try: while True: @@ -948,7 +1124,7 @@ class ProxyTransport(Transport): await self._kill() # After the engine is gone, never before: a key removed while the process that # reads it is still running would be a key removed from under a restart. - self._clear_tls_material() + self._clear_crypto_material() class ExternalTransport(Transport): @@ -993,11 +1169,14 @@ class ExternalTransport(Transport): f"Deactivate them, or move the service to the proxy transport, which " f"filters and can still forward." ) - if srv.terminates_tls: + if srv.decrypts: raise UnsupportedChain( - "Decrypting exists so that filters can see the plaintext, and this " - "service has no filters — nothing of firegex is in its path. Let your own " - "proxy speak TLS, or move the service to the proxy layer." + f"Decrypting exists so that filters can see the plaintext, and this " + f"service has no filters — nothing of firegex is in its path. Let your own " + f"proxy speak " + f"{'QUIC' if str(srv.proto) == L4.QUIC else 'HTTP itself' if str(srv.proto) == L4.HTTP else 'TLS'}" + f", or move " + f"the service to the proxy layer." ) async def start(self, chain: list[ChainLink]) -> dict: diff --git a/backend/proxysrc/Cargo.lock b/backend/proxysrc/Cargo.lock index 5b0af0e..585e9b7 100644 --- a/backend/proxysrc/Cargo.lock +++ b/backend/proxysrc/Cargo.lock @@ -2,12 +2,24 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "bytes" version = "1.12.1" @@ -30,17 +42,65 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "fgex-proxy" version = "0.1.0" dependencies = [ + "bytes", + "h2", + "h3", + "h3-quinn", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", "libc", + "quinn", "rcgen", "rustls", "rustls-pemfile", @@ -57,6 +117,100 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -64,8 +218,160 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h3" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10872b55cfb02a821b69dc7cf8dc6a71d6af25eb9a79662bec4a9d016056b3be" +dependencies = [ + "bytes", + "fastrand", + "futures-util", + "http", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "h3-quinn" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2e732c8d91a74731663ac8479ab505042fbf547b9a207213ab7fbcbfc4f8b4" +dependencies = [ + "bytes", + "futures", + "h3", + "quinn", + "tokio", + "tokio-util", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", ] [[package]] @@ -74,6 +380,17 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.189" @@ -86,6 +403,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "memchr" version = "2.8.3" @@ -146,6 +469,63 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -155,6 +535,38 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + [[package]] name = "rcgen" version = "0.13.2" @@ -176,12 +588,18 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustls" version = "0.23.43" @@ -212,6 +630,7 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ + "web-time", "zeroize", ] @@ -226,6 +645,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "serde" version = "1.0.229" @@ -275,6 +700,18 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + [[package]] name = "socket2" version = "0.6.5" @@ -302,6 +739,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.3.55" @@ -321,6 +778,21 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +[[package]] +name = "tinyvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.53.1" @@ -357,6 +829,46 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -369,12 +881,76 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/backend/proxysrc/Cargo.toml b/backend/proxysrc/Cargo.toml index 99e2945..e97884d 100644 --- a/backend/proxysrc/Cargo.toml +++ b/backend/proxysrc/Cargo.toml @@ -19,10 +19,54 @@ serde_json = "1" tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12", "logging"] } rustls = { version = "0.23", default-features = false, features = ["ring", "tls12", "std", "logging"] } rustls-pemfile = "2" +# QUIC, terminated by the engine for the same reason TLS is: after the Initial packet +# everything — frames, stream boundaries, even the packet number — is encrypted, so a +# layer that only forwards has nothing a filter could be shown. `ring` again, and the +# same rustls the TCP side already links. +quinn = { version = "0.11", default-features = false, features = ["runtime-tokio", "rustls-ring", "log"] } +# HTTP/3, terminated on top of QUIC. A binding rather than a second implementation: the +# frames and QPACK are somebody else's problem, and what this engine does with them — +# presenting each exchange to the chain as the HTTP/1.1 it would have been — is the part +# that belongs here. +h3 = "0.0.8" +h3-quinn = "0.0.10" +# HTTP/2, terminated on TCP for the reason HTTP/3 is terminated on QUIC: its method, path +# and headers are HPACK-compressed, so a layer that only forwards shows a filter a +# compression format. A binding again — HPACK, multiplexing and flow control are the +# crate's problem, and a second implementation of HTTP/2 in this tree is exactly what +# `AGENTS.md` forbids. +h2 = "0.4" +# HTTP/1.1 towards the service, for the one thing this engine could not do: speak an +# older version to the service than the client used. A client edge that is terminated — +# HTTP/3 over QUIC — is already rendered to the chain as HTTP/1.1, so putting those same +# bytes on a socket is not a translation invented here, it is the rendering being sent +# where the service can read it. The *response* is what needs the crate: HTTP/1.1 is the +# one version this engine never parsed, and inventing a parser for it beside hyper's is +# the thing `AGENTS.md` forbids. Client and http1 only — nothing here serves HTTP/1.1, +# and the h2 client above stays h2's. +hyper = { version = "1", default-features = false, features = ["client", "http1"] } +hyper-util = { version = "0.1", default-features = false, features = ["tokio"] } +http-body-util = "0.1" +http-body = "1" +bytes = "1" +http = "1" [dev-dependencies] # Test certificates are generated, never committed. rcgen = "0.13" +# The QUIC tests stand a real client and a real service either side of the relay: the +# same crates the engine links, so what they prove is what will run. +quinn = { version = "0.11", default-features = false, features = ["runtime-tokio", "rustls-ring"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +# And a real HTTP/3 service and client either side of it, for the same reason. +h3 = "0.0.8" +h3-quinn = "0.0.10" +# The same, for HTTP/2: the tests stand a real h2 client and a real h2 service either +# side of the proxy, so what they prove is what will run. +h2 = "0.4" +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12", "logging"] } +bytes = "1" +http = "1" # Catching a panicking filter is what keeps the datapath alive, so the binary # must unwind rather than abort. This is not a tuning knob, it is load-bearing. diff --git a/backend/proxysrc/src/capture.rs b/backend/proxysrc/src/capture.rs index 6eab314..bf6d972 100644 --- a/backend/proxysrc/src/capture.rs +++ b/backend/proxysrc/src/capture.rs @@ -19,13 +19,24 @@ //! follow; what it costs is that this is not evidence of what was on the wire, and the //! documentation says so. //! +//! **A QUIC stream is reconstructed the same way, one TCP stream each.** A QUIC stream +//! is ordered, reliable and starts at zero, which is precisely what a [`Tap`] rebuilds — +//! but every stream of one connection shares the connection's four-tuple, so writing +//! them out as they are would interleave a hundred of them into a single conversation +//! that no tool could take apart. So each is given a **synthetic client port** by +//! [`Tap::open_stream`], and that is one more piece of invented framing on top of the +//! invented framing this whole file is: the port identifies a stream and was never a +//! port anybody bound. It has to be said wherever a capture from here is offered, next +//! to the sentence about sequence numbers, because an operator reading a port back out +//! of Wireshark will otherwise go looking for a socket that does not exist. +//! //! Failure is always silent and always local. If the interface is not there, or the //! socket cannot be opened, or a send fails, the traffic goes on exactly as it would //! have: a capture aid must never be able to interrupt a service it is only watching. use std::ffi::CString; use std::net::{IpAddr, SocketAddr}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering}; use std::sync::Arc; /// The interface the decrypted traffic is written to. A fixed name on purpose: it is @@ -112,9 +123,58 @@ pub struct Tap { up: AtomicU32, /// Next sequence number for service → client. down: AtomicU32, + /// Whether the end of the stream has been written. + ended: AtomicBool, +} + +/// A reconstruction always gets an end, whichever way its owner left. +/// +/// The TCP path closes its tap where the connection is torn down, and could; the paths +/// that carry one stream of many have a dozen ways out — a refused request returns before +/// anything is joined — and a stream left without a FIN is one Wireshark holds open +/// waiting for bytes that are never coming. So the last holder emits it, and +/// [`Tap::closed`] stays callable where the end is known and worth saying explicitly. +impl Drop for Tap { + fn drop(&mut self) { + self.closed(); + } +} + +/// Hands out the ports that tell one multiplexed stream from another. +/// +/// A counter and not a hash of the stream id: ids restart at zero on every QUIC +/// connection, so deriving the port from one would drop two clients' first streams into +/// the same conversation. Cycling the ephemeral range instead gives the same answer a +/// NAT gives, and reuse only becomes possible after 16384 live streams — at which point +/// a new SYN is what separates them, exactly as it does for a real port that came round +/// again. +static NEXT_STREAM_PORT: AtomicU16 = AtomicU16::new(0); + +const EPHEMERAL_BASE: u16 = 49152; +const EPHEMERAL_SPAN: u16 = 65535 - EPHEMERAL_BASE + 1; + +fn stream_port() -> u16 { + EPHEMERAL_BASE + NEXT_STREAM_PORT.fetch_add(1, Ordering::Relaxed) % EPHEMERAL_SPAN } impl Tap { + /// Start a reconstruction for one stream of a multiplexed connection. + /// + /// The client's address keeps its IP and loses its port, because the port is the only + /// field left that can tell two streams of one connection apart: the service's end is + /// the service's and changing it would misattribute the traffic, and the client's IP + /// is what an operator filters on. What comes back is therefore a conversation + /// between a real host and a real service on a port that never existed. + pub fn open_stream( + capture: Option>, + client: SocketAddr, + server: SocketAddr, + ) -> Option> { + let mut client = client; + client.set_port(stream_port()); + Tap::open(capture, client, server) + } + /// Start a reconstruction, and emit the handshake that opens it. /// /// The handshake is not decoration: without a SYN, Wireshark has no beginning for @@ -132,6 +192,7 @@ impl Tap { server, up: AtomicU32::new(0), down: AtomicU32::new(0), + ended: AtomicBool::new(false), }); // Interleaved with the increments, not emitted and then counted: each of these // reads the two counters as they stand, so a SYN-ACK sent before the client's @@ -159,7 +220,13 @@ impl Tap { } /// Close the reconstruction, so the stream has an end as well as a beginning. + /// + /// Idempotent: [`Drop`] calls it too, and a stream that ended twice would read as one + /// that was closed and then closed again. pub fn closed(&self) { + if self.ended.swap(true, Ordering::Relaxed) { + return; + } self.emit(true, FIN | ACK, &[]); self.emit(false, FIN | ACK, &[]); } @@ -312,3 +379,28 @@ fn build_v6( out.extend_from_slice(payload); out } + +#[cfg(test)] +mod tests { + use super::*; + + /// The one property that makes multiplexed streams separable in a capture. + /// + /// Two streams of one connection differ in nothing a capture tool looks at — same + /// addresses, same service port — so if this stopped handing out distinct ports they + /// would arrive as one interleaved conversation, which is worse than not capturing + /// them at all: it reads as a single stream whose bytes make no sense. + #[test] + fn every_stream_gets_a_port_of_its_own() { + let mut seen = std::collections::HashSet::new(); + for _ in 0..EPHEMERAL_SPAN as usize { + let port = stream_port(); + assert!(port >= EPHEMERAL_BASE, "{port} is not an ephemeral port"); + assert!(seen.insert(port), "{port} came round before the range was spent"); + } + // And then it does come round, rather than running out: a long-lived engine has + // more streams than there are ports, and a new SYN is what separates the reuse — + // exactly as it does for a real port the kernel hands out again. + assert!(!seen.insert(stream_port())); + } +} diff --git a/backend/proxysrc/src/control.rs b/backend/proxysrc/src/control.rs index 6b30ca7..16b7e54 100644 --- a/backend/proxysrc/src/control.rs +++ b/backend/proxysrc/src/control.rs @@ -9,8 +9,9 @@ use std::time::Duration; use tokio::io::{AsyncBufReadExt, BufReader}; use crate::filter::{ChainHandle, FilterChain}; +use crate::proxy::{Edge, Onward, Published, Targets}; +use crate::relays::Relays; use crate::rules::parse_ruleset; -use crate::udp::UdpManager; fn ack(ok: bool, detail: &str) { if ok { @@ -29,7 +30,12 @@ fn ack(ok: bool, detail: &str) { /// opposite of the fail-open rule and deliberately so. Failing open covers a filter /// that breaks at runtime; it is not a licence to silently drop a ruleset the /// operator asked for. -pub async fn serve_stdin(chain: ChainHandle, deadline: Duration, udp: UdpManager) { +pub async fn serve_stdin( + chain: ChainHandle, + deadline: Duration, + relays: Relays, + targets: std::sync::Arc, +) { let mut lines = BufReader::new(tokio::io::stdin()).lines(); loop { let line = match lines.next_line().await { @@ -51,10 +57,21 @@ pub async fn serve_stdin(chain: ChainHandle, deadline: Duration, udp: UdpManager continue; } + // `ADD_UDP` whatever is behind it: the command names what the *rules* match on, + // and a QUIC service is UDP to the kernel. The backend asks for a port to point + // an address at, and that question has one answer whichever relay gives it. if let Some(rest) = trimmed.strip_prefix("ADD_UDP ") { - let target_str = rest.trim(); + // `
` or `
` — what the service behind this relay + // speaks, which is the relay's own answer and not the process's. + let mut parts = rest.split_whitespace(); + let target_str = parts.next().unwrap_or("").trim(); + let onward = match parts.next() { + Some("plain") => Onward::Plain, + Some("tls") => Onward::Tls, + _ => Onward::Same, + }; match target_str.parse::() { - Ok(upstream) => match udp.add_relay(upstream).await { + Ok(upstream) => match relays.add_relay(upstream, onward).await { Ok(port) => { println!("UDP {upstream} {port}"); ack(true, ""); @@ -73,6 +90,68 @@ pub async fn serve_stdin(chain: ChainHandle, deadline: Duration, udp: UdpManager continue; } + // `PUBLISH ` and `WITHDRAW `: one address is being + // served on a port the service does not listen on. Sent while the engine runs + // because an address can be added to a running service, and the startup list + // (`FGEX_PROXY_TARGETS`) says the same thing for the ones it began with. + if let Some(rest) = trimmed.strip_prefix("PUBLISH ") { + // `PUBLISH []`: what is spoken at one + // address, what is spoken to the service behind it, and — where it is not + // simply the service — where that service is. A dash for the last says the + // address *is* the service, which is every address until an operator says + // otherwise. + let mut parts = rest.split_whitespace(); + let public = parts.next().map(str::parse::); + let edge = match parts.next() { + Some("tls") => Some(Edge::Tls), + Some("any") | Some("clear") | None => Some(Edge::Whatever), + Some(_) => None, + }; + let onward = match parts.next() { + Some("plain") => Some(Onward::Plain), + Some("tls") => Some(Onward::Tls), + Some("same") | None => Some(Onward::Same), + Some(_) => None, + }; + let target = match parts.next() { + None | Some("-") => Ok(None), + Some(addr) => addr.parse::().map(Some), + }; + match (public, edge, onward, target) { + (Some(Ok(public)), Some(edge), Some(upstream), Ok(target)) => { + targets.publish(public, Published { target, edge, upstream }); + match target { + Some(target) => { + eprintln!("[info] [control] {public} now fronts {target}") + } + None => eprintln!("[info] [control] {public} declared"), + } + ack(true, ""); + } + _ => { + let err = + format!("PUBLISH wants `
[]`, got {rest:?}"); + eprintln!("[warn] [control] {err}"); + ack(false, &err); + } + } + continue; + } + if let Some(rest) = trimmed.strip_prefix("WITHDRAW ") { + match rest.trim().parse::() { + Ok(public) => { + targets.withdraw(&public); + ack(true, ""); + } + Err(e) => { + let err = format!("invalid address {rest:?}: {e}"); + eprintln!("[warn] [control] {err}"); + ack(false, &err); + } + } + continue; + } + match parse_ruleset(trimmed).and_then(|filters| { // Everything that has to be started is started here, while the old chain // is still the one enforcing. diff --git a/backend/proxysrc/src/filter.rs b/backend/proxysrc/src/filter.rs index d28a7ac..e49c0d7 100644 --- a/backend/proxysrc/src/filter.rs +++ b/backend/proxysrc/src/filter.rs @@ -72,10 +72,52 @@ pub struct ConnectionMeta { pub client: std::net::SocketAddr, /// Where it was actually headed — the protected service, not this proxy. pub server: std::net::SocketAddr, - /// Whether this is a TCP connection or a UDP flow. A filter is shown it because the - /// library's stream and HTTP models only apply to one of them, and a filter written - /// against `RawPacket` may legitimately want to know which it is looking at. - pub tcp: bool, + /// What is underneath. A filter is shown it because the library's stream and HTTP + /// models need a stream to be built on, and a filter written against `RawPacket` may + /// legitimately want to know what it is looking at. + pub l4: L4, +} + +/// What carries a connection. +/// +/// A boolean until QUIC arrived, which broke the question in two: a QUIC stream is +/// ordered and reliable, so every model that needs a stream applies to it, and it is +/// carried by UDP, so every rule that matches the wire matches UDP. Answering either of +/// those with "is it TCP" gets the other one wrong — as a boolean this either hid the +/// stream models from QUIC or told a filter it was looking at TCP. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum L4 { + Tcp, + Udp, + /// A stream inside a QUIC connection. + Quic, + /// A QUIC DATAGRAM frame: inside a QUIC connection, and outside its streams. + /// + /// Its own value rather than [`L4::Quic`] with a flag, because the two answer the + /// stream question differently and everything the library builds on a stream turns + /// on that answer. Telling a filter `udp` instead would answer both questions + /// correctly — unreliable, and UDP on the wire — and lose the only thing left worth + /// knowing, which is that this arrived inside a connection that is also carrying + /// streams. + QuicDatagram, +} + +impl L4 { + /// Whether the bytes arrive as an ordered, reliable stream — which is what the + /// library's stream and HTTP models are built on. + pub fn is_stream(self) -> bool { + matches!(self, L4::Tcp | L4::Quic) + } + + /// The name the filter library knows it by. + pub fn name(self) -> &'static str { + match self { + L4::Tcp => "tcp", + L4::Udp => "udp", + L4::Quic => "quic", + L4::QuicDatagram => "quic-datagram", + } + } } /// Identifies one connection for the lifetime of the process. diff --git a/backend/proxysrc/src/h1up.rs b/backend/proxysrc/src/h1up.rs new file mode 100644 index 0000000..8e04bc7 --- /dev/null +++ b/backend/proxysrc/src/h1up.rs @@ -0,0 +1,324 @@ +//! Speaking HTTP/1.1 to the service, under an edge that is not HTTP/1.1. +//! +//! Everything else in this engine forwards to the service in the version the client +//! arrived in: the rendering is a view shown to the filters, never a translation the +//! service sees. This module is the one exception, and it is the operator's to ask for +//! (`FGEX_PROXY_UPSTREAM`): an ordinary web service that speaks HTTP/1.1 and nothing else +//! can be put behind an HTTP/3 edge, with firegex terminating QUIC in front of it. +//! +//! **Half of it is free and half of it needed a crate.** The request direction is already +//! done by the time it gets here — [`crate::http1`] renders every exchange as the HTTP/1.1 +//! it would have been, because that is what the chain has to be shown, so putting those +//! same bytes on a socket invents nothing. The *answer* is what could not be done: HTTP/1.1 +//! is the one version this engine has never parsed. An HTTP/1.1 connection elsewhere is a +//! byte pump — the filters look at bytes, and what parses them is the library on the Python +//! side — so there was no response parser to reach for, and writing one beside hyper's is +//! exactly what `AGENTS.md` forbids. So the answer comes back through hyper's client, and +//! this module is a binding: framing, chunked encoding and keep-alive are the crate's +//! problem. +//! +//! **One connection per exchange.** HTTP/1.1 has no multiplexing, and the edge above does: +//! a QUIC stream is one request, so it gets one upstream connection of its own, dialled +//! from the client's own address like every other dial this engine makes. Pooling would +//! put two clients' requests on one socket, and with it the thing `ChainSessions` exists +//! to prevent — one client's bytes deciding another's verdict — at a layer where nobody +//! would think to look for it. + +use std::io; +use std::net::SocketAddr; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::BodyExt; + +use crate::http1::{Answer, Incoming, Outbound, OutboundBody}; + +/// The body of a request being written piece by piece into hyper. +/// +/// hyper takes a body as one value and pulls from it; the chain produces pieces as it +/// accepts them. A channel is the join between the two, and its bound is what keeps a +/// fast client from buying memory in this process while a slow service reads. +struct Streamed { + rx: tokio::sync::mpsc::Receiver>, +} + +impl http_body::Body for Streamed { + type Data = Bytes; + type Error = io::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + self.get_mut().rx.poll_recv(cx).map(|frame| frame.map(Ok)) + } +} + +/// How many pieces of one request may be waiting for the service at once. +const IN_FLIGHT: usize = 8; + +/// How this engine reaches a service that speaks HTTP/1.1. +/// +/// Dial parameters and nothing else: there is no connection here, because each exchange +/// opens its own. Cloned per exchange the way the HTTP/3 sender above it is. +#[derive(Clone)] +pub(crate) struct H1Upstream { + pub(crate) upstream: SocketAddr, + /// Dialled as this address, so the service sees the real client. The same + /// unconditional source preservation every other dial here makes. + pub(crate) client: SocketAddr, + /// Present when the service speaks TLS: the client half of the handshake. + pub(crate) tls: Option>, + pub(crate) connect_timeout: Duration, + pub(crate) self_mark: Option, + /// Dial as the client. Off is the fallback the TCP path also has, and it is here for + /// the same reason: a service seeing one source address for everyone is a regression + /// nobody would attribute to us. + pub(crate) spoof: bool, +} + +impl H1Upstream { + async fn connect(&self) -> io::Result { + if !self.spoof { + return tokio::time::timeout( + self.connect_timeout, + crate::transparent::connect_plain(self.upstream, self.self_mark), + ) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "upstream connect timed out"))?; + } + // Losing the client's address is bad, losing the connection is worse — the same + // trade, and just as loud about it, as the TCP and QUIC dials make. + match tokio::time::timeout( + self.connect_timeout, + crate::transparent::connect_as(self.client.ip(), self.upstream, self.self_mark), + ) + .await + { + Ok(Ok(stream)) => Ok(stream), + failed => { + let why = match failed { + Ok(Err(e)) => e.to_string(), + _ => "timed out (is the return path diverted?)".to_string(), + }; + eprintln!( + "[warn] [h1] cannot reach {} as {}: {why}. Falling back to our own \ + address — the service will not see real client IPs.", + self.upstream, + self.client.ip(), + ); + tokio::time::timeout( + self.connect_timeout, + crate::transparent::connect_plain(self.upstream, self.self_mark), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "upstream connect timed out") + })? + } + } + } +} + +impl Outbound for H1Upstream { + type Error = io::Error; + type Body = H1Body; + type Answer = H1Answer; + + async fn open( + &mut self, + request: http::Request<()>, + ends_it: bool, + ) -> io::Result<(Self::Body, Self::Answer)> { + let socket = self.connect().await?; + let _ = socket.set_nodelay(true); + let (tx, rx) = tokio::sync::mpsc::channel(IN_FLIGHT); + let body = Streamed { rx }; + // The head goes out as HTTP/1.1 whatever it arrived as, and that is more than + // restating the version. HTTP/2 and HTTP/3 carry the target split across + // `:scheme`, `:authority` and `:path`, which `http::Uri` holds as one absolute + // URI — and written to a socket unchanged that is the *proxy* form of a request + // line, `GET https://host/path`, which an origin server is entitled to refuse and + // which every service reading the path back gets wrong. So the target is reduced + // to its origin form and the authority moves to `host`, which is exactly the + // decision `http1::render_request` already makes for the chain: what the filters + // were shown and what the service receives are the same message, and the two + // deciding it differently is the drift this engine is arranged against. + let (mut parts, _) = request.into_parts(); + parts.version = http::Version::HTTP_11; + if let Some(authority) = parts.uri.authority().map(|a| a.to_string()) { + if !parts.headers.contains_key(http::header::HOST) { + if let Ok(value) = http::HeaderValue::from_str(&authority) { + parts.headers.insert(http::header::HOST, value); + } + } + } + parts.uri = parts + .uri + .path_and_query() + .map(|target| target.as_str()) + .unwrap_or("/") + .parse() + .map_err(|e| io::Error::other(format!("cannot render the request target: {e}")))?; + let request = http::Request::from_parts(parts, body); + + let answer = match &self.tls { + Some(config) => { + let name = crate::tls::server_name(&self.upstream.ip().to_string())?; + let stream = tokio::time::timeout( + self.connect_timeout, + crate::tls::connector(config.clone()).connect(name, socket), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "upstream TLS handshake timed out") + })??; + Self::handshake(stream, request).await? + } + None => Self::handshake(socket, request).await?, + }; + // A head that is the whole message gets a body that is already over: hyper then + // frames the request with no body at all, which is what HTTP/1.1 does for one. + // The flag exists for HTTP/2, where it has to ride on the head; here it only + // saves a channel nobody will write to. + Ok((H1Body { tx: (!ends_it).then_some(tx) }, answer)) + } +} + +impl H1Upstream { + /// The hyper side of opening one exchange, shared by the two kinds of socket. + async fn handshake(io: S, request: http::Request) -> io::Result + where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, + { + let (mut sender, connection) = + hyper::client::conn::http1::handshake(hyper_util::rt::TokioIo::new(io)) + .await + .map_err(io::Error::other)?; + // hyper drives the socket from here; without this task nothing is ever written. + // Its end is this exchange's end, so it is dropped with the answer rather than + // detached — one connection per exchange is what makes that safe to say. + let driving = tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("[info] [h1] the connection to the service ended: {e}"); + } + }); + // Not awaited here: the service is allowed to answer before it has read the whole + // request — a refusal, a redirect, a `413` — and the body is still being written + // into the channel above by the other half of this exchange. Awaiting it now is a + // deadlock with a name. + let pending = Box::pin(sender.send_request(request)); + Ok(H1Answer { + pending: Some(pending), + body: None, + trailers: None, + _driving: driving, + }) + } +} + +/// The request body's writing half. +pub(crate) struct H1Body { + /// Taken by `finish`, because dropping the sender is what ends the body: hyper reads + /// the close as the last frame and writes the terminating chunk. Waiting on the + /// channel instead — for the *receiver* to go — is waiting for the service to answer + /// a request this end has not finished sending, which is a deadlock with a name. + tx: Option>>, +} + +impl H1Body { + async fn send(&mut self, frame: http_body::Frame) -> io::Result<()> { + let Some(tx) = self.tx.as_ref() else { + return Err(io::Error::other("this request body has already ended")); + }; + tx.send(frame) + .await + .map_err(|_| io::Error::other("the service stopped reading the request")) + } +} + +impl OutboundBody for H1Body { + type Error = io::Error; + + async fn data(&mut self, chunk: Bytes) -> io::Result<()> { + self.send(http_body::Frame::data(chunk)).await + } + + async fn trailers(&mut self, trailers: http::HeaderMap) -> io::Result<()> { + self.send(http_body::Frame::trailers(trailers)).await + } + + async fn finish(&mut self) -> io::Result<()> { + self.tx.take(); + Ok(()) + } +} + +type Pending = Pin< + Box>> + Send>, +>; + +/// The service's answer, read the way every other answer in this engine is. +pub(crate) struct H1Answer { + /// Taken by `response()`, which is the first thing the reading half does. + pending: Option, + body: Option, + /// A trailer section arrives as a frame among the body's, so it is kept here until + /// the body is done and somebody asks for it — which is the order the rendering + /// wants it in. + trailers: Option, + /// Dropped with the answer, which cancels the connection task: one connection per + /// exchange means its life is exactly this one's. + _driving: tokio::task::JoinHandle<()>, +} + +impl Answer for H1Answer { + async fn response(&mut self) -> io::Result> { + let pending = self + .pending + .take() + .ok_or_else(|| io::Error::other("the answer was already taken"))?; + let response = pending.await.map_err(io::Error::other)?; + let (parts, body) = response.into_parts(); + self.body = Some(body); + Ok(http::Response::from_parts(parts, ())) + } +} + +impl Incoming for H1Answer { + type Error = io::Error; + + fn ended(&self) -> bool { + // hyper knows: a response framed with no body at all — `204`, `304`, a declared + // length of zero — is over before anybody reads it. + use http_body::Body; + self.body.as_ref().is_none_or(|body| body.is_end_stream()) + } + + async fn data(&mut self) -> io::Result> { + let Some(body) = self.body.as_mut() else { + return Ok(None); + }; + while let Some(frame) = body.frame().await { + let frame = frame.map_err(io::Error::other)?; + match frame.into_data() { + Ok(data) => return Ok(Some(data)), + // Not data, so it is the trailer section, and it is the last thing the + // body will produce. Kept for whoever asks next rather than dropped. + Err(frame) => { + if let Ok(trailers) = frame.into_trailers() { + self.trailers = Some(trailers); + } + } + } + } + Ok(None) + } + + async fn trailers(&mut self) -> io::Result> { + Ok(self.trailers.take()) + } +} diff --git a/backend/proxysrc/src/h2.rs b/backend/proxysrc/src/h2.rs new file mode 100644 index 0000000..91cbfb9 --- /dev/null +++ b/backend/proxysrc/src/h2.rs @@ -0,0 +1,800 @@ +//! HTTP/2, terminated by the engine — and shown to the filters as HTTP/1.1. +//! +//! The rendering lives in [`crate::http1`]; what is here is the adapter, and the reason it +//! exists is the same one that put [`crate::h3`] there. HTTP/2 puts the method, the path +//! and the headers in an **HPACK-compressed** HEADERS frame, so a connection that +//! negotiated `h2` and was only forwarded showed a filter a compression format: a pattern +//! written against a request line matched nothing, and a filter asking for an +//! `HttpRequest` was handed the `PRI * HTTP/2.0` preface — parsed as one fictitious +//! request — and then frames it could not read, so it was never called again. No block, no +//! log, no counter: the exact failure everything else in this engine is arranged against. +//! Only a body travelled in the clear. Since gRPC *is* HTTP/2, that also meant gRPC could +//! not be filtered at all. +//! +//! Unlike QUIC, terminating was never forced here — the bytes are framed by a transport +//! the kernel understands, and firegex could have gone on forwarding them. It is forced by +//! the promise instead: a filter written once has to work everywhere, and there is no way +//! to keep that promise over HPACK without decompressing it, which is what parsing HTTP/2 +//! means. +//! +//! Three things differ from h3 and are worth stating, because each of them is a way to get +//! this wrong: +//! +//! * **Flow control is ours to run.** `h2` hands received bytes over and waits for them to +//! be released; a proxy that releases on read has no backpressure and buffers whatever +//! the faster side sends, and one that never releases stalls after the first window. So +//! capacity is released **after the piece has been forwarded**, which is what makes the +//! window follow the service rather than this process, and outbound writes reserve +//! capacity first rather than queueing in memory. +//! * **The end of a message rides on its last frame**, not on a separate close. A response +//! of head-and-nothing-else must be sent with `end_of_stream` **on the head**: send it +//! open and then close it with an empty DATA frame and a *trailers-only* answer stops +//! being one — which is precisely the shape of a gRPC status-only reply, and gRPC +//! clients refuse it. [`head_is_the_message`] is the one place that decision is made, +//! for both directions. +//! * **A stream is a connection to the chain**, which on HTTP/1.1 it is not. There a +//! keep-alive connection carries many requests through one set of filter state; here the +//! streams are concurrent and interleaved, and sharing state between them is exactly +//! "one client's bytes deciding another client's verdict" — the thing [`ChainSessions`] +//! exists to prevent. So an h2 stream gets its own, as an h3 request stream does. +//! +//! **Server push is not carried, and the service is told so in the handshake** +//! (`enable_push(false)`) rather than having its pushes dropped — the same courtesy the +//! QUIC edge pays about datagrams. **Extended CONNECT is not advertised**, and a plain +//! `CONNECT` stream is refused rather than rendered: a tunnel is opaque bytes, and +//! inventing an HTTP/1.1 request for one would be the fabrication this module exists to +//! avoid. + +use std::io; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::capture::{Capture, Tap}; +use crate::filter::{ + next_connection_id, ChainHandle, ChainSessions, ConnectionId, ConnectionMeta, Direction, L4, +}; +use crate::http1::{ + carried, close_body, head_framing, judge, push_body, render_request, render_response, Answer, + Framing, Incoming, Outbound, OutboundBody, Rendered, +}; +use crate::proxy::ProxyStats; + +/// How long a refused connection is given to put its GOAWAY on the wire. +/// +/// A refusal that simply dropped the socket would reach the client as a truncated TLS +/// stream, which is the peer having to guess — the same reason the byte pumps shut down +/// rather than abort. Bounded, because a peer that has stopped reading must not be able to +/// hold the task open. +const GOAWAY_GRACE: Duration = Duration::from_secs(2); + +impl Incoming for h2::RecvStream { + type Error = h2::Error; + + fn data(&mut self) -> impl std::future::Future, h2::Error>> + Send { + // Named through the trait's own path rather than `self.data()`: the inherent + // method has the same name, and which one a bare call resolves to is not something + // this file should depend on. + async move { h2::RecvStream::data(self).await.transpose() } + } + + fn trailers( + &mut self, + ) -> impl std::future::Future, h2::Error>> + Send { + h2::RecvStream::trailers(self) + } + + fn release(&mut self, taken: usize) -> Result<(), h2::Error> { + self.flow_control().release_capacity(taken) + } + + fn ended(&self) -> bool { + self.is_end_stream() + } +} + +/// HTTP/2 towards the service, which is what an HTTP/2 client edge used to always get. +/// +/// A handle to the connection `carry` opened, cloned per stream — which is what HTTP/2 +/// multiplexing is, and why this is a clone rather than a dial. +#[derive(Clone)] +pub(crate) struct H2Upstream(h2::client::SendRequest); + +impl Outbound for H2Upstream { + type Error = io::Error; + type Body = H2Send; + type Answer = H2Recv; + + async fn open( + &mut self, + request: http::Request<()>, + ends_it: bool, + ) -> io::Result<(Self::Body, Self::Answer)> { + // The parts travel as they arrived; only the version is restated, because what + // goes out is HTTP/2 again. + let (mut parts, _) = request.into_parts(); + parts.version = http::Version::HTTP_2; + let (response, to_service) = self + .0 + .send_request(http::Request::from_parts(parts, ()), ends_it) + .map_err(io::Error::other)?; + Ok(( + H2Send { inner: to_service, done: ends_it }, + H2Recv { pending: Some(response), body: None, trailers: None }, + )) + } +} + +/// The request body on its way to the service, over HTTP/2. +pub(crate) struct H2Send { + inner: h2::SendStream, + /// Whether the end of the stream has already gone out. Writing past it is a user + /// error `h2` reports rather than ignores, and the end can leave on the head, on a + /// trailer section, or on an empty final frame — three ways to get here once. + done: bool, +} + +impl OutboundBody for H2Send { + type Error = io::Error; + + async fn data(&mut self, chunk: Bytes) -> io::Result<()> { + send_body(&mut self.inner, chunk, false).await.map_err(io::Error::other) + } + + async fn trailers(&mut self, trailers: http::HeaderMap) -> io::Result<()> { + self.done = true; + self.inner.send_trailers(trailers).map_err(io::Error::other) + } + + async fn finish(&mut self) -> io::Result<()> { + if self.done { + return Ok(()); + } + self.done = true; + self.inner.send_data(Bytes::new(), true).map_err(io::Error::other) + } +} + +/// The service's answer over HTTP/2, read the way every other answer in this engine is. +pub(crate) struct H2Recv { + pending: Option, + body: Option, + trailers: Option, +} + +impl Answer for H2Recv { + async fn response(&mut self) -> io::Result> { + let pending = self + .pending + .take() + .ok_or_else(|| io::Error::other("the answer was already taken"))?; + let response = pending.await.map_err(io::Error::other)?; + let (parts, body) = response.into_parts(); + self.body = Some(body); + Ok(http::Response::from_parts(parts, ())) + } +} + +impl Incoming for H2Recv { + type Error = io::Error; + + async fn data(&mut self) -> io::Result> { + let Some(body) = self.body.as_mut() else { + return Ok(None); + }; + Incoming::data(body).await.map_err(io::Error::other) + } + + async fn trailers(&mut self) -> io::Result> { + if let Some(trailers) = self.trailers.take() { + return Ok(Some(trailers)); + } + let Some(body) = self.body.as_mut() else { + return Ok(None); + }; + Incoming::trailers(body).await.map_err(io::Error::other) + } + + fn release(&mut self, taken: usize) -> io::Result<()> { + match self.body.as_mut() { + Some(body) => body.release(taken).map_err(io::Error::other), + None => Ok(()), + } + } + + fn ended(&self) -> bool { + self.body.as_ref().is_some_and(|body| body.is_end_stream()) + } +} + +/// Everything one terminated HTTP/2 connection is carried with. +/// +/// Cloned per stream: every field is a handle already, so a clone is a few reference +/// counts and the streams need nothing from each other. +#[derive(Clone)] +pub struct Carriage { + pub client: SocketAddr, + pub upstream: SocketAddr, + pub chain: ChainHandle, + pub stats: Arc, + /// Where a reconstruction of each stream is written, when anything is listening. + pub capture: Option>, + /// How long this connection may say nothing at all. The question is the + /// *connection's*, exactly as it is on QUIC: a peer that opens twenty streams and + /// speaks on none is the shape it is there to catch. + pub first_byte_timeout: Option, +} + +impl Carriage { + fn rendered(&self) -> Rendered<'_> { + Rendered { + chain: &self.chain, + stats: &self.stats, + client: self.client, + layer: "h2", + } + } + + /// Tell the chain about a stream, in the words the rest of the engine uses. + /// + /// `L4::Tcp`, with nothing to distinguish it from HTTP/1.1 on the same service — which + /// is the whole point. An h2 stream is a stream, and it is TCP on the wire, so both + /// questions the enum answers have the same answer they have one layer up. + fn open(&self) -> ConnectionId { + let connection = next_connection_id(); + self.chain.current().connection_opened( + connection, + &ConnectionMeta { + client: self.client, + server: self.upstream, + l4: L4::Tcp, + }, + ); + connection + } + + /// A reconstruction of one stream, for whoever is watching the capture interface. + /// + /// Per stream and not per connection: a stream is what has a beginning, an order and + /// an end. What it has not got is a port of its own — every stream of one connection + /// shares the four-tuple — so [`Tap::open_stream`] invents one, and the documentation + /// says so wherever this interface is offered. + fn tap(&self) -> Option> { + Tap::open_stream(self.capture.clone(), self.client, self.upstream) + } +} + +/// Ends the whole connection because a rule refused something on one of its streams. +/// +/// A refusal ends the connection, not the stream. Resetting one stream of many would leave +/// the client free to ask again on the next, which is not what a block means anywhere else +/// in firegex — and on HTTP/1.1 keep-alive a refused request already takes the connection +/// with it. +#[derive(Clone)] +struct Refusal(Arc>); + +impl Refusal { + fn new() -> (Self, tokio::sync::watch::Receiver) { + let (tx, rx) = tokio::sync::watch::channel(false); + (Self(Arc::new(tx)), rx) + } + + fn refuse(&self) { + let _ = self.0.send(true); + } +} + +/// Carry one connection that both ends agreed to speak HTTP/2 on. +pub async fn carry(client_io: C, service_io: S, ctx: Carriage) -> io::Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send + 'static, + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let (sender, driver) = h2::client::Builder::new() + // Said in the handshake rather than by dropping what arrives: a pushed response + // this proxy cannot render is one the service should not spend a stream on, and a + // peer that learns from SETTINGS is better off than one that watches pushes + // vanish. The same answer the QUIC edge gives about datagrams on h3. + .enable_push(false) + .handshake::<_, Bytes>(service_io) + .await + .map_err(io::Error::other)?; + carry_with(client_io, H2Upstream(sender), Some(tokio::spawn(driver)), ctx).await +} + +/// Carry an HTTP/2 client in front of a service that speaks **HTTP/1.1**. +/// +/// The edge the QUIC side has had all along, arriving here: what the filters are shown is +/// the same HTTP/1.1 rendering either way, so sending that rendering on to the service +/// invents nothing. It is what lets an ordinary cleartext web service be reached over +/// HTTP/2 with only firegex holding a certificate — the deployment that could already be +/// reached over HTTP/3 and, until this existed, not over HTTP/2, for no reason an operator +/// could see. +/// +/// No connection driver, because there is no connection: `H1Upstream` dials once per +/// exchange, which is also what keeps two clients' requests off one socket. +pub(crate) async fn carry_to_h1( + client_io: C, + upstream: crate::h1up::H1Upstream, + ctx: Carriage, +) -> io::Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + carry_with(client_io, upstream, None, ctx).await +} + +async fn carry_with( + client_io: C, + upstream: U, + mut driving: Option>>, + ctx: Carriage, +) -> io::Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send + 'static, + U: Outbound + Clone + Send + 'static, +{ + // Extended CONNECT is left unadvertised, which is the default: a service that wants + // WebTransport over HTTP/2 needs a session model this proxy has not got, and + // advertising it would invite streams whose rendering would be a fabrication. Push is + // turned off towards the service above, in the same spirit. + let mut server = h2::server::Builder::new() + .handshake::<_, Bytes>(client_io) + .await + .map_err(io::Error::other)?; + + let (refusal, mut refused) = Refusal::new(); + + // Set by the first head that crosses in either direction. A service that answers + // before its client has spoken satisfies it, exactly as a request would — the + // deadline is about silence, not about who speaks first. + let spoken = Arc::new(AtomicBool::new(false)); + let watchdog = ctx.first_byte_timeout.map(|deadline| { + let spoken = Arc::clone(&spoken); + let refusal = refusal.clone(); + let stats = Arc::clone(&ctx.stats); + tokio::spawn(async move { + tokio::time::sleep(deadline).await; + if !spoken.load(Ordering::Relaxed) { + stats.no_first_byte.fetch_add(1, Ordering::Relaxed); + // Through the same channel a refusal uses, so the connection is wound + // down with a GOAWAY rather than dropped: a peer dropped mid-stream reads + // a truncation, and this connection has done nothing to deserve one. + refusal.refuse(); + } + }) + }); + + // The exchanges already in flight, kept rather than detached. A client winding the + // connection down leaves `accept()` answering `None` while answers are still owed, and + // walking away there would cut off the last request of every well-behaved client. + let mut inflight = tokio::task::JoinSet::new(); + // A refusal and a first-byte deadline both end the connection here rather than + // letting it run out; the difference between them is in the log, not in what is + // done about it. + let mut ended_early = false; + + loop { + tokio::select! { + accepted = server.accept() => match accepted { + Some(Ok((request, respond))) => { + let ctx = ctx.clone(); + let refusal = refusal.clone(); + let mut sender = upstream.clone(); + let spoken = Arc::clone(&spoken); + inflight.spawn(async move { + let connection = ctx.open(); + let outcome = + exchange(&ctx, &refusal, &spoken, connection, request, respond, &mut sender) + .await; + // Whatever happened, the filter's state for this stream goes — a + // refused request and a completed one both end it. + ctx.chain.current().connection_closed(connection); + if let Err(e) = outcome { + eprintln!("[info] [h2] stream ended: {e}"); + } + }); + } + Some(Err(e)) => { + eprintln!("[info] [h2] the client's connection ended: {e}"); + break; + } + // The client is done with the connection, which is the ordinary way out. + None => break, + }, + // The service ended its side. Waiting for the client to notice would mean + // holding a connection that has nowhere to go. Only where there *is* a + // connection: an HTTP/1.1 upstream opens one per exchange, so there is no + // shared driver to outlive and nothing to watch here. + reason = async { driving.as_mut().unwrap().await }, if driving.is_some() => { + if let Ok(Err(e)) = reason { + eprintln!("[info] [h2] the service ended the connection: {e}"); + } + break; + } + _ = refused.changed() => { + ended_early = true; + break; + } + } + } + + if ended_early { + // CANCEL rather than REFUSED_STREAM: the latter tells a client the request was + // never acted on and may be retried, and a block a client retries past is not a + // block. The reason reaches the operator through the service log, which is where + // the filter that refused it is already named. + server.abrupt_shutdown(h2::Reason::CANCEL); + // Polled rather than dropped, so the GOAWAY is actually written before the socket + // goes. Bounded: a peer that has stopped reading must not hold this open. + let _ = tokio::time::timeout( + GOAWAY_GRACE, + std::future::poll_fn(|cx| server.poll_closed(cx)), + ) + .await; + // Drained even here, and this is the part that is easy to get wrong: the + // exchanges still in flight have to *end* rather than be dropped, because ending + // is what releases the filter state each of them holds — for Python, a whole set + // of module globals, living in the worker under a connection id that only + // `connection_closed` frees. Aborting them by dropping the set leaked one of + // those per stream, on the path an attacker triggers on purpose. They finish + // immediately: the connection under them is already shut down. + let _ = tokio::time::timeout(GOAWAY_GRACE, drain(&mut inflight)).await; + } else { + // Nothing here is unbounded: every filter an exchange consults runs under the + // chain's own deadline, and the connection has the socket's. Unbounded on + // purpose, though — a client winding the connection down leaves answers still + // owed, and walking away would cut off its last exchange. + drain(&mut inflight).await; + } + drop(upstream); + if let Some(driving) = driving { + driving.abort(); + } + if let Some(watchdog) = watchdog { + watchdog.abort(); + } + Ok(()) +} + +/// Wait for every exchange still in flight to end. +/// +/// Ending is what releases what a filter is holding for that stream, so this is not +/// tidiness: a set of exchanges dropped rather than joined leaves their state behind. +async fn drain(inflight: &mut tokio::task::JoinSet<()>) { + while inflight.join_next().await.is_some() {} +} + +/// Whether a message's head is the whole of the message. +/// +/// Its own function because getting it wrong is invisible on HTTP/1.1 and fatal on +/// HTTP/2: the end of a message rides on its last frame, so a head that *is* the entire +/// message has to be sent as one. Sending it open and closing with an empty DATA frame +/// turns a trailers-only answer — a gRPC status-only reply is exactly that — into a +/// message with a body, which gRPC clients refuse. +/// +/// Three things have to hold together, and the sender is the authority on the third: no +/// first chunk was read, no trailer section is still owed, and the peer really did end its +/// half. The last one is why a `content-length: 0` request stays headers-only through this +/// proxy instead of growing an empty DATA frame it did not have. +fn head_is_the_message( + first: &Option, + early: &Option>, + from: &I, +) -> bool { + first.is_none() && !matches!(early, Some(Some(_))) && from.ended() +} + +/// One request and its answer. +/// +/// The two directions are carried **at the same time**, not one after the other, which is +/// the same shape the byte pumps have and for the same reason: a service is allowed to +/// answer before it has read the whole request — a refusal, a redirect, a `413` — and a +/// proxy that insists on finishing the upload first turns that into a stall neither end +/// can explain. +#[allow(clippy::too_many_arguments)] +async fn exchange( + ctx: &Carriage, + refusal: &Refusal, + spoken: &AtomicBool, + connection: ConnectionId, + request: http::Request, + respond: h2::server::SendResponse, + sender: &mut O, +) -> io::Result<()> { + spoken.store(true, Ordering::Relaxed); + let (parts, mut from_client) = request.into_parts(); + let head = http::Request::from_parts(parts, ()); + + // A tunnel is opaque bytes with no HTTP/1.1 message to render, so it is refused rather + // than shown to the chain as something it is not. Naming it beats a stream that hangs. + if head.method() == http::Method::CONNECT { + eprintln!( + "[warn] [h2] {}: a CONNECT stream arrived. firegex terminates HTTP/2 to show \ + the filters each exchange as HTTP/1.1, and a tunnel has no such exchange in \ + it, so the stream is refused rather than carried unread.", + ctx.client + ); + let mut respond = respond; + respond.send_reset(h2::Reason::REFUSED_STREAM); + return Ok(()); + } + + // A request that carries no body must not be shown `transfer-encoding: chunked`, which + // it never had — a filter looking for that header is usually looking for smuggling, + // and one invented by the proxy in front of it is the worst possible answer. So the + // framing is settled before the head is rendered, and settled from as little as will + // do it. + let (framing, first, early) = head_framing(&mut from_client, head.headers()) + .await + .map_err(io::Error::other)?; + let head_ends_it = head_is_the_message(&first, &early, &from_client); + + // One reconstruction for the exchange, not one per direction: a request and its + // response are one conversation, which is what they would have been on HTTP/1.1. + let tap = ctx.tap(); + let rendered = ctx.rendered(); + + let mut sessions = ChainSessions::new(connection); + let mut view = render_request(&head, &framing); + if let Some(chunk) = &first { + push_body(&mut view, chunk, &framing); + } + // A message that ended with a trailer section and no body is rendered whole here, + // terminator and trailers included: there is no later chunk to hang them off, and a + // trailer section shown to nobody is a piece of the message that travelled unread. + if let Some(trailers) = early.as_ref() { + close_body(&mut view, &framing, trailers.as_ref()); + } + if !judge(&rendered, Direction::ClientToServer, &view, &mut sessions, tap.as_ref()).await { + refusal.refuse(); + return Ok(()); + } + + // Only now does the service hear about it, in whatever version it speaks — HTTP/2 + // again where the service speaks it, HTTP/1.1 where the address said the service is a + // cleartext one. The rendering above is the same either way, which is what makes the + // second case a forward rather than a translation invented here. + let (parts, _) = head.into_parts(); + let (mut to_service, from_service) = sender + .open(http::Request::from_parts(parts, ()), head_ends_it) + .await + .map_err(io::Error::other)?; + + if let Some(chunk) = first { + let len = chunk.len(); + to_service.data(chunk).await.map_err(io::Error::other)?; + // After the forward, never before: the window that reopens is the one the service + // has actually taken the bytes off, which is what backpressure means here. + from_client.release(len).map_err(io::Error::other)?; + } + + let (asked, answered) = tokio::join!( + carry_request( + ctx, refusal, from_client, to_service, framing, sessions, tap.clone(), early, + head_ends_it, + ), + // Its own sessions, the same id: both halves of one exchange are one connection to + // a filter that keeps state, and each direction keeps its own position in it. + carry_response(ctx, refusal, spoken, connection, from_service, respond, tap.clone()), + ); + if let Some(tap) = tap { + tap.closed(); + } + asked?; + answered +} + +/// The rest of the request body, judged chunk by chunk and forwarded. +#[allow(clippy::too_many_arguments)] +async fn carry_request( + ctx: &Carriage, + refusal: &Refusal, + mut from_client: h2::RecvStream, + mut to_service: B, + framing: Framing, + mut sessions: ChainSessions, + tap: Option>, + early: Option>, + head_ended_it: bool, +) -> io::Result<()> +where + B::Error: std::error::Error + Send + Sync + 'static, +{ + // The head carried the end of the stream, so this half is already finished. Touching + // it again would be writing past the end, which `h2` reports as a user error rather + // than ignoring. + if head_ended_it { + return Ok(()); + } + let rendered = ctx.rendered(); + let trailers = match early { + // The body was over before the head was rendered, so both were shown to the chain + // together. Reading either again would be reading past the end of the stream. + Some(trailers) => trailers, + None => { + // A pattern split across two DATA frames is still a pattern, so the session + // carries on from the head. + while let Some(chunk) = Incoming::data(&mut from_client) + .await + .map_err(io::Error::other)? + { + let mut view = Vec::new(); + push_body(&mut view, &chunk, &framing); + if !judge(&rendered, Direction::ClientToServer, &view, &mut sessions, tap.as_ref()) + .await + { + refusal.refuse(); + return Ok(()); + } + let len = chunk.len(); + to_service.data(chunk).await.map_err(io::Error::other)?; + from_client.release(len).map_err(io::Error::other)?; + } + let trailers = Incoming::trailers(&mut from_client) + .await + .map_err(io::Error::other)?; + let mut tail = Vec::new(); + close_body(&mut tail, &framing, trailers.as_ref()); + if !tail.is_empty() + && !judge(&rendered, Direction::ClientToServer, &tail, &mut sessions, tap.as_ref()) + .await + { + refusal.refuse(); + return Ok(()); + } + carried(&rendered, framing, trailers) + } + }; + if let Some(trailers) = trailers { + to_service.trailers(trailers).await.map_err(io::Error::other)?; + } + to_service.finish().await.map_err(io::Error::other) +} + +/// The answer, rendered and judged the same way. +#[allow(clippy::too_many_arguments)] +async fn carry_response( + ctx: &Carriage, + refusal: &Refusal, + spoken: &AtomicBool, + connection: ConnectionId, + mut from_service: A, + mut respond: h2::server::SendResponse, + tap: Option>, +) -> io::Result<()> +where + ::Error: std::error::Error + Send + Sync + 'static, +{ + let rendered = ctx.rendered(); + let head = from_service.response().await.map_err(io::Error::other)?; + spoken.store(true, Ordering::Relaxed); + + let mut sessions = ChainSessions::new(connection); + // The answer's framing is settled the same way the request's was, and for the second + // half of the same reason: a service that sends its head and then waits — a streaming + // RPC, a long poll — must not have that head held here, or the client is waiting for + // something that is waiting for the client. + let (framing, first, early) = head_framing(&mut from_service, head.headers()) + .await + .map_err(io::Error::other)?; + let head_ends_it = head_is_the_message(&first, &early, &from_service); + + let mut view = render_response(&head, &framing); + if let Some(chunk) = &first { + push_body(&mut view, chunk, &framing); + } + if let Some(trailers) = early.as_ref() { + close_body(&mut view, &framing, trailers.as_ref()); + } + if !judge(&rendered, Direction::ServerToClient, &view, &mut sessions, tap.as_ref()).await { + refusal.refuse(); + return Ok(()); + } + + let (mut parts, _) = head.into_parts(); + parts.version = http::Version::HTTP_2; + // `end_of_stream` on the head where the head is the message: a status-only gRPC answer + // is HEADERS and nothing else, and closing it with an empty DATA frame instead would + // make it a message with a body that gRPC clients then refuse. + let mut to_client = respond + .send_response(http::Response::from_parts(parts, ()), head_ends_it) + .map_err(io::Error::other)?; + + if let Some(chunk) = first { + let len = chunk.len(); + send_body(&mut to_client, chunk, false).await.map_err(io::Error::other)?; + from_service.release(len).map_err(io::Error::other)?; + } + + if head_ends_it { + return Ok(()); + } + + let trailers = match early { + Some(trailers) => trailers, + None => { + while let Some(chunk) = Incoming::data(&mut from_service) + .await + .map_err(io::Error::other)? + { + let mut view = Vec::new(); + push_body(&mut view, &chunk, &framing); + if !judge(&rendered, Direction::ServerToClient, &view, &mut sessions, tap.as_ref()) + .await + { + refusal.refuse(); + return Ok(()); + } + let len = chunk.len(); + send_body(&mut to_client, chunk, false).await.map_err(io::Error::other)?; + from_service.release(len).map_err(io::Error::other)?; + } + let trailers = Incoming::trailers(&mut from_service) + .await + .map_err(io::Error::other)?; + let mut tail = Vec::new(); + close_body(&mut tail, &framing, trailers.as_ref()); + if !tail.is_empty() + && !judge(&rendered, Direction::ServerToClient, &tail, &mut sessions, tap.as_ref()) + .await + { + refusal.refuse(); + return Ok(()); + } + carried(&rendered, framing, trailers) + } + }; + finish(&mut to_client, trailers).map_err(io::Error::other) +} + +/// End a half of an exchange, with the trailer section it is allowed to carry. +/// +/// A message whose head already carried `end_of_stream` is finished, and touching the +/// stream again would be writing past the end of it; `h2` reports that as a user error +/// rather than ignoring it, so the emptiness is checked here rather than at each call. +fn finish( + to: &mut h2::SendStream, + trailers: Option, +) -> Result<(), h2::Error> { + match trailers { + Some(trailers) => to.send_trailers(trailers), + None => to.send_data(Bytes::new(), true), + } +} + +/// Write one piece of a body, taking the window as it opens. +/// +/// `send_data` on its own queues whatever it is given inside the connection, so a client +/// that uploads faster than the service reads would be buying memory in this process — +/// which is the resource the connection limit exists to defend. Reserving first and +/// writing what is granted is what turns the far end's window into backpressure on the +/// near one, and it is the other half of releasing capacity only after a forward. +async fn send_body( + to: &mut h2::SendStream, + mut data: Bytes, + end: bool, +) -> Result<(), h2::Error> { + if data.is_empty() { + return to.send_data(data, end); + } + while !data.is_empty() { + to.reserve_capacity(data.len()); + let granted = match std::future::poll_fn(|cx| to.poll_capacity(cx)).await { + Some(granted) => granted?, + // The far end is gone. Saying so as an error lets the caller wind the + // exchange down the way every other failure here does. + None => return to.send_data(data, end), + }; + if granted == 0 { + continue; + } + let piece = data.split_to(granted.min(data.len())); + to.send_data(piece, end && data.is_empty())?; + } + Ok(()) +} diff --git a/backend/proxysrc/src/h3.rs b/backend/proxysrc/src/h3.rs new file mode 100644 index 0000000..45ff05a --- /dev/null +++ b/backend/proxysrc/src/h3.rs @@ -0,0 +1,486 @@ +//! HTTP/3, terminated on top of QUIC — and shown to the filters as HTTP/1.1. +//! +//! The rendering itself lives in [`crate::http1`], which is where the reasoning about it +//! belongs now that HTTP/2 is shown to the chain the same way. What is left here is the +//! adapter: hyperium's `h3` parses and re-encodes the frames, this module decides when to +//! ask the chain and what to do with the answer. +//! +//! Terminating is not a choice on this layer the way it is on TCP. Past its Initial packet +//! QUIC encrypts the frames, the stream boundaries and the packet number, so a layer that +//! only forwards has nothing a filter could be shown. What goes on to the service is +//! HTTP/3 again, re-encoded — the rendering is a view, never a translation the service +//! sees. Unless the operator says the service speaks HTTP/1.1 and nothing else, which is +//! the one place that sentence stops being true and is written down where it is chosen: +//! see [`crate::h1up`]. +//! +//! One request is one connection to the chain, which is [`Carrier::open`]'s rule applied +//! to h3: a request stream carries exactly one exchange, so a filter's per-connection +//! state — for Python, a whole set of module globals — lives exactly as long as the +//! request it belongs to. + +use std::future::Future; +use std::io; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use bytes::{Buf, Bytes}; +use h3::error::StreamError; + +use crate::capture::Tap; +use crate::filter::{ChainSessions, ConnectionId, Direction}; +use crate::http1::{ + carried, close_body, head_framing, judge, push_body, render_request, render_response, Answer, + Framing, Incoming, Outbound, OutboundBody, Rendered, +}; +use crate::quic::{Behind, Carrier}; + +impl Incoming for h3::server::RequestStream +where + S: h3::quic::RecvStream + Send, + B: Buf + Send, +{ + type Error = StreamError; + + fn data(&mut self) -> impl Future, StreamError>> + Send { + async move { Ok(self.recv_data().await?.map(collect)) } + } + + fn trailers( + &mut self, + ) -> impl Future, StreamError>> + Send { + self.recv_trailers() + } +} + +impl Incoming for h3::client::RequestStream +where + S: h3::quic::RecvStream + Send, + B: Buf + Send, +{ + type Error = StreamError; + + fn data(&mut self) -> impl Future, StreamError>> + Send { + async move { Ok(self.recv_data().await?.map(collect)) } + } + + fn trailers( + &mut self, + ) -> impl Future, StreamError>> + Send { + self.recv_trailers() + } +} + +/// How this connection's exchanges are described to the shared rendering. +fn rendered<'a>(carrier: &'a Carrier) -> Rendered<'a> { + Rendered { + chain: &carrier.chain, + stats: &carrier.stats, + client: carrier.client, + layer: "h3", + } +} + +/// The two halves of one h3 request stream, as the shared rendering wants them. +type H3Send = h3::client::RequestStream< + as h3::quic::BidiStream>::SendStream, + Bytes, +>; +type H3Recv = h3::client::RequestStream< + as h3::quic::BidiStream>::RecvStream, + Bytes, +>; + +/// HTTP/3 towards the service, which is what every HTTP/3 client edge used to get. +impl Outbound for h3::client::SendRequest { + type Error = StreamError; + type Body = H3Send; + type Answer = H3Recv; + + async fn open( + &mut self, + request: http::Request<()>, + // HTTP/3 frames the end of a message on the stream, so nothing has to be said on + // the head: the body half is finished by the caller when there is none, exactly + // as it is when there is one. The flag is HTTP/2's problem. + _ends_it: bool, + ) -> Result<(Self::Body, Self::Answer), StreamError> { + // The parts travel as they arrived; only the version is restated, because what + // goes out is HTTP/3 again. + let (mut parts, _) = request.into_parts(); + parts.version = http::Version::HTTP_3; + let stream = self + .send_request(http::Request::from_parts(parts, ())) + .await?; + let (send, recv) = stream.split(); + Ok((send, recv)) + } +} + +impl OutboundBody for H3Send { + type Error = StreamError; + + async fn data(&mut self, chunk: Bytes) -> Result<(), StreamError> { + self.send_data(chunk).await + } + + async fn trailers(&mut self, trailers: http::HeaderMap) -> Result<(), StreamError> { + self.send_trailers(trailers).await + } + + async fn finish(&mut self) -> Result<(), StreamError> { + h3::client::RequestStream::finish(self).await + } +} + +impl Answer for H3Recv { + async fn response(&mut self) -> Result, StreamError> { + self.recv_response().await + } +} + +/// Carry one HTTP/3 connection: accept requests, and forward each one. +/// +/// Where they are forwarded *to* is the operator's choice, and the only thing that +/// changes with it is which sender the exchanges are handed. Everything the chain is +/// shown is built by [`crate::http1`] either way — which is the whole reason the choice +/// could be offered at all. +pub(crate) async fn carry(carrier: &Carrier) { + // **No GREASE, in either direction.** The reserved frame and stream types exist so + // that implementations keep their handling of unknown extensions honest, which is a + // worthy thing for a browser to do and is not firegex's to do on somebody's service. + // Sending them means putting a frame on the wire that the client never sent, and the + // first real server this was pointed at — aioquic — stopped answering requests + // entirely because of it: the trailing grease frame left its end-of-stream + // unreported, so the request was received and never completed. A firewall that + // breaks the service it is protecting in order to exercise that service's protocol + // handling has its priorities backwards. + let mut server = match h3::server::builder() + .send_grease(false) + .build(h3_quinn::Connection::new(carrier.peer.clone())) + .await + { + Ok(connection) => connection, + Err(e) => { + eprintln!("[info] [h3] the client's HTTP/3 connection did not start: {e}"); + return; + } + }; + + match carrier.service.clone() { + Behind::Quic(connection) => { + let (mut driver, sender) = match h3::client::builder() + .send_grease(false) + .build(h3_quinn::Connection::new(connection)) + .await + { + Ok(pair) => pair, + Err(e) => { + eprintln!("[warn] [h3] {} does not speak HTTP/3: {e}", carrier.upstream); + carrier.peer.close(0u32.into(), b"the service did not speak HTTP/3"); + return; + } + }; + // The client half of h3 needs something to run its connection: control + // streams, settings, and the QPACK streams underneath every header block. + // Without this task nothing upstream ever completes. + let driving = tokio::spawn(async move { driver.wait_idle().await.to_string() }); + accept_loop(carrier, &mut server, sender, Some(driving)).await; + } + // A service that speaks HTTP/1.1 has no connection to run and nothing to keep + // alive between exchanges: each one dials its own, so there is no driver here + // and nothing for the loop to watch but the client. + Behind::Http1(upstream) => accept_loop(carrier, &mut server, upstream, None).await, + } +} + +/// Take requests until the client is done, and let each one run on its own. +async fn accept_loop( + carrier: &Carrier, + server: &mut h3::server::Connection, + sender: O, + driving: Option>, +) where + O: Outbound + Clone + Send + 'static, +{ + // The requests already in flight, kept rather than detached. A client winding the + // connection down — a GOAWAY, or simply no more requests — leaves `accept()` + // answering `None` while answers are still owed, and walking away there would cut + // off the last exchange of every well-behaved client. + let mut inflight = tokio::task::JoinSet::new(); + let mut driving = driving; + + loop { + let accepted = tokio::select! { + accepted = server.accept() => accepted, + // The service ended its side. Waiting for the client to notice would mean + // waiting out the idle timeout with a connection that has nowhere to go. + // With an HTTP/1.1 service there is no such side to watch — its connections + // live and die with one exchange — so this arm simply never fires. + reason = async { + match driving.as_mut() { + Some(driver) => driver.await.ok(), + None => std::future::pending().await, + } + } => { + if let Some(reason) = reason { + eprintln!("[info] [h3] the service ended the connection: {reason}"); + } + carrier.peer.close(0u32.into(), b"the service closed the connection"); + break; + } + }; + match accepted { + Ok(Some(resolver)) => { + let carrier = carrier.clone(); + let mut sender = sender.clone(); + inflight.spawn(async move { + let connection = carrier.open(); + let outcome = exchange(&carrier, connection, resolver, &mut sender).await; + // Whatever happened, the filter's state for this request goes — a + // refused request and a completed one both end it. + carrier.chain.current().connection_closed(connection); + if let Err(e) = outcome { + eprintln!("[info] [h3] request ended: {e}"); + } + }); + } + // The client is done with the connection, which is the ordinary way out. + Ok(None) => break, + Err(e) => { + eprintln!("[info] [h3] the client's connection ended: {e}"); + break; + } + } + } + // Nothing here is unbounded: each exchange rides on connections with an idle + // timeout, and every filter it consults runs under the chain's own deadline. + while inflight.join_next().await.is_some() {} + if let Some(driving) = driving { + driving.abort(); + } +} + +/// One request and its answer. +/// +/// The two directions are carried **at the same time**, not one after the other, which +/// is the same shape the TCP pumps have and for the same reason: a service is allowed to +/// answer before it has read the whole request — a refusal, a redirect, a `413` — and a +/// proxy that insists on finishing the upload first turns that into a stall neither end +/// can explain. +/// +/// Generic over where it is going, and over nothing else: what the chain is shown is the +/// same rendering whatever answers, which is the promise the whole module makes. +async fn exchange( + carrier: &Carrier, + connection: ConnectionId, + resolver: h3::server::RequestResolver, + sender: &mut O, +) -> io::Result<()> { + let (request, mut from_client) = resolver.resolve_request().await.map_err(io::Error::other)?; + carrier.spoken.store(true, Ordering::Relaxed); + + // A request that carries no body must not be shown `transfer-encoding: chunked`, + // which it never had — a filter looking for that header is usually looking for + // smuggling, and one invented by the proxy in front of it is the worst possible + // answer. So the framing is settled before the head is rendered, and settled from + // as little as will do it. + let (framing, first, early) = head_framing(&mut from_client, request.headers()) + .await + .map_err(io::Error::other)?; + + // One reconstruction for the exchange, not one per direction: a request and its + // response are one conversation, which is what they would have been on TCP. + let tap = carrier.tap(); + let rendered = rendered(carrier); + + // This direction's filter state, and the head as the chain will see it. + let mut sessions = ChainSessions::new(connection); + let mut view = render_request(&request, &framing); + if let Some(chunk) = &first { + push_body(&mut view, chunk, &framing); + } + // A message that ended with a trailer section and no body is rendered whole here, + // terminator and trailers included: there is no later chunk to hang them off, and a + // trailer section shown to nobody is a piece of the message that travelled unread. + if let Some(trailers) = early.as_ref() { + close_body(&mut view, &framing, trailers.as_ref()); + } + if !judge(&rendered, Direction::ClientToServer, &view, &mut sessions, tap.as_ref()).await { + carrier.refuse(); + return Ok(()); + } + + // Only now does the service hear about it, in whatever version it speaks. + let (parts, _) = request.into_parts(); + // Whether this head is the whole message. On HTTP/3 the request stream says so the + // way HTTP/2 does, and the flag only matters to an upstream that has to state it on + // the head — but it is computed from the same three answers wherever it is asked. + let ends_it = first.is_none() && !matches!(early, Some(Some(_))) && from_client.ended(); + let (mut to_service, from_service) = sender + .open(http::Request::from_parts(parts, ()), ends_it) + .await + .map_err(io::Error::other)?; + if let Some(chunk) = first { + to_service.data(chunk).await.map_err(io::Error::other)?; + } + + let (to_client, from_client) = from_client.split(); + + let (asked, answered) = tokio::join!( + carry_request(carrier, from_client, to_service, framing, sessions, tap.clone(), early), + // Its own sessions, the same id: both halves of one exchange are one connection + // to a filter that keeps state, and each direction keeps its own position in it. + carry_response(carrier, connection, from_service, to_client, tap.clone()), + ); + if let Some(tap) = tap { + tap.closed(); + } + asked?; + answered +} + +/// The rest of the request body, judged chunk by chunk and forwarded. +#[allow(clippy::too_many_arguments)] +async fn carry_request( + carrier: &Carrier, + mut from_client: h3::server::RequestStream< + as h3::quic::BidiStream>::RecvStream, + Bytes, + >, + mut to_service: B, + framing: Framing, + mut sessions: ChainSessions, + tap: Option>, + early: Option>, +) -> io::Result<()> +where + B::Error: std::error::Error + Send + Sync + 'static, +{ + let rendered = rendered(carrier); + let trailers = match early { + // The body was over before the head was rendered, so both were shown to the chain + // together. Reading either again would be reading past the end of the stream. + Some(trailers) => trailers, + None => { + // A pattern split across two DATA frames is still a pattern, so the session + // carries on from the head. + while let Some(chunk) = from_client.data().await.map_err(io::Error::other)? { + let mut view = Vec::new(); + push_body(&mut view, &chunk, &framing); + if !judge(&rendered, Direction::ClientToServer, &view, &mut sessions, tap.as_ref()) + .await + { + carrier.refuse(); + return Ok(()); + } + to_service.data(chunk).await.map_err(io::Error::other)?; + } + let trailers = from_client.trailers().await.map_err(io::Error::other)?; + let mut tail = Vec::new(); + close_body(&mut tail, &framing, trailers.as_ref()); + if !tail.is_empty() + && !judge(&rendered, Direction::ClientToServer, &tail, &mut sessions, tap.as_ref()) + .await + { + carrier.refuse(); + return Ok(()); + } + carried(&rendered, framing, trailers) + } + }; + if let Some(trailers) = trailers { + to_service + .trailers(trailers) + .await + .map_err(io::Error::other)?; + } + to_service.finish().await.map_err(io::Error::other) +} + +/// The answer, rendered and judged the same way. +async fn carry_response( + carrier: &Carrier, + connection: ConnectionId, + mut from_service: A, + mut to_client: h3::server::RequestStream< + as h3::quic::BidiStream>::SendStream, + Bytes, + >, + tap: Option>, +) -> io::Result<()> +where + ::Error: std::error::Error + Send + Sync + 'static, +{ + let rendered = rendered(carrier); + let response = from_service.response().await.map_err(io::Error::other)?; + let mut sessions = ChainSessions::new(connection); + // The answer's framing is settled the same way the request's was, and for the second + // half of the same reason: a service that sends its head and then waits — a streaming + // RPC, a long poll — must not have that head held here, or the client is waiting for + // something that is waiting for the client. + let (framing, first, early) = head_framing(&mut from_service, response.headers()) + .await + .map_err(io::Error::other)?; + + let mut view = render_response(&response, &framing); + if let Some(chunk) = &first { + push_body(&mut view, chunk, &framing); + } + if let Some(trailers) = early.as_ref() { + close_body(&mut view, &framing, trailers.as_ref()); + } + if !judge(&rendered, Direction::ServerToClient, &view, &mut sessions, tap.as_ref()).await { + carrier.refuse(); + return Ok(()); + } + + let (mut parts, _) = response.into_parts(); + parts.version = http::Version::HTTP_3; + to_client + .send_response(http::Response::from_parts(parts, ())) + .await + .map_err(io::Error::other)?; + if let Some(chunk) = first { + to_client.send_data(chunk).await.map_err(io::Error::other)?; + } + let trailers = match early { + Some(trailers) => trailers, + None => { + while let Some(chunk) = from_service.data().await.map_err(io::Error::other)? { + let mut view = Vec::new(); + push_body(&mut view, &chunk, &framing); + if !judge(&rendered, Direction::ServerToClient, &view, &mut sessions, tap.as_ref()) + .await + { + carrier.refuse(); + return Ok(()); + } + to_client.send_data(chunk).await.map_err(io::Error::other)?; + } + let trailers = from_service.trailers().await.map_err(io::Error::other)?; + let mut tail = Vec::new(); + close_body(&mut tail, &framing, trailers.as_ref()); + if !tail.is_empty() + && !judge(&rendered, Direction::ServerToClient, &tail, &mut sessions, tap.as_ref()) + .await + { + carrier.refuse(); + return Ok(()); + } + carried(&rendered, framing, trailers) + } + }; + if let Some(trailers) = trailers { + to_client + .send_trailers(trailers) + .await + .map_err(io::Error::other)?; + } + to_client.finish().await.map_err(io::Error::other) +} + +/// One DATA frame's bytes, however many pieces the parser had them in. +fn collect(mut buf: impl Buf) -> Bytes { + let len = buf.remaining(); + buf.copy_to_bytes(len) +} diff --git a/backend/proxysrc/src/http1.rs b/backend/proxysrc/src/http1.rs new file mode 100644 index 0000000..dd18ff9 --- /dev/null +++ b/backend/proxysrc/src/http1.rs @@ -0,0 +1,393 @@ +//! The HTTP/1.1 view every terminated HTTP protocol is shown to the chain as. +//! +//! A filter written against `HttpRequest` is a filter written once. The same file is +//! supposed to work on every layer, and the library decides when to call it from what its +//! parameters are annotated with — not from what the service happens to speak. HTTP/2 +//! puts the method, the path and the headers in an HPACK-compressed HEADERS frame and +//! HTTP/3 in a QPACK one, so a filter handed the bytes of either would be handed a +//! compression format, and every pattern written for the HTTP/1.1 service beside it would +//! silently stop matching. +//! +//! So each exchange is rendered as the HTTP/1.1 message it would have been, and that is +//! what the chain is shown. `HttpRequest`, `HttpResponse` and a hyperscan pattern all mean +//! the same thing on every version, and nothing had to learn a new protocol. The same +//! bargain the capture interface makes: **the bytes are real and the framing is +//! reconstructed**. Concretely, and this belongs wherever this is documented — +//! +//! * the request line says `HTTP/1.1`, because there is no other version an HTTP/1 parser +//! will read, and the version on the wire was not it; +//! * a body with no `content-length` is shown chunked, which is what that message is in +//! HTTP/1.1 — and the head is held until it is known whether a body is coming, so that a +//! request without one is not shown a framing header it never had. Held *briefly*: a +//! declared length settles the question outright and waits for nothing, a message with +//! no body has already ended its stream, and what is left is a sender holding the stream +//! open, which [`HEAD_HOLD`] ends rather than deadlocking an exchange the service is +//! supposed to speak first in; +//! * a **trailer section is always shown to the chain**, which is what decides the framing +//! of a message that has one and no body: in HTTP/1.1 a trailer section belongs to a +//! chunked message and nowhere else. The one message that cannot carry one in the +//! rendering is one that declared its length, and there the trailers are dropped rather +//! than forwarded past a filter that never saw them; +//! * `host` is rendered from the `:authority` pseudo-header, which is where both HTTP/2 +//! and HTTP/3 put it; +//! * hop-by-hop headers do not appear, because both forbid sending them. +//! +//! A message whose declared `content-length` disagrees with the body it then sends is +//! malformed in HTTP/2 and HTTP/3 as well, and nothing here tries to repair it: the frames +//! are forwarded as they came, and the rendering carries the same disagreement — which an +//! HTTP/1 parser reads as a message that does not end where it said it would. That is what +//! the library's invalid-encoding action is for, and it refuses by default, which is the +//! same answer the same traffic gets over HTTP/1.1. +//! +//! What it is *not* is a translation the service ever sees: upstream, the message goes +//! back out in the version it arrived in, re-encoded by the same library that parsed it. +//! +//! **The capture interface is shown this same rendering**, one TCP conversation per +//! exchange, which is the only way `firegex0` can keep the promise it makes everywhere +//! else: what is written there is what the filters saw. It is not what left this process +//! towards the service, so the rendering is a view in the capture exactly as it is a view +//! to a filter, and the two cannot disagree because they are the same bytes. +//! +//! # Why this module exists at all +//! +//! It was inside `h3.rs`, and HTTP/2 wanted every line of it. Copying it would have been +//! two renderings that can come to disagree — the same argument the [`Incoming`] trait +//! already makes *within* one protocol, one level up. The day they diverge the symptom is +//! a pattern that matches over HTTP/3 and not over HTTP/2, silently, which is the failure +//! this whole engine is arranged against. So the rendering lives here once, and +//! [`crate::h2`] and [`crate::h3`] are what is left of each protocol after it is taken +//! out: an adapter to somebody else's parser. + +use std::future::Future; +use std::net::SocketAddr; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; + +use crate::capture::Tap; +use crate::filter::{ChainHandle, ChainSessions, Direction, Verdict}; +use crate::proxy::ProxyStats; + +/// How the body of one message is framed in the view the filters are shown. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Framing { + /// The message said how long it is, so the body is shown as it arrives. + Sized, + /// It did not, so it is shown chunked — the HTTP/1.1 spelling of that. + Chunked, + /// There is no body at all, and nothing is added to say so. + Empty, +} + +/// How long the head is held while it is still unknown whether a body follows. +/// +/// Only ever paid by a message that declared no length *and* has not yet said whether it +/// has a body — which is to say, one whose sender is holding the stream open. A message +/// with a `content-length` is rendered from its head alone and waits for nothing; one +/// without a body has already finished its stream, so the answer is there before this +/// timer starts. +/// +/// It exists because holding the head indefinitely deadlocks an exchange the service is +/// supposed to speak first in: a bidirectional RPC — gRPC's among them — sends a head and +/// then waits to be answered, and a proxy that will not forward the head until a body +/// arrives is waiting for something that is waiting for it. When it expires the message is +/// rendered chunked, which is what a message of undeclared length *is* in HTTP/1.1, so +/// nothing is invented that the sender did not already imply. +pub(crate) const HEAD_HOLD: Duration = Duration::from_millis(100); + +/// Everything the rendering needs about who is being carried, and no more. +/// +/// Neither protocol's connection type appears here on purpose: what this module does is +/// the same on both, and a parameter that differed would be the seam along which the two +/// renderings could start to drift again. +pub(crate) struct Rendered<'a> { + pub(crate) chain: &'a ChainHandle, + pub(crate) stats: &'a Arc, + pub(crate) client: SocketAddr, + /// `h2` or `h3`, for the one log line this module writes. + pub(crate) layer: &'static str, +} + +/// The receiving half of one direction of an exchange. +/// +/// Four types end up here — a server and a client request stream on each of two crates — +/// and nothing unifies them, while every question this module asks a message is the same +/// on all of them. One trait rather than the same framing decision written four times: the +/// copies are renderings that can come to disagree, and the whole point of the rendering +/// is that a filter sees one thing. +pub(crate) trait Incoming { + /// Whatever the underlying crate says went wrong. Never inspected here — the caller + /// owns the protocol and its errors, this module owns the framing. + type Error; + + fn data(&mut self) -> impl Future, Self::Error>> + Send; + fn trailers( + &mut self, + ) -> impl Future, Self::Error>> + Send; + + /// Give back the window one piece took, once it has been forwarded and not before. + /// + /// HTTP/2 is the only protocol here that asks: it hands received bytes over and waits + /// for them to be released, so releasing on *read* would mean no backpressure and + /// this process buying memory for whichever side is faster, while never releasing + /// stalls the stream after its first window. Everything else frames its own flow and + /// answers with the default. + fn release(&mut self, taken: usize) -> Result<(), Self::Error> { + let _ = taken; + Ok(()) + } + + /// Whether the sender has already said there is nothing more coming. + /// + /// Only meaningful beside a head that produced no body and no trailer section: the + /// three together are what decide whether the head **is** the message, which HTTP/2 + /// has to state on the head itself (see [`Outbound::open`]). A protocol whose sender + /// cannot say it in advance answers `false`, which is always safe — the message is + /// then framed the ordinary way and ends when its body does. + fn ended(&self) -> bool { + false + } +} + +/// Where a rendered message is sent once the chain has agreed to it. +/// +/// The mirror of [`Incoming`], and it exists for the same reason one level further out: +/// what leaves this engine towards the service used to be, always, the protocol the +/// client arrived in — so the sending half could be written against one crate's concrete +/// types. It is a choice now (`FGEX_PROXY_UPSTREAM`), because a service that speaks +/// HTTP/1.1 can be put behind an HTTP/3 edge, and the alternative to this trait was a +/// second copy of the orchestration in [`crate::h3`] — request in one direction, answer +/// in the other, framing decided on both — which is the drift this module exists to +/// prevent, one level down. +/// +/// Note what is *not* here: nothing about rendering. The bytes a filter sees are built by +/// this module whatever the service turns out to speak, so an implementation of this +/// trait only has to know how to put a head, a body and a trailer section on the wire. +pub(crate) trait Outbound: Send { + type Error: std::error::Error + Send + Sync + 'static; + /// Where this exchange's body is written. + type Body: OutboundBody + Send; + /// Where its answer comes back. + type Answer: Answer + Send; + + /// Send the head, and take the two halves the rest of the exchange runs on. + /// + /// `ends_it` says the head **is** the whole message. It is a parameter rather than a + /// later `finish()` because not every protocol can say it afterwards: on HTTP/2 the + /// end of a message rides on its last frame, so a head sent open and then closed with + /// an empty DATA frame is a message *with a body* — which is exactly what a + /// status-only gRPC answer is not, and what gRPC clients refuse. HTTP/1.1 frames it + /// from the headers and has nothing to do with the flag but close the body early. + fn open( + &mut self, + request: http::Request<()>, + ends_it: bool, + ) -> impl Future> + Send; +} + +/// The body of a message on its way to the service. +/// +/// Imperative rather than a stream handed over whole, because that is the shape both ends +/// of this already have: the chain judges one piece at a time and forwards what it +/// accepts, so a piece is written when it is allowed and not before. +pub(crate) trait OutboundBody: Send { + type Error; + + fn data(&mut self, chunk: Bytes) -> impl Future> + Send; + fn trailers( + &mut self, + trailers: http::HeaderMap, + ) -> impl Future> + Send; + /// No more of it is coming. Separate from dropping, because on some protocols the + /// end of a message is a frame and not the absence of one. + fn finish(&mut self) -> impl Future> + Send; +} + +/// The service's answer: its head first, then the body every [`Incoming`] gives. +pub(crate) trait Answer: Incoming + Send { + fn response( + &mut self, + ) -> impl Future, ::Error>> + Send; +} + +/// What has to be known before a message's head can be rendered, and no more than that. +/// +/// Three answers come back together because they are one decision: the framing, the first +/// body chunk if reading it is what settled the question, and — where the body was over +/// before it began — the trailer section, which is already in hand by then and which +/// decides the framing too. A message that ends with trailers and no body *is* a chunked +/// message in HTTP/1.1, and that is the only place a trailer section can be rendered, so +/// it has to be known here rather than discovered after the head has gone out. +/// +/// `Some(trailers)` in the third slot means the body stream is finished and its trailer +/// section has already been taken; the caller must not read either again. +#[allow(clippy::type_complexity)] +pub(crate) async fn head_framing( + body: &mut B, + headers: &http::HeaderMap, +) -> Result<(Framing, Option, Option>), B::Error> { + // A declared length frames the body by itself, so there is nothing to wait for and + // nothing a later frame could change. This is also what keeps an upload that the + // service answers early — a `413`, a redirect, a refusal — moving in both directions. + if headers.contains_key(http::header::CONTENT_LENGTH) { + return Ok((Framing::Sized, None, None)); + } + // Both crates keep a half-read frame's state in the stream rather than in the future, + // so letting the deadline drop it loses nothing. + match tokio::time::timeout(HEAD_HOLD, body.data()).await { + Ok(first) => match first? { + Some(chunk) => Ok((Framing::Chunked, Some(chunk), None)), + None => { + // The body is over. Whatever ended it — the end of the stream or a + // trailer section — has already been read into the stream, so this + // resolves without waiting for anything. + let trailers = body.trailers().await?; + let framing = if trailers.is_some() { + Framing::Chunked + } else { + Framing::Empty + }; + Ok((framing, None, Some(trailers))) + } + }, + Err(_) => Ok((Framing::Chunked, None, None)), + } +} + +/// The trailer section this engine is willing to forward. +/// +/// A trailer section the chain was never shown is a piece of the message that travelled +/// unfiltered, which is the one thing nothing else here does. Only one framing leaves +/// nowhere to render it: a message that declared its length, whose HTTP/1.1 view ends +/// where the header said it would — anything appended after that is not part of the +/// message any parser reading the rendering would see. So it is dropped, which is what an +/// intermediary is allowed to do with a trailer section it cannot carry, and said out loud +/// rather than done quietly. +pub(crate) fn carried( + rendered: &Rendered<'_>, + framing: Framing, + trailers: Option, +) -> Option { + if framing == Framing::Sized && trailers.is_some() { + eprintln!( + "[warn] [{}] {}: a trailer section arrived on a message that declared its \ + length. The HTTP/1.1 the filters are shown has nowhere to put one, so it is \ + dropped rather than forwarded unread.", + rendered.layer, rendered.client, + ); + return None; + } + trailers +} + +/// Ask the chain about one piece of the rendered message. +pub(crate) async fn judge( + rendered: &Rendered<'_>, + direction: Direction, + view: &[u8], + sessions: &mut ChainSessions, + tap: Option<&Arc>, +) -> bool { + // Re-read every time, so a ruleset pushed mid-request takes effect without anybody + // losing their connection — the same rule the byte pumps follow. + match rendered + .chain + .current() + .run(direction, view, sessions) + .await + { + Verdict::Accept => { + // Written on the way past, and only once the chain has agreed: what a capture + // shows has to be what was let through, or it is a record of a decision that + // was not taken. A refused piece leaves the conversation ending where the + // traffic did, which is the truth about it. + if let Some(tap) = tap { + tap.wrote(direction == Direction::ClientToServer, view); + } + true + } + Verdict::Reject(_) => { + rendered + .stats + .closed_by_filter + .fetch_add(1, Ordering::Relaxed); + false + } + } +} + +/// The request, as the HTTP/1.1 it would have been. +pub(crate) fn render_request(request: &http::Request<()>, framing: &Framing) -> Vec { + let target = request + .uri() + .path_and_query() + .map(|p| p.as_str().to_string()) + .unwrap_or_else(|| "/".to_string()); + let mut view = format!("{} {} HTTP/1.1\r\n", request.method(), target).into_bytes(); + // Both versions carry it as `:authority`, and an HTTP/1.1 parser wants a `Host` line. + // A request that also sent `host` outright keeps its own. + if !request.headers().contains_key(http::header::HOST) { + if let Some(authority) = request.uri().authority() { + view.extend_from_slice(format!("host: {authority}\r\n").as_bytes()); + } + } + render_headers(&mut view, request.headers(), framing); + view +} + +/// And the response. +pub(crate) fn render_response(response: &http::Response<()>, framing: &Framing) -> Vec { + let status = response.status(); + let mut view = format!( + "HTTP/1.1 {} {}\r\n", + status.as_u16(), + status.canonical_reason().unwrap_or("") + ) + .into_bytes(); + render_headers(&mut view, response.headers(), framing); + view +} + +fn render_headers(view: &mut Vec, headers: &http::HeaderMap, framing: &Framing) { + for (name, value) in headers { + view.extend_from_slice(name.as_str().as_bytes()); + view.extend_from_slice(b": "); + view.extend_from_slice(value.as_bytes()); + view.extend_from_slice(b"\r\n"); + } + if matches!(framing, Framing::Chunked) { + view.extend_from_slice(b"transfer-encoding: chunked\r\n"); + } + view.extend_from_slice(b"\r\n"); +} + +/// One body frame, framed the way the head said it would be. +pub(crate) fn push_body(view: &mut Vec, chunk: &[u8], framing: &Framing) { + match framing { + Framing::Chunked => { + view.extend_from_slice(format!("{:x}\r\n", chunk.len()).as_bytes()); + view.extend_from_slice(chunk); + view.extend_from_slice(b"\r\n"); + } + _ => view.extend_from_slice(chunk), + } +} + +/// The end of the body, where the framing has one to show. +pub(crate) fn close_body(view: &mut Vec, framing: &Framing, trailers: Option<&http::HeaderMap>) { + if !matches!(framing, Framing::Chunked) { + return; + } + view.extend_from_slice(b"0\r\n"); + if let Some(trailers) = trailers { + for (name, value) in trailers { + view.extend_from_slice(name.as_str().as_bytes()); + view.extend_from_slice(b": "); + view.extend_from_slice(value.as_bytes()); + view.extend_from_slice(b"\r\n"); + } + } + view.extend_from_slice(b"\r\n"); +} diff --git a/backend/proxysrc/src/lib.rs b/backend/proxysrc/src/lib.rs index 47b01b0..7bae667 100644 --- a/backend/proxysrc/src/lib.rs +++ b/backend/proxysrc/src/lib.rs @@ -8,14 +8,20 @@ pub mod control; pub mod debug; pub mod filter; +pub mod h1up; +pub mod h2; +pub mod h3; +pub mod http1; pub mod hyperscan; pub mod proxy; +pub mod quic; pub mod pyworker; pub mod rules; pub mod spec; pub mod capture; pub mod tls; pub mod transparent; +pub mod relays; pub mod udp; pub use filter::{ diff --git a/backend/proxysrc/src/main.rs b/backend/proxysrc/src/main.rs index e77bf9f..d703d67 100644 --- a/backend/proxysrc/src/main.rs +++ b/backend/proxysrc/src/main.rs @@ -6,6 +6,8 @@ use std::time::Duration; use fgex_proxy::control::serve_stdin; use fgex_proxy::filter::{ChainHandle, FilterChain}; use fgex_proxy::proxy::{Proxy, ProxyConfig, TlsSetup, Upstream}; +use fgex_proxy::quic::{QuicConfig, QuicManager, QuicSetup}; +use fgex_proxy::relays::Relays; use fgex_proxy::spec::parse_filters; use fgex_proxy::transparent::probe_capability; @@ -47,26 +49,67 @@ fn env_usize(key: &str) -> Option { std::env::var(key).ok()?.trim().parse().ok() } -fn build_tls() -> Result { +/// The certificate and key this service terminates with, read once. +/// +/// Read rather than passed: a certificate on a command line or in an environment dump is +/// a certificate in a log somewhere. Both edges that need it — TLS over TCP and QUIC — +/// ask for the same pair, so it is read here and handed to whichever wants it. +fn read_material() -> Result, String> { let cert_path = std::env::var("FGEX_PROXY_TLS_CERT").ok(); let key_path = std::env::var("FGEX_PROXY_TLS_KEY").ok(); - let server = match (cert_path, key_path) { + match (cert_path, key_path) { (Some(cert), Some(key)) => { - let cert = + let certificate = std::fs::read_to_string(&cert).map_err(|e| format!("cannot read {cert}: {e}"))?; let key = std::fs::read_to_string(&key).map_err(|e| format!("cannot read the key: {e}"))?; - Some(fgex_proxy::tls::server_config(&cert, &key)?) + Ok(Some((certificate, key))) } - (None, None) => None, - _ => return Err("FGEX_PROXY_TLS_CERT and FGEX_PROXY_TLS_KEY go together".to_string()), - }; - let upstream = if env_flag("FGEX_PROXY_TLS_UPSTREAM") { - Some(fgex_proxy::tls::client_config()?) + (None, None) => Ok(None), + _ => Err("FGEX_PROXY_TLS_CERT and FGEX_PROXY_TLS_KEY go together".to_string()), + } +} + +/// The protocols offered to the service, in the operator's order. +/// +/// Offered to the *service*, which is what makes the list needed at all: the client's own +/// list arrives inside an encrypted Initial and cannot be read before answering, so this +/// is what firegex has to go on. `h3` alone by default, because that is what a QUIC +/// service is nine times in ten; anything else — a game protocol, a CTF's own — is named +/// here or its handshake fails saying so. +fn quic_alpn() -> Vec> { + let raw = std::env::var("FGEX_PROXY_QUIC_ALPN").unwrap_or_default(); + let listed: Vec> = raw + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| s.as_bytes().to_vec()) + .collect(); + if listed.is_empty() { + vec![b"h3".to_vec()] } else { - None + listed + } +} + +fn build_tls(material: Option<&(String, String)>, optional: bool) -> Result { + let server = match material { + Some((cert, key)) => Some(fgex_proxy::tls::server_config(cert, key)?), + None => None, }; - Ok(TlsSetup { server, upstream }) + // Built whenever this engine terminates anything, and *used* per address: whether a + // given connection is re-encrypted on the way out is the address's answer + // (`Onward`), not the process's. A plain TCP service terminates nothing, so it has + // no client configuration and never did — which is why none of this reaches it. + let upstream = match server { + Some(_) => Some(fgex_proxy::tls::client_config()?), + None => None, + }; + Ok(TlsSetup { + server, + upstream, + optional, + }) } /// Answer one regex-debug request and exit, instead of starting a datapath. @@ -178,9 +221,38 @@ async fn run() { } }; - // TLS terminated here rather than by an nginx in front. Rules see plaintext that - // never leaves the process. - let tls = match build_tls() { + // Read before either edge is built: TLS over TCP and QUIC terminate with the same + // certificate, and a service speaks one of them or the other. + let material = match read_material() { + Ok(material) => material, + Err(e) => { + eprintln!("[fatal] [main] {e}"); + exit(2); + } + }; + + // QUIC is UDP on the wire, so it takes the place of the datagram relays rather than + // sitting beside them: the same map of one port per protected address, bound by + // something that terminates instead of something that forwards. + let quic = env_flag("FGEX_PROXY_QUIC"); + + // Whether the TCP listener terminates TLS, and whether it insists on it. + // + // `FGEX_PROXY_TLS_OPTIONAL` is what an `http` service sets: one certificate, one + // chain, and each connection carried as whatever the client opened it with — in the + // clear on one port and under TLS on another. It also means TLS on TCP and QUIC on + // UDP live in **one** engine process, which they could not before: the two used to be + // exclusive here, on the reasoning that a service speaks one or the other, and an + // HTTP service speaks both. + let tls_optional = env_flag("FGEX_PROXY_TLS_OPTIONAL"); + // Absent, the old rule: a certificate means terminate, unless this is a QUIC service, + // where the certificate belongs to the QUIC edge and the TCP listener has nothing + // pointed at it. Said explicitly by anything that wants both. + let terminate_tls = env_flag("FGEX_PROXY_TLS") || tls_optional || !quic; + let tls = match build_tls( + if terminate_tls { material.as_ref() } else { None }, + tls_optional, + ) { Ok(setup) => setup, Err(e) => { eprintln!("[fatal] [main] {e}"); @@ -208,15 +280,82 @@ async fn run() { secs => Some(Duration::from_secs(secs as u64)), }; + // Opened once for the process and shared by every path that reconstructs something: + // the interface belongs to the instance, not to a service or a connection. + let capture = fgex_proxy::capture::Capture::open(); + + // Where each published address sends its traffic, for the ones this engine starts + // with. `10.0.0.1:443=10.0.0.1:80` reads as "what arrives at the first is the + // service at the second" — an address published on a port the service does not + // listen on. Absent, and for every address not named, the answer is the address + // itself, which is what transparent means and what every service had before. + let targets = std::sync::Arc::new(fgex_proxy::proxy::Targets::default()); + if let Ok(spec) = std::env::var("FGEX_PROXY_TARGETS") { + for entry in spec.split(',').map(str::trim).filter(|s| !s.is_empty()) { + // `10.0.0.1:443|tls=10.0.0.1:80`, and every part after the address optional: + // the edge says what is spoken there and the target where the service is, + // and an address that says neither is one this engine is simply in front of. + let (address, target) = match entry.split_once('=') { + Some((address, target)) => ( + address, + Some(parse_addr("FGEX_PROXY_TARGETS", target.trim())), + ), + None => (entry, None), + }; + // `
||`, both words optional and both defaulting to + // the answer every address gave before there was a question. + let mut words = address.split('|'); + let address = words.next().unwrap_or(address); + let edge = match words.next() { + Some("tls") => fgex_proxy::proxy::Edge::Tls, + None | Some("clear") | Some("any") => fgex_proxy::proxy::Edge::Whatever, + Some(other) => { + eprintln!("[fatal] [main] FGEX_PROXY_TARGETS: unknown edge '{other}'"); + exit(2); + } + }; + let upstream = match words.next() { + Some("plain") => fgex_proxy::proxy::Onward::Plain, + Some("tls") => fgex_proxy::proxy::Onward::Tls, + None | Some("same") => fgex_proxy::proxy::Onward::Same, + Some(other) => { + eprintln!("[fatal] [main] FGEX_PROXY_TARGETS: unknown upstream '{other}'"); + exit(2); + } + }; + let public = parse_addr("FGEX_PROXY_TARGETS", address.trim()); + // Said out loud, because everything else about an address is visible in the + // rules and this is not: it lives in this process and nowhere else, so an + // operator wondering why a port behaves the way it does has nothing to read. + eprintln!( + "[info] [main] {public} carries {} to {}", + match edge { + fgex_proxy::proxy::Edge::Tls => "TLS only", + _ => "whatever arrives", + }, + match (target, upstream) { + (Some(target), fgex_proxy::proxy::Onward::Plain) => { + format!("{target} in the clear") + } + (Some(target), fgex_proxy::proxy::Onward::Tls) => format!("{target} on TLS"), + (Some(target), _) => format!("{target} as it arrived"), + (None, fgex_proxy::proxy::Onward::Plain) => "the service in the clear".into(), + (None, fgex_proxy::proxy::Onward::Tls) => "the service on TLS".into(), + (None, _) => "the service as it arrived".into(), + } + ); + targets.publish(public, fgex_proxy::proxy::Published { target, edge, upstream }); + } + } + let cfg = ProxyConfig { listen, upstream, spoof_source, connect_timeout: Duration::from_millis(connect_timeout), tls, - // Opened once for the process and shared: the interface exists only while some - // TLS service is running, and `None` here simply means nothing is watching. - capture: fgex_proxy::capture::Capture::open(), + // `None` simply means nothing is watching, which is the ordinary case. + capture: capture.clone(), // Zero means no limit, which is what it was before there was one. The backend // always sends a value; the default here is for anyone running the engine by // hand, where a surprise limit would be worse than none. @@ -224,6 +363,7 @@ async fn run() { over_limit_forwards, first_byte_timeout, self_mark: cfg_self_mark, + targets: std::sync::Arc::clone(&targets), }; let chain = ChainHandle::new(FilterChain::new(filters, Duration::from_millis(deadline))); let chain_handle = chain.clone(); @@ -251,16 +391,62 @@ async fn run() { (spoof_source={spoof_source})" ); - // UDP manager: binds and tracks UDP relays, shared between startup configuration - // and dynamic commands arriving on stdin. - let udp_manager = fgex_proxy::udp::UdpManager::new( - chain_handle.clone(), - cfg_self_mark, - spoof_source, - max_connections, - over_limit_forwards, - Arc::clone(&counters), - ); + // The relays this service's addresses get, bound at startup and added to on demand. + // Which kind is not a per-address choice: a service speaks QUIC or it speaks + // datagrams, and the one it speaks is what every one of its addresses gets. + let relays = if quic { + let (cert, key) = match &material { + Some(pair) => pair, + None => { + eprintln!( + "[fatal] [main] a QUIC service needs a certificate: QUIC carries TLS 1.3 \ + inside it, and terminating it is the only way a filter sees anything" + ); + exit(2); + } + }; + let alpn = quic_alpn(); + let setup = match QuicSetup::build(cert, key, alpn.clone()) { + Ok(setup) => Arc::new(setup), + Err(e) => { + eprintln!("[fatal] [main] {e}"); + exit(2); + } + }; + eprintln!( + "[info] [main] QUIC terminated here; offering the service {}", + alpn.iter() + .map(|p| String::from_utf8_lossy(p).into_owned()) + .collect::>() + .join(", ") + ); + Relays::Quic(QuicManager::new( + chain_handle.clone(), + QuicConfig { + setup, + capture: capture.clone(), + self_mark: cfg_self_mark, + spoof_source, + max_connections, + over_limit_forwards, + first_byte_timeout, + connect_timeout: Duration::from_millis(connect_timeout), + // The default for a relay that was added without saying; the backend + // says, per address, on `FGEX_PROXY_UDP` and on `ADD_UDP`. + upstream: fgex_proxy::proxy::Onward::Same, + }, + Arc::clone(&counters), + )) + } else { + Relays::Datagram(fgex_proxy::udp::UdpManager::new( + chain_handle.clone(), + cfg_self_mark, + spoof_source, + max_connections, + over_limit_forwards, + Arc::clone(&counters), + )) + }; // UDP, when the backend asks for it: a comma-separated list of protected // addresses, one relay each. One socket per address rather than one for all of @@ -269,8 +455,21 @@ async fn run() { // there is nothing to recover. if let Ok(spec) = std::env::var("FGEX_PROXY_UDP") { for target in spec.split(',').filter(|s| !s.trim().is_empty()) { + // `
` or `
|`: what the service behind this one + // relay speaks. Per relay because a relay is one protected address, and + // two ports of one service can be reached differently. + let (target, onward) = match target.trim().split_once('|') { + Some((target, "plain")) => (target, fgex_proxy::proxy::Onward::Plain), + Some((target, "tls")) => (target, fgex_proxy::proxy::Onward::Tls), + Some((target, "same")) => (target, fgex_proxy::proxy::Onward::Same), + Some((_, other)) => { + eprintln!("[fatal] [main] FGEX_PROXY_UDP: unknown upstream '{other}'"); + exit(2); + } + None => (target.trim(), fgex_proxy::proxy::Onward::Same), + }; let upstream = parse_addr("FGEX_PROXY_UDP", target.trim()); - match udp_manager.add_relay(upstream).await { + match relays.add_relay(upstream, onward).await { Ok(port) => { println!("UDP {upstream} {port}"); use std::io::Write; @@ -289,8 +488,8 @@ async fn run() { // before, during and after a rule change or relay addition. let deadline_dur = Duration::from_millis(deadline); let control_chain = chain_handle.clone(); - let control_udp = udp_manager.clone(); - tokio::spawn(async move { serve_stdin(control_chain, deadline_dur, control_udp).await }); + let control_relays = relays.clone(); + tokio::spawn(async move { serve_stdin(control_chain, deadline_dur, control_relays, targets).await }); // Both counters live on the proxy, not on the chain: a chain is replaced wholesale // every time a ruleset is pushed, and counters that reset on a rule edit would make diff --git a/backend/proxysrc/src/proxy.rs b/backend/proxysrc/src/proxy.rs index 99b1786..4c264ad 100644 --- a/backend/proxysrc/src/proxy.rs +++ b/backend/proxysrc/src/proxy.rs @@ -11,7 +11,7 @@ use tokio::net::{TcpListener, TcpStream}; use crate::filter::{ next_connection_id, ChainHandle, ChainSessions, ConnectionId, ConnectionMeta, Direction, - FilterChain, Verdict, + FilterChain, Verdict, L4, }; use crate::capture::{Capture, Tap}; use crate::tls; @@ -36,6 +36,22 @@ pub struct TlsSetup { pub server: Option>, /// Re-encrypt towards the service, the way nginx's `proxy_ssl on` does. pub upstream: Option>, + /// Terminate it **only if the client actually starts a handshake**. + /// + /// What an `http` service is: one chain in front of a daemon that answers in the + /// clear on one port and under TLS on another, with one certificate and one set of + /// filters. The alternative was one service per edge with the chain copied between + /// them by hand, which is the failure several addresses under one service already + /// exist to prevent. + /// + /// It is a per-connection question because that is the only place it has an answer, + /// and answering it from what the client sent is the same rule the ALPN mirroring + /// follows: firegex carries what the two ends are doing rather than deciding it for + /// them. The upstream leg mirrors it too — a connection that arrived in the clear is + /// forwarded in the clear, one that arrived under TLS is re-encrypted — because the + /// service is being dialled on the port the client chose, and that port's edge is the + /// one the client just demonstrated. + pub optional: bool, } impl TlsSetup { @@ -44,6 +60,21 @@ impl TlsSetup { } } +/// What a connection turned out to be, from the bytes the client opened it with. +/// +/// **Peeked, never read.** Everything stays in the socket, so a connection this tells +/// apart is handed on exactly as it arrived — there is nothing buffered to replay, and no +/// way for the question to lose a byte. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Opening { + /// A TLS record: `0x16` handshake, major version 3. Nothing else begins that way. + Tls, + /// The HTTP/2 connection preface, which is what "prior knowledge" looks like. + Http2, + /// Anything else, including a service that spoke before its client did. + Other, +} + const RELAY_BUF: usize = 64 * 1024; /// Backstop for a direction that will not wind down — one blocked writing to a peer @@ -51,6 +82,102 @@ const RELAY_BUF: usize = 64 * 1024; /// prompt. const REJECT_GRACE: Duration = Duration::from_secs(2); +/// What is spoken at one published address, and where it goes. +/// +/// Firegex is transparent by default and was transparent only, for as long as there was +/// one answer to "where is the service": the address the client dialled. The engine +/// recovers that from conntrack and dials it as the client, so the service goes on +/// seeing connections to the port it listens on from the addresses they came from. An +/// address with no entry here behaves exactly that way, which is why the empty map is +/// the default and why every field below is optional. +/// +/// An entry is the operator saying something about one address in particular: +/// +/// * **where the service is**, when it is not on this port — one daemon answering on +/// `:80` reached over TLS on `:443`, without being moved or reconfigured; +/// * **what is spoken here**, when the service is reached in more than one way at once. +/// Only `http` services have that, and only there is the question asked. +/// +/// Only the TCP path needs any of it. A UDP or QUIC address already has a relay of its +/// own, bound to the service it fronts and terminating what it was built to terminate. +#[derive(Clone, Copy, Default, PartialEq, Eq)] +pub enum Edge { + /// Whatever the client turns out to be speaking. The default and the ordinary case: + /// one listener fronts a whole service, a cleartext port and an encrypted one are + /// the same listener, and which a connection is cannot be known before it starts. + /// + /// There is deliberately no "cleartext only" beside this. Refusing a client that + /// brings TLS to a port nobody promised would be a rule with nothing behind it, and + /// the permissive answer is what an `http` service has always given. + #[default] + Whatever, + /// This address is the encrypted one, and was said to be. A client that opens it in + /// the clear is refused rather than carried, because "this port is HTTPS" is a + /// promise about the port rather than a guess about the connection — and a service + /// expecting HTTPS being handed a cleartext request is the kind of surprise an + /// operator would rather have at the door. + Tls, +} + +#[derive(Clone, Copy, Default)] +pub struct Published { + pub target: Option, + pub edge: Edge, + /// What the service behind *this* address speaks. Per address rather than per + /// process because that is where it has an answer: one daemon reached over TLS on + /// one port and in the clear on another is re-encrypted for the first and handed + /// the plaintext for the second, and what firegex does on the way out is a property + /// of the way in. + pub upstream: Onward, +} + +/// What this engine speaks onward, to the service behind one address. +/// +/// Not to be confused with [`Upstream`] beside it, which is *where* to forward. This is +/// what is spoken when it gets there. +#[derive(Clone, Copy, Default, PartialEq, Eq)] +pub enum Onward { + /// Whatever arrived: a connection terminated here goes back out encrypted, one that + /// arrived in the clear is forwarded in the clear. The default, and what every + /// address did before there was a choice. + #[default] + Same, + /// The service speaks in the clear, whatever the client used to get here. + Plain, + /// The service speaks TLS, whatever the client used to get here. + Tls, +} + +#[derive(Default)] +pub struct Targets(std::sync::RwLock>); + +impl Targets { + /// Say something about one address. Replaces any previous answer for it. + pub fn publish(&self, public: SocketAddr, what: Published) { + if let Ok(mut map) = self.0.write() { + map.insert(public, what); + } + } + + /// Forget one published address. A stale entry could only ever be consulted by + /// traffic the rules no longer send here, but an engine whose map disagrees with the + /// rules is a thing somebody will one day read and believe. + pub fn withdraw(&self, public: &SocketAddr) { + if let Ok(mut map) = self.0.write() { + map.remove(public); + } + } + + /// What was said about the address this connection was dialled at, if anything. + pub fn resolve(&self, original: SocketAddr) -> Published { + self.0 + .read() + .ok() + .and_then(|map| map.get(&original).copied()) + .unwrap_or_default() + } +} + /// Where a connection should be forwarded. #[derive(Clone, Copy, Debug)] pub enum Upstream { @@ -93,6 +220,9 @@ pub struct ProxyConfig { /// the operator's to choose; refusing is the default because a filter that quietly /// stops filtering is the worse surprise. pub over_limit_forwards: bool, + /// Where published addresses send their traffic, when it is not where they were + /// dialled. Empty is the transparent case and the default. + pub targets: Arc, /// How long a connection may carry no bytes at all before it is closed. `None` is /// off, which is what it was before. /// @@ -124,6 +254,7 @@ impl ProxyConfig { over_limit_forwards: false, first_byte_timeout: None, self_mark: None, + targets: Arc::new(Targets::default()), } } } @@ -158,7 +289,7 @@ pub struct ProxyStats { /// a dozen ways out — a refused chain, a dial that fails, a handshake that times out, a /// panic — and a counter that leaks on any one of them is a limit that tightens until it /// refuses everything. -struct Slot(Arc); +pub(crate) struct Slot(pub(crate) Arc); impl Drop for Slot { fn drop(&mut self) { @@ -338,6 +469,10 @@ async fn handle_connection( } }, }; + // What the operator said about the address this was dialled at: nothing, for every + // address that is simply the service, which is the transparent case and the default. + let published = cfg.targets.resolve(upstream); + let upstream = published.target.unwrap_or(upstream); // A rule that steers our own outbound traffic back at us would spin forever. if upstream == cfg.listen { @@ -358,6 +493,345 @@ async fn handle_connection( let _ = client.set_nodelay(true); let _ = server.set_nodelay(true); + // Nothing is sniffed for a chain that has nothing to say: a bypassed service is a + // byte pump whatever its clients speak, so the question has no consequence and the + // peek would be work paid for no answer. An `http` service in that state carries the + // client's TLS through untouched, which is exactly what "not filtering" means. + let opening = if chain.current().is_bypassed() || !(cfg.tls.is_off() || cfg.tls.optional) { + Opening::Other + } else { + sniff(&client, &server).await + }; + + // An address declared to be the encrypted one keeps that promise: a client opening + // it in the clear is refused rather than carried to a service expecting HTTPS. Every + // other address is `Whatever`, where the sniff is the only answer there can be — + // one listener fronts both a cleartext port and an encrypted one, and which a + // connection is cannot be known before it starts. + if published.edge == Edge::Tls && opening != Opening::Tls && !cfg.tls.is_off() { + stats.closed_by_filter.fetch_add(1, Ordering::Relaxed); + // Said out loud, and at `warn`, because the operator on the other end of this has + // no way to guess it. What a cleartext client gets back is either nothing or a + // TLS alert — seven bytes a browser offers to save as a file — and neither says + // which of the two ends is wrong. It is nearly always the address: a port carrying + // the cleartext site declared as the encrypted one. Naming the address and what + // was expected turns an afternoon into a sentence. + eprintln!( + "[warn] [proxy] {peer} spoke something that is not TLS to {upstream}, which this service declares as its encrypted address. The connection is refused rather than carried to a service expecting HTTPS. If that address is meant to carry the site in the clear, it is the one to change." + ); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{upstream} is published as TLS and this connection did not start one"), + )); + } + if cfg.tls.is_off() || (cfg.tls.optional && opening != Opening::Tls) { + // HTTP/2 in the clear, which a client announces by opening with a fixed 24-byte + // preface. It is worth catching for the same reason the TLS path catches `h2`: + // without it a gRPC service that speaks plaintext — which is most of them behind + // a load balancer — is carried as HPACK nobody can read. + if opening == Opening::Http2 { + return crate::h2::carry( + client, + server, + crate::h2::Carriage { + client: peer, + upstream, + chain: chain.clone(), + stats: Arc::clone(&stats), + // Unlike every other plaintext connection, this one *is* worth + // reconstructing: what is on the wire is a compression format, so the + // capture interface is the only place the exchange can be read as the + // filters saw it. + capture: cfg.capture.clone(), + first_byte_timeout: cfg.first_byte_timeout, + }, + ) + .await; + } + // Only where there is something to reconstruct: with no TLS and no HTTP/2, the + // bytes on the wire *are* the plaintext, and anybody wanting them can capture the + // interface they are already crossing rather than a copy this process invents. + let (client_rd, client_wr) = tokio::io::split(client); + let (server_rd, server_wr) = tokio::io::split(server); + return relay( + client_rd, client_wr, server_rd, server_wr, peer, upstream, &cfg, &chain, &stats, None, + ) + .await; + } + + // Handshakes before anything is relayed, and all of them on the deadline: a peer + // that opens a connection and then says nothing must not tie up a task. + // + // The order is the interesting part. The client's handshake is *started* and + // then held open at the ClientHello, because that message carries the protocols + // the client is willing to speak and this proxy has no business deciding them. + // The upstream handshake goes next, offering the service exactly that list; what + // the service picks is what the client is then told. Terminating in the middle + // of a connection means answering for a service, and answering something the + // service did not say is how a proxy breaks a protocol it was only supposed to + // carry — a client told `h2` while the service speaks HTTP/1.1 sends frames to + // something that cannot read them. + // + // Terminating HTTP/2 below changes nothing about that: the client is still told + // exactly what the service chose. What changes is what firegex then *does* with a + // connection that agreed on `h2` — it speaks it, on both sides, instead of + // forwarding frames no filter can read. + let (started, wanted) = match &cfg.tls.server { + Some(_) => { + let start = tokio::time::timeout( + cfg.connect_timeout, + tokio_rustls::LazyConfigAcceptor::new(rustls::server::Acceptor::default(), client), + ) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "TLS handshake timed out"))??; + let wanted: Vec> = start + .client_hello() + .alpn() + .map(|it| it.map(|p| p.to_vec()).collect()) + .unwrap_or_default(); + (Some(start), wanted) + } + None => (None, Vec::new()), + }; + + // Per address: the configuration is built once, and whether this connection uses it + // is the address's answer. `Same` is the old rule — re-encrypt what was terminated — + // and is why a plain TCP service, which terminates nothing, is untouched by any of + // this and always was. + let upstream_tls = match published.upstream { + Onward::Plain => None, + // The two coincide here and that is not an oversight: this point is only + // reached for a connection whose TLS *was* terminated, so "what arrived" and + // "TLS" are the same answer. They differ on the QUIC edge, where there is no + // arriving TLS to mirror and the choice decides the whole upstream leg. + Onward::Tls | Onward::Same => cfg.tls.upstream.as_ref(), + }; + let (server, agreed): (Duplex, Option>) = match upstream_tls { + Some(config) => { + let name = tls::server_name(&upstream.ip().to_string())?; + let connected = tokio::time::timeout( + cfg.connect_timeout, + tls::connector(tls::with_alpn(config, &wanted)).connect(name, server), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "upstream TLS handshake timed out") + })??; + let agreed = connected.get_ref().1.alpn_protocol().map(|p| p.to_vec()); + (Box::new(connected), agreed) + } + // Nothing upstream to *ask*, which is two situations wearing one shape. Where + // the connection was never terminated there is nothing to decide and `wanted` is + // empty anyway. Where it was terminated and the address says the service speaks + // HTTP/1.1 whatever arrives (`Onward::Plain`), firegex is not carrying somebody + // else's protocol — it **is** the far end — so it answers for itself, and saying + // `http/1.1` is the whole of what it can honestly promise today. + // + // **`h2` is promised only while the chain will render it.** A bypassed chain is a + // byte pump, and a byte pump between an HTTP/2 client and an HTTP/1.1 service + // carries frames nothing at the far end can read — the exact breakage the + // mirroring rule exists to prevent, arriving from the other direction. The trade + // is the one the bypass already documents: a connection admitted while the chain + // was empty keeps the protocol it was admitted with. + // + // This is the other half of an asymmetry that stood for a while and had no reason + // behind it: the same cleartext HTTP/1.1 service was reachable over HTTP/3, + // because the QUIC edge picks from its own candidate list and speaks HTTP/1.1 + // onwards through `h1up`, and not reachable over HTTP/2 at all, because this edge + // had nobody to copy and therefore said nothing. One rendering underneath, + // opposite answers. + None => { + let picked = if matches!(published.upstream, Onward::Plain) { + let renders = !chain.current().is_bypassed(); + let offers = |name: &[u8]| wanted.iter().any(|p| p.as_slice() == name); + if renders && offers(b"h2") { + Some(b"h2".to_vec()) + } else if offers(b"http/1.1") { + Some(b"http/1.1".to_vec()) + } else { + None + } + } else { + None + }; + (Box::new(server), picked) + } + }; + + let client: Duplex = match (started, &cfg.tls.server) { + (Some(start), Some(config)) => { + let accepted = tokio::time::timeout( + cfg.connect_timeout, + start.into_stream(tls::answering_with(config, agreed.as_deref())), + ) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "TLS handshake timed out"))??; + Box::new(accepted) + } + _ => unreachable!("a started handshake means a server config"), + }; + + // Both ends agreed to speak HTTP/2, so the engine speaks it: terminated here, each + // exchange rendered to the chain as the HTTP/1.1 it would have been, and re-encoded + // on the way to the service. Without this the connection was forwarded as frames, + // and a pattern written against a request line matched nothing while an + // `HttpRequest` filter was never called — switched on, reporting nothing, running + // never. + // + // Not when the chain is bypassed. Rendering for a chain that has nothing to say is + // work paid for no answer, and this is the path a service with no filters and the + // `over_limit_forwards` fallback both take. The cost is that a connection carried + // while the chain was empty stays a byte pump for its whole life, even if a filter + // is pushed a moment later — the same trade the fallback itself makes, and the + // reason it is written down here. + if agreed.as_deref() == Some(b"h2") && !chain.current().is_bypassed() { + let carriage = crate::h2::Carriage { + client: peer, + upstream, + chain: chain.clone(), + stats: Arc::clone(&stats), + capture: cfg.capture.clone(), + first_byte_timeout: cfg.first_byte_timeout, + }; + // Which version leaves is the address's answer, not the client's. `Plain` is the + // operator saying the service speaks HTTP/1.1 whatever arrives, so what leaves is + // the HTTP/1.1 the chain was already shown — the same bargain the QUIC edge makes, + // and the reason `h2.rs` takes its upstream as an `Outbound` rather than a + // connection of its own kind. + return match published.upstream { + Onward::Plain => { + crate::h2::carry_to_h1( + client, + crate::h1up::H1Upstream { + upstream, + client: peer, + tls: None, + connect_timeout: cfg.connect_timeout, + self_mark: cfg.self_mark, + spoof: cfg.spoof_source, + }, + carriage, + ) + .await + } + _ => crate::h2::carry(client, server, carriage).await, + }; + } + + let (client_rd, client_wr) = tokio::io::split(client); + let (server_rd, server_wr) = tokio::io::split(server); + relay( + client_rd, + client_wr, + server_rd, + server_wr, + peer, + upstream, + &cfg, + &chain, + &stats, + Tap::open(cfg.capture.clone(), peer, upstream), + ) + .await +} + +/// What a client opening an HTTP/2 connection in the clear sends before anything else. +/// +/// RFC 9113's connection preface: a fixed 24 bytes, written as one piece, that no other +/// protocol begins with. It is what "prior knowledge" means — the client has been told the +/// service speaks HTTP/2 and does not negotiate. +const H2C_PREFACE: &[u8] = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; + +/// How long the opening is waited for when neither end has said anything yet. +/// +/// Short, and paid only by a connection where *nobody* has spoken — which is the shape the +/// first-byte deadline already exists to close, measured in seconds, so this adds nothing +/// anybody will notice. It is here so that two silent peers cannot hold this function, and +/// the two descriptors behind it, open forever. +const SNIFF_WAIT: Duration = Duration::from_millis(250); + +/// What this connection opens with, without taking anything out of the socket. +/// +/// It races the service, and that is the part that matters: a protocol where the *server* +/// speaks first — SMTP, SSH, most game protocols — would otherwise be held here waiting for +/// a client that is correctly waiting for a banner, and firegex would look like it was +/// breaking them at random. Neither TLS nor HTTP/2 has a client that waits to be spoken to, +/// so the service having spoken is proof this is neither. +async fn sniff(client: &TcpStream, server: &TcpStream) -> Opening { + async fn opening(client: &TcpStream) -> Opening { + let mut buf = [0u8; H2C_PREFACE.len()]; + loop { + let n = match client.peek(&mut buf).await { + Ok(0) | Err(_) => return Opening::Other, + Ok(n) => n, + }; + // Decided from three bytes, which arrive in the first segment of every TLS + // connection there has ever been: a handshake record, major version 3. Not + // from one — an opening byte on its own says nothing, and guessing from it + // would send a connection down the wrong edge for good. + if buf[0] == 0x16 { + if n < 3 { + tokio::time::sleep(Duration::from_millis(1)).await; + continue; + } + return if buf[1] == 0x03 { + Opening::Tls + } else { + Opening::Other + }; + } + if n >= H2C_PREFACE.len() { + return if buf == H2C_PREFACE { + Opening::Http2 + } else { + Opening::Other + }; + } + // Fewer bytes than the preface, but so far they *are* the preface: the + // segment boundary fell inside it. Peeking again costs nothing and is bounded + // by the caller — and treating a short read as "not HTTP/2" would be a + // connection carried unfiltered for a reason nobody could see. + if H2C_PREFACE.starts_with(&buf[..n]) { + tokio::time::sleep(Duration::from_millis(1)).await; + } else { + return Opening::Other; + } + } + } + + let mut first = [0u8; 1]; + tokio::select! { + biased; + answer = opening(client) => answer, + _ = server.peek(&mut first) => Opening::Other, + _ = tokio::time::sleep(SNIFF_WAIT) => Opening::Other, + } +} + +/// Carry one connection as two streams of bytes, which is every case HTTP/2 is not. +/// +/// Its own function because the plaintext path and the TLS one reach it from different +/// places now: the h2 branch returns before this, and what is left has to open the +/// chain's connection, pump, and close it in one piece rather than in two copies. +#[allow(clippy::too_many_arguments)] +async fn relay( + client_rd: CR, + client_wr: CW, + server_rd: SR, + server_wr: SW, + peer: SocketAddr, + upstream: SocketAddr, + cfg: &Arc, + chain: &ChainHandle, + stats: &Arc, + tap: Option>, +) -> io::Result<()> +where + CR: AsyncReadExt + Unpin + Send + 'static, + CW: AsyncWriteExt + Unpin + Send + 'static, + SR: AsyncReadExt + Unpin + Send + 'static, + SW: AsyncWriteExt + Unpin + Send + 'static, +{ // Both directions share it: to a filter this is one stream, and a filter keeping // per-stream state must not see the two halves as unrelated clients. let connection = next_connection_id(); @@ -369,127 +843,24 @@ async fn handle_connection( &ConnectionMeta { client: peer, server: upstream, - tcp: true, + l4: L4::Tcp, }, ); - // Only where there is something to reconstruct: with TLS off, the bytes on the wire - // *are* the plaintext, and anybody wanting them can capture the interface they are - // already crossing rather than a copy this process invents. - let tap = if cfg.tls.is_off() { - None - } else { - Tap::open(cfg.capture.clone(), peer, upstream) - }; + let (up, down) = spawn_pumps( + client_rd, + client_wr, + server_rd, + server_wr, + chain, + stats, + connection, + tap.clone(), + Arc::new(AtomicBool::new(false)), + cfg.first_byte_timeout, + ); - let (mut up, mut down) = if cfg.tls.is_off() { - let (client_rd, client_wr) = tokio::io::split(client); - let (server_rd, server_wr) = tokio::io::split(server); - spawn_pumps( - client_rd, client_wr, server_rd, server_wr, &chain, &stats, connection, None, - cfg.first_byte_timeout, - ) - } else { - // Handshakes before anything is relayed, and all of them on the deadline: a peer - // that opens a connection and then says nothing must not tie up a task. - // - // The order is the interesting part. The client's handshake is *started* and - // then held open at the ClientHello, because that message carries the protocols - // the client is willing to speak and this proxy has no business deciding them. - // The upstream handshake goes next, offering the service exactly that list; what - // the service picks is what the client is then told. Terminating in the middle - // of a connection means answering for a service, and answering something the - // service did not say is how a proxy breaks a protocol it was only supposed to - // carry — a client told `h2` while the service speaks HTTP/1.1 sends frames to - // something that cannot read them. - let (started, wanted) = match &cfg.tls.server { - Some(_) => { - let start = tokio::time::timeout( - cfg.connect_timeout, - tokio_rustls::LazyConfigAcceptor::new( - rustls::server::Acceptor::default(), - client, - ), - ) - .await - .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "TLS handshake timed out"))??; - let wanted: Vec> = start - .client_hello() - .alpn() - .map(|it| it.map(|p| p.to_vec()).collect()) - .unwrap_or_default(); - (Some(start), wanted) - } - None => (None, Vec::new()), - }; - - let (server, agreed): (Duplex, Option>) = match &cfg.tls.upstream { - Some(config) => { - let name = tls::server_name(&upstream.ip().to_string())?; - let connected = tokio::time::timeout( - cfg.connect_timeout, - tls::connector(tls::with_alpn(config, &wanted)).connect(name, server), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "upstream TLS handshake timed out") - })??; - let agreed = connected.get_ref().1.alpn_protocol().map(|p| p.to_vec()); - (Box::new(connected), agreed) - } - // Nothing upstream to ask, so nothing to relay back: the client is told no - // protocol rather than one this end invented. - None => (Box::new(server), None), - }; - - let client: Duplex = match (started, &cfg.tls.server) { - (Some(start), Some(config)) => { - let accepted = tokio::time::timeout( - cfg.connect_timeout, - start.into_stream(tls::answering_with(config, agreed.as_deref())), - ) - .await - .map_err(|_| { - io::Error::new(io::ErrorKind::TimedOut, "TLS handshake timed out") - })??; - Box::new(accepted) - } - _ => unreachable!("a started handshake means a server config"), - }; - let (client_rd, client_wr) = tokio::io::split(client); - let (server_rd, server_wr) = tokio::io::split(server); - spawn_pumps( - client_rd, client_wr, server_rd, server_wr, &chain, &stats, connection, - tap.clone(), cfg.first_byte_timeout, - ) - }; - - // A rule blocks a connection, not a direction. Half-closing would leave the - // other side free to answer anyway — a service that replies without waiting for - // the request would then still reach a client whose request was refused. - fn rejected(r: &Result, tokio::task::JoinError>) -> bool { - matches!(r, Ok(Ok(PumpOutcome::Rejected))) - } - async fn wind_down( - other: &mut tokio::task::JoinHandle>, - rejected: bool, - ) { - if !rejected { - let _ = other.await; - return; - } - if tokio::time::timeout(REJECT_GRACE, &mut *other) - .await - .is_err() - { - other.abort(); - } - } - - tokio::select! { - r = &mut up => wind_down(&mut down, rejected(&r)).await, - r = &mut down => wind_down(&mut up, rejected(&r)).await, - } + join_pumps(up, down).await; // Both directions are finished, so anything a filter was keeping for this stream // can go. Filters whose state lives in a session were already freed by dropping // it; this is for the ones whose state lives somewhere the session cannot reach. @@ -511,10 +882,47 @@ pub enum PumpOutcome { Rejected, } +/// Wait for both directions to finish, and report whether a rule ended them. +/// +/// A rule blocks a connection, not a direction. Half-closing would leave the other side +/// free to answer anyway — a service that replies without waiting for the request would +/// then still reach a client whose request was refused. The refused direction is already +/// winding the other one down through the stop channel; the grace period is the backstop +/// for one blocked writing to a peer that has stopped reading. +/// +/// The answer comes back rather than being acted on here, because what a refusal ends +/// differs by layer: on TCP the connection *is* the stream, while a QUIC stream is one of +/// many and the connection carrying it has to be closed on purpose. +pub(crate) async fn join_pumps( + mut up: tokio::task::JoinHandle>, + mut down: tokio::task::JoinHandle>, +) -> bool { + fn rejected(r: &Result, tokio::task::JoinError>) -> bool { + matches!(r, Ok(Ok(PumpOutcome::Rejected))) + } + async fn wind_down( + other: &mut tokio::task::JoinHandle>, + rejected: bool, + ) -> bool { + if !rejected { + return matches!(other.await, Ok(Ok(PumpOutcome::Rejected))); + } + if tokio::time::timeout(REJECT_GRACE, &mut *other).await.is_err() { + other.abort(); + } + true + } + + tokio::select! { + r = &mut up => wind_down(&mut down, rejected(&r)).await, + r = &mut down => wind_down(&mut up, rejected(&r)).await, + } +} + /// Start both directions. Split out so the plain and the TLS paths share it. #[allow(clippy::type_complexity)] #[allow(clippy::too_many_arguments)] -fn spawn_pumps( +pub(crate) fn spawn_pumps( client_rd: CR, client_wr: CW, server_rd: SR, @@ -523,6 +931,7 @@ fn spawn_pumps( stats: &Arc, connection: ConnectionId, tap: Option>, + spoken: Arc, cfg_first_byte: Option, ) -> ( tokio::task::JoinHandle>, @@ -542,7 +951,11 @@ where // Set by whichever direction moves a byte first. Both pumps hold it, so a server // that speaks before its client satisfies the deadline just as a request would. - let spoken = Arc::new(AtomicBool::new(false)); + // + // Passed in rather than made here: on QUIC one connection carries many streams, and + // the question the deadline asks — has this peer said anything at all — is about the + // connection. A flag per stream would answer it once per stream and never for the + // connection that opened a hundred of them and spoke on none. if let Some(deadline) = cfg_first_byte { let spoken = Arc::clone(&spoken); let stop = stop_tx.clone(); @@ -602,7 +1015,7 @@ where // used exactly once, so a struct would be a wrapper around the argument list. #[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)] -async fn pump( +pub(crate) async fn pump( mut rd: R, mut wr: W, dir: Direction, diff --git a/backend/proxysrc/src/pyworker.rs b/backend/proxysrc/src/pyworker.rs index e93b72c..2b1c88d 100644 --- a/backend/proxysrc/src/pyworker.rs +++ b/backend/proxysrc/src/pyworker.rs @@ -39,7 +39,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; use std::time::{Duration, Instant}; -use crate::filter::{ConnectionId, ConnectionMeta, Direction, Filter, FilterCtx, Verdict}; +use crate::filter::{ConnectionId, ConnectionMeta, Direction, Filter, FilterCtx, Verdict, L4}; const VERDICT_ACCEPT: u8 = 0; const VERDICT_REJECT: u8 = 1; @@ -292,7 +292,13 @@ impl Filter for PyWorkerRule { "server_ip": meta.server.ip().to_string(), "server_port": meta.server.port(), "is_ipv6": meta.server.is_ipv6(), - "is_tcp": meta.tcp, + // Both, and they are not the same question. `is_tcp` is what the NFQUEUE + // side has always sent and what a filter reads to know the wire; `l4` is what + // decides whether a stream can be built, which is true of QUIC and not of a + // datagram. A library that only received the first would have to guess one + // of them. + "is_tcp": meta.l4 == L4::Tcp, + "l4": meta.l4.name(), }) .to_string(); let mut slot = match self.child.lock() { diff --git a/backend/proxysrc/src/quic.rs b/backend/proxysrc/src/quic.rs new file mode 100644 index 0000000..5c38b8c --- /dev/null +++ b/backend/proxysrc/src/quic.rs @@ -0,0 +1,994 @@ +//! QUIC, terminated by the engine. +//! +//! TLS over TCP could in principle be left alone: the bytes on the wire are framed by a +//! transport the kernel understands, and a layer that only forwards still has packets to +//! count. QUIC does not leave even that. After the Initial packet every frame, every +//! stream boundary and the packet number itself are encrypted, so a layer that forwards +//! has nothing a filter could be shown — the choice is to terminate it or to admit that +//! nothing is being inspected. That is why QUIC lives here and on no other layer. +//! +//! On the wire it is UDP, which is what the rules match and what the relay map is made +//! of: one endpoint per protected address, exactly as [`crate::udp`] binds one socket +//! per address and for the same reason — `SO_ORIGINAL_DST` is TCP and SCTP only, so a +//! single listener could not recover where a datagram was headed. +//! +//! **The ALPN is asked of the service, not of the client**, which is the one place this +//! differs from the TLS path beside it. There the ClientHello is held open +//! (`LazyConfigAcceptor`), the service is asked, and the client is told what the service +//! picked. In QUIC the ClientHello arrives inside an encrypted Initial whose processing +//! *is* the handshake, and there is nothing to hold it at. So the order is reversed: the +//! service is offered the candidates, and the client is told the one thing the service +//! agreed to. The invariant that matters is unchanged — the client is never told a +//! protocol the service did not choose — and what is lost is knowing in advance whether +//! the client would have accepted it. When it does not, the handshake fails and says so, +//! rather than a protocol being quietly broken. + +use std::collections::HashMap; +use std::io; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig}; +use quinn::{Connection, Endpoint, Incoming, VarInt}; + +use crate::capture::{Capture, Tap}; +use crate::filter::{ + next_connection_id, ChainHandle, ChainSessions, ConnectionId, ConnectionMeta, Direction, + FilterChain, Verdict, L4, +}; +use crate::proxy::{join_pumps, pump, spawn_pumps, Onward, ProxyStats, PumpOutcome, Slot}; +use crate::transparent::unmap; + +/// What the client is told when a rule refuses one of its streams. +/// +/// An application error code, because by then the connection is established and QUIC has +/// no other way to say anything: `0x1f3` is in the application range and firegex is the +/// application here. The reason travels with it, so a developer who blocks themselves out +/// of their own service reads why in their client rather than guessing from a reset. +const REFUSED_CODE: u32 = 0x1f3; + +/// How long an idle QUIC connection is kept. +/// +/// Longer than the datagram relay's flow timeout, and for the opposite reason: a QUIC +/// connection *does* have a close to observe, so this is a backstop for a peer that +/// vanished rather than the only thing that ends it. +const IDLE: Duration = Duration::from_secs(60); + +/// How much unread datagram traffic one connection may have waiting. +/// +/// Bounded on purpose, and small: datagrams are dropped when it is full, which is what +/// unreliable means and exactly what the sender already has to handle. Unbounded, a peer +/// that sends faster than the chain judges would be buying memory in this process. +const DATAGRAM_BUFFER: usize = 256 * 1024; + +/// Everything a QUIC service needs decided once, before any client arrives. +pub struct QuicSetup { + /// One accept-side configuration per protocol the service might pick, built up front + /// rather than per connection: the candidate list is short and known, and a rustls + /// config clone plus a cipher-suite lookup on the accept path is work done while a + /// client is waiting. + answering: HashMap, Arc>, + /// For the case the service agreed to nothing. QUIC requires ALPN, so this is a + /// configuration that will refuse — kept so the refusal is the handshake's, with a + /// reason, rather than a panic here. + silent: Arc, + /// Offered to the service, in the operator's order. + client: quinn::ClientConfig, + candidates: Vec>, +} + +impl QuicSetup { + /// Build both edges from the service's certificate and the protocols it may speak. + pub fn build(cert_pem: &str, key_pem: &str, candidates: Vec>) -> Result { + if candidates.is_empty() { + return Err("a QUIC service needs at least one ALPN protocol".to_string()); + } + // Two of them, differing in one advertisement. **HTTP/3 connections do not carry + // datagrams here**, and the honest way to say that is not to advertise the + // extension on them: a peer then knows from the handshake and falls back, instead + // of sending datagrams that vanish. The reason is the one thing a datagram would + // need that this engine cannot give it — an HTTP/3 datagram names the stream it + // belongs to, and the request streams this proxy opens towards the service are + // not the ones the client opened, so the name would point somewhere else on the + // far side. Everything that is not h3 has no such coupling and carries them. + let streams_only = transport_config(false); + let with_datagrams = transport_config(true); + let base = crate::tls::server_config(cert_pem, key_pem)?; + + let mut answering = HashMap::new(); + for protocol in &candidates { + let transport = if protocol == b"h3" { &streams_only } else { &with_datagrams }; + answering.insert( + protocol.clone(), + Arc::new(server_config(&base, &[protocol.clone()], transport)?), + ); + } + let silent = Arc::new(server_config(&base, &[], &streams_only)?); + + let mut rustls_client = crate::tls::quic_client_config()?; + rustls_client.alpn_protocols = candidates.clone(); + let crypto = QuicClientConfig::try_from(rustls_client) + .map_err(|e| format!("cannot configure QUIC towards the service: {e}"))?; + let mut client = quinn::ClientConfig::new(Arc::new(crypto)); + // The service is dialled before it has said which protocol it speaks, so this + // edge is decided by the candidate list rather than by the answer: with h3 the + // only candidate there is nothing a datagram could be for, and advertising it + // upstream would invite what this end would then have to drop. + client.transport_config(if candidates.iter().all(|p| p == b"h3") { + Arc::clone(&streams_only) + } else { + Arc::clone(&with_datagrams) + }); + + Ok(Self { + answering, + silent, + client, + candidates, + }) + } + + /// The configuration that advertises the one protocol the service agreed to. + fn answering_with(&self, agreed: Option<&[u8]>) -> Arc { + agreed + .and_then(|p| self.answering.get(p)) + .map(Arc::clone) + .unwrap_or_else(|| Arc::clone(&self.silent)) + } + + /// What the endpoint answers with before a service has been asked. Only ever used + /// for the connections this relay turns away, which need a configuration to be + /// turned away *by*. + fn default_server(&self) -> Arc { + self.candidates + .first() + .and_then(|p| self.answering.get(p)) + .map(Arc::clone) + .unwrap_or_else(|| Arc::clone(&self.silent)) + } +} + +fn server_config( + base: &Arc, + alpn: &[Vec], + transport: &Arc, +) -> Result { + let mut rustls_config = (**base).clone(); + rustls_config.alpn_protocols = alpn.to_vec(); + // Left at zero deliberately. 0-RTT data is replayable by anyone who watched it go + // past, and a filter that refused a request has no way to un-deliver the copy of it + // the service already acted on. A round trip is the price of that not being true. + rustls_config.max_early_data_size = 0; + let crypto = QuicServerConfig::try_from(rustls_config) + .map_err(|e| format!("the certificate cannot carry QUIC: {e}"))?; + let mut config = quinn::ServerConfig::with_crypto(Arc::new(crypto)); + config.transport_config(Arc::clone(transport)); + Ok(config) +} + +/// The transport parameters an edge is built with. +fn transport_config(datagrams: bool) -> Arc { + let mut transport = quinn::TransportConfig::default(); + transport.max_idle_timeout(Some( + IDLE.try_into().expect("the idle timeout fits in a VarInt"), + )); + // Advertised only where they are carried, because a peer that finds the extension + // announced and then watches its datagrams disappear is worse off than one that knows + // from the handshake to do without. What a filter is shown for one is a `RawPacket` + // and nothing more, which is the same answer the datagram relay gives on plain UDP — + // there is no stream, so there is nothing for the stream models to be built on. + transport.datagram_receive_buffer_size(datagrams.then_some(DATAGRAM_BUFFER)); + Arc::new(transport) +} + +/// The settings a relay is built with, shared by every address of one service. +#[derive(Clone)] +pub struct QuicConfig { + pub setup: Arc, + /// Where a reconstruction of each stream is written, when anything is listening. + /// `None` is the ordinary case — no capture interface — and never a failure. + pub capture: Option>, + pub self_mark: Option, + pub spoof_source: bool, + pub max_connections: usize, + pub over_limit_forwards: bool, + pub first_byte_timeout: Option, + pub connect_timeout: Duration, + /// What the service behind this *relay* speaks — QUIC unless it says otherwise. + /// + /// One per relay rather than one per process, because a relay is one protected + /// address and the question belongs to the address: two ports of one service can + /// be reached differently. The other two values put an ordinary HTTP/1.1 service + /// behind an HTTP/3 edge, with firegex terminating QUIC on its behalf. + pub upstream: Onward, +} + +/// One protected address, terminated. +pub struct QuicRelay { + endpoint: Endpoint, + upstream: SocketAddr, + chain: ChainHandle, + cfg: QuicConfig, + stats: Arc, +} + +impl QuicRelay { + pub fn bind( + listen: SocketAddr, + upstream: SocketAddr, + chain: ChainHandle, + cfg: QuicConfig, + stats: Arc, + ) -> io::Result { + // Unmarked, exactly as the datagram relay's listener is. The mark is what tells + // the intercept rules to leave *our own dial* alone; this socket is the one the + // rules are pointing at, and its answers go back the way conntrack sends them. + let socket = std::net::UdpSocket::bind(listen)?; + let runtime = quinn::default_runtime() + .ok_or_else(|| io::Error::other("no async runtime for the QUIC endpoint"))?; + let endpoint = Endpoint::new( + quinn::EndpointConfig::default(), + Some((*cfg.setup.default_server()).clone()), + socket, + runtime, + )?; + Ok(Self { + endpoint, + upstream, + chain, + cfg, + stats, + }) + } + + pub fn local_addr(&self) -> io::Result { + self.endpoint.local_addr() + } + + /// Accept forever. Nothing one connection does may end this loop. + pub async fn serve(self: Arc) { + while let Some(incoming) = self.endpoint.accept().await { + let relay = Arc::clone(&self); + tokio::spawn(async move { relay.admit(incoming).await }); + } + eprintln!("[warn] [quic] the endpoint for {} closed", self.upstream); + } + + /// Decide whether this connection is carried at all, and on what terms. + async fn admit(self: Arc, incoming: Incoming) { + let client = unmap(incoming.remote_address()); + + // Before anything at all is opened towards the service. An Initial packet is one + // datagram carrying no proof that its source address is real, and answering it by + // opening a connection to the protected service is how a spoofed source turns + // this relay into the amplifier — the same shape as the datagram flood the flow + // cap exists for, but with a handshake in front of it. A Retry costs the honest + // client one round trip and costs the forged one everything, because the token + // has to come back from an address that can receive it. + if !incoming.remote_address_validated() { + if let Err(e) = incoming.retry() { + // Only possible if it was validated after all, which this branch has + // just established it was not. Refuse rather than assume. + e.into_incoming().refuse(); + } + return; + } + + self.stats.accepted.fetch_add(1, Ordering::Relaxed); + let live = self.stats.live.fetch_add(1, Ordering::Relaxed) + 1; + let slot = Slot(Arc::clone(&self.stats)); + let limit = self.cfg.max_connections; + let over = limit > 0 && live > limit as u64; + if over { + self.stats.over_limit.fetch_add(1, Ordering::Relaxed); + if !self.stats.warned_limit.swap(true, Ordering::Relaxed) { + eprintln!( + "[warn] [quic] {limit} concurrent connections reached; further \ + connections are being {} until it clears", + if self.cfg.over_limit_forwards { + "carried unfiltered" + } else { + "refused" + }, + ); + } + if !self.cfg.over_limit_forwards { + // A refusal the client can read, rather than a silence it has to time + // out on: this connection reached a working service that chose not to + // carry it. + incoming.refuse(); + drop(slot); + return; + } + } else if live * 2 <= limit as u64 { + self.stats.warned_limit.store(false, Ordering::Relaxed); + } + + // Past the limit and told to carry it anyway: the streams are relayed against an + // empty chain. The traffic reaches the service and nothing claims to have looked + // at it, which is the operator's choice made literal. It is not free here the way + // it is on TCP — the connection is still terminated, decrypted and re-encrypted, + // because there is no such thing as forwarding a QUIC connection unopened. + let chain = if over { + ChainHandle::new(FilterChain::empty()) + } else { + self.chain.clone() + }; + + let _slot = slot; + if let Err(e) = self.serve_connection(incoming, client, chain).await { + eprintln!("[info] [quic] connection from {client} ended: {e}"); + } + } + + /// Both handshakes, in the order that lets the service decide the protocol. + async fn serve_connection( + &self, + incoming: Incoming, + client: SocketAddr, + chain: ChainHandle, + ) -> io::Result<()> { + // Two shapes, decided by what the service speaks. With QUIC behind, the service + // is dialled *first* and asked which protocol it wants — that ordering is the + // whole reason the client can be told something the service actually chose. + // With HTTP/1.1 behind there is nobody to ask: firegex is the thing speaking + // HTTP/3, so it answers `h3` for itself and nothing is dialled until there is a + // request to send. + let (endpoint, behind, agreed) = match self.cfg.upstream { + Onward::Same => { + // The endpoint is held for as long as the connection is: it owns the + // socket the connection speaks through, and dropping it would take the + // connection with it. + let (endpoint, service) = self.dial_upstream(client).await?; + let agreed = negotiated_protocol(&service); + (Some(endpoint), Behind::Quic(service), agreed) + } + protocol => ( + None, + Behind::Http1(crate::h1up::H1Upstream { + upstream: self.upstream, + client, + tls: match protocol { + Onward::Tls => Some( + crate::tls::client_config().map_err(io::Error::other)?, + ), + _ => None, + }, + connect_timeout: self.cfg.connect_timeout, + self_mark: self.cfg.self_mark, + spoof: self.cfg.spoof_source, + }), + Some(b"h3".to_vec()), + ), + }; + + let accepting = incoming + .accept_with(self.cfg.setup.answering_with(agreed.as_deref())) + .map_err(io::Error::other)?; + let peer = match tokio::time::timeout(self.cfg.connect_timeout, accepting).await { + Ok(Ok(connection)) => connection, + Ok(Err(e)) => { + // Nearly always the one mismatch this ordering can produce, so it is + // named rather than left as a handshake error the operator has to decode. + // Which end chose it differs: with QUIC behind it is the service's pick, + // and with HTTP/1.1 behind it is `h3`, which is what firegex itself is + // offering to speak on the service's behalf. + if let Some(protocol) = &agreed { + eprintln!( + "[warn] [quic] {client} would not take `{}`, which is what {} chose: {e}", + String::from_utf8_lossy(protocol), + match behind { + Behind::Quic(_) => self.upstream.to_string(), + Behind::Http1(_) => "firegex, for a service that speaks HTTP/1.1" + .to_string(), + }, + ); + } + if let Some(service) = behind.quic() { + service.close(VarInt::from_u32(0), b"client handshake failed"); + } + return Err(io::Error::other(e)); + } + Err(_) => { + if let Some(service) = behind.quic() { + service.close(VarInt::from_u32(0), b"client handshake timed out"); + } + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "the client's QUIC handshake timed out", + )); + } + }; + + self.relay(peer, behind, client, chain, agreed.as_deref()).await; + drop(endpoint); + Ok(()) + } + + /// Open the connection to the service, from the client's own address where we can. + async fn dial_upstream(&self, client: SocketAddr) -> io::Result<(Endpoint, Connection)> { + let endpoint = match self.endpoint_as(client) { + Ok(endpoint) => endpoint, + Err(e) => { + // Losing the client's address is bad; losing the connection is worse — + // the same trade the TCP dial makes, and just as loud about it. + self.stats + .source_spoof_failures + .fetch_add(1, Ordering::Relaxed); + if !self.stats.warned_spoof.swap(true, Ordering::Relaxed) { + eprintln!( + "[warn] [quic] cannot reach {} as {}: {e}. Falling back to our own \ + address — the service will not see real client IPs.", + self.upstream, + client.ip() + ); + } + self.endpoint_plain()? + } + }; + + // An IP literal is what the TCP path presents too. Nothing verifies it — the + // service being protected is the thing we are defending, not a peer to be + // authenticated — but rustls needs a name to put in the handshake. + let name = self.upstream.ip().to_string(); + let connecting = endpoint + .connect_with(self.cfg.setup.client.clone(), self.upstream, &name) + .map_err(io::Error::other)?; + let connection = tokio::time::timeout(self.cfg.connect_timeout, connecting) + .await + .map_err(|_| { + io::Error::new( + io::ErrorKind::TimedOut, + format!("{} did not complete a QUIC handshake", self.upstream), + ) + })? + .map_err(io::Error::other)?; + Ok((endpoint, connection)) + } + + /// An endpoint wearing the client's address. + fn endpoint_as(&self, client: SocketAddr) -> io::Result { + if !self.cfg.spoof_source { + return self.endpoint_plain(); + } + let socket = + crate::transparent::bind_as_udp(client.ip(), self.upstream, self.cfg.self_mark)?; + let runtime = quinn::default_runtime() + .ok_or_else(|| io::Error::other("no async runtime for the QUIC endpoint"))?; + Endpoint::new(quinn::EndpointConfig::default(), None, socket, runtime) + } + + fn endpoint_plain(&self) -> io::Result { + let bind: SocketAddr = if self.upstream.is_ipv6() { + "[::]:0".parse().unwrap() + } else { + "0.0.0.0:0".parse().unwrap() + }; + let socket = std::net::UdpSocket::bind(bind)?; + if let Some(mark) = self.cfg.self_mark { + use std::os::fd::AsRawFd; + crate::transparent::set_self_mark(socket.as_raw_fd(), mark)?; + } + let runtime = quinn::default_runtime() + .ok_or_else(|| io::Error::other("no async runtime for the QUIC endpoint"))?; + Endpoint::new(quinn::EndpointConfig::default(), None, socket, runtime) + } + + /// Carry one connection until either end is done with it. + async fn relay( + &self, + peer: Connection, + service: Behind, + client: SocketAddr, + chain: ChainHandle, + agreed: Option<&[u8]>, + ) { + // One flag for the whole connection, not one per stream. The question the + // deadline asks is whether this peer has said anything at all, and a connection + // that opens a hundred streams and speaks on none is exactly the shape it is + // there to catch. + let spoken = Arc::new(AtomicBool::new(false)); + let watchdog = self.cfg.first_byte_timeout.map(|deadline| { + let spoken = Arc::clone(&spoken); + let peer = peer.clone(); + let service = service.quic().cloned(); + let stats = Arc::clone(&self.stats); + tokio::spawn(async move { + tokio::select! { + _ = tokio::time::sleep(deadline) => { + if !spoken.load(Ordering::Relaxed) { + stats.no_first_byte.fetch_add(1, Ordering::Relaxed); + peer.close(VarInt::from_u32(0), b"nothing was ever said"); + if let Some(service) = &service { + service.close(VarInt::from_u32(0), b"nothing was ever said"); + } + } + } + // Closed rather than slept through, or a service taking short + // connections would hold one sleeping task per connection for the + // whole deadline. + _ = peer.closed() => {} + } + }) + }); + + let carrier = Carrier { + peer: peer.clone(), + service, + client, + upstream: self.upstream, + capture: self.cfg.capture.clone(), + chain, + stats: Arc::clone(&self.stats), + spoken, + }; + + // HTTP/3 is carried by something that understands it, and everything else by + // something that does not have to. The difference is not an optimisation: h3 + // puts a request's method, path and headers in a QPACK-compressed HEADERS frame, + // so a filter shown the raw stream would be shown a compression format. Ending + // the protocol here is what lets the chain be shown a request. + if agreed == Some(b"h3") { + crate::h3::carry(&carrier).await; + } else if carrier.service.quic().is_none() { + // Everything that is not HTTP/3 is opaque bytes on a stream, and opaque bytes + // have no HTTP/1.1 form to be sent in. Refused here, once, with a reason the + // client can read — rather than carried into a translation nobody could + // describe, or dropped silently. + eprintln!( + "[warn] [quic] {client}: this service is reached over HTTP/1.1, so only \ + HTTP/3 can be carried to it — the connection agreed on something else.", + ); + peer.close( + VarInt::from_u32(REFUSED_CODE), + b"this endpoint carries HTTP/3 only", + ); + } else { + // Together, because a connection carrying both must not have one wait for the + // other: each ends when the connection does. + tokio::join!(carrier.carry_streams(), carrier.carry_datagrams()); + } + + if let Some(watchdog) = watchdog { + watchdog.abort(); + } + } +} + +/// What one established connection's streams are carried with. +/// +/// Cloned per stream rather than shared behind a lock: everything in it is a handle +/// already — two connection handles, the chain's watch receiver, two `Arc`s — so a clone +/// is a few reference counts and the streams need nothing from each other. +/// What is behind this connection, and therefore what its exchanges are forwarded to. +/// +/// It was always a QUIC connection: the engine dialled the service before accepting the +/// client, so that the protocol the client is told is the one the service chose. That is +/// still the default and still the only thing a *raw* QUIC stream can be relayed to — a +/// stream of opaque bytes has nowhere else to go. +/// +/// The second arm is for the service that does not speak QUIC at all. An HTTP/3 exchange +/// is already rendered to the chain as HTTP/1.1, so it can be sent to a service that +/// speaks HTTP/1.1 and nothing else, which is what puts firegex in front of an ordinary +/// web service as the thing that terminates QUIC for it. Two things follow and are worth +/// stating where they are decided: +/// +/// * **it is HTTP/3 or nothing.** A raw QUIC stream has no HTTP/1.1 form, so a connection +/// that agrees on anything else is refused rather than carried into a translation +/// nobody could describe; +/// * **the ALPN stops being mirrored**, because there is no longer a service to ask. The +/// invariant everywhere else — never tell the client something the service did not say +/// — changes meaning here rather than being broken: the service does not speak QUIC, +/// firegex does, and `h3` is what firegex is speaking. +#[derive(Clone)] +pub(crate) enum Behind { + /// The service speaks QUIC. Streams are opened on this connection. + Quic(Connection), + /// The service speaks HTTP/1.1, dialled once per exchange. + Http1(crate::h1up::H1Upstream), +} + +impl Behind { + /// The upstream QUIC connection, where there is one. + pub(crate) fn quic(&self) -> Option<&Connection> { + match self { + Behind::Quic(connection) => Some(connection), + Behind::Http1(_) => None, + } + } +} + +#[derive(Clone)] +pub(crate) struct Carrier { + pub(crate) peer: Connection, + pub(crate) service: Behind, + pub(crate) client: SocketAddr, + pub(crate) upstream: SocketAddr, + pub(crate) capture: Option>, + pub(crate) chain: ChainHandle, + pub(crate) stats: Arc, + pub(crate) spoken: Arc, +} + +impl Carrier { + /// Relay whatever streams either end opens, as byte streams and nothing more. + async fn carry_streams(&self) { + let peer = self.peer.clone(); + // Raw streams are opaque bytes, so the only thing they can be relayed to is + // another QUIC connection. A service that speaks HTTP/1.1 is refused before a + // client is ever accepted — there is no HTTP/1.1 form of "some bytes on a + // stream" — so this is unreachable rather than a case to handle. + let Some(service) = self.service.quic().cloned() else { + return; + }; + let client = self.client; + loop { + tokio::select! { + // A stream the client opened. Both halves exist already, so the only + // thing that can block is opening its twin upstream — which is done in + // the task, or one stream waiting on the service's flow control would + // stop this loop accepting anybody else's. + opened = peer.accept_bi() => match opened { + Ok((send, recv)) => self.spawn_bi(recv, send, Origin::Client), + Err(e) => { service.close(VarInt::from_u32(0), b"the client is gone"); break log_end(client, "client", e); } + }, + opened = peer.accept_uni() => match opened { + Ok(recv) => self.spawn_uni(recv, Origin::Client), + Err(e) => { service.close(VarInt::from_u32(0), b"the client is gone"); break log_end(client, "client", e); } + }, + // And one the service opened. Rare outside HTTP/3's control streams, but + // a protocol nobody here has heard of may do it at any time. + opened = service.accept_bi() => match opened { + Ok((send, recv)) => self.spawn_bi(recv, send, Origin::Service), + Err(e) => { peer.close(VarInt::from_u32(0), b"the service is gone"); break log_end(client, "service", e); } + }, + opened = service.accept_uni() => match opened { + Ok(recv) => self.spawn_uni(recv, Origin::Service), + Err(e) => { peer.close(VarInt::from_u32(0), b"the service is gone"); break log_end(client, "service", e); } + }, + } + } + } +} + +/// Close both ends because a rule refused something on one of them. +fn refuse(peer: &Connection, service: &Connection) { + peer.close(VarInt::from_u32(REFUSED_CODE), b"blocked by firegex"); + service.close(VarInt::from_u32(REFUSED_CODE), b"blocked by firegex"); +} + +fn log_end(client: SocketAddr, side: &str, e: quinn::ConnectionError) { + match e { + quinn::ConnectionError::ApplicationClosed(_) + | quinn::ConnectionError::LocallyClosed + | quinn::ConnectionError::ConnectionClosed(_) => {} + other => eprintln!("[info] [quic] {client}: the {side} ended the connection: {other}"), + } +} + +/// Which end opened a stream. Not the same question as which direction its bytes go: +/// both ends can write to a bidirectional stream whoever opened it. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Origin { + Client, + Service, +} + +impl Carrier { + /// End this connection because a rule said so. + /// + /// Both ends, with the same reason, spelled once: a client reading why it was cut off + /// and a service reading why its peer went away are the same event, and the two + /// halves drifting apart is how a close comes to mean two things. + pub(crate) fn refuse(&self) { + match self.service.quic() { + Some(service) => refuse(&self.peer, service), + // An HTTP/1.1 service has no connection of its own to end: the socket under + // this exchange is dropped with it, which is what closing one means there. + None => self + .peer + .close(VarInt::from_u32(REFUSED_CODE), b"blocked by firegex"), + } + } + + /// Tell the chain about a stream, in the words the rest of the engine uses. + /// + /// **One stream is one connection to a filter.** That is the honest mapping rather + /// than a convenience: a filter's state — for Python, a whole set of module globals — + /// follows a stream of bytes from its start to its end, which is what a QUIC stream + /// is and what a QUIC *connection* is not. On HTTP/3 it also lands where an operator + /// would expect, one request to a stream. The addresses are the connection's, because + /// they are the same for every stream on it. + pub(crate) fn open(&self) -> ConnectionId { + // Neither of the two answers that existed before this: a QUIC stream is a stream, + // so everything the library builds on one applies, and it rides on UDP, which is + // what the rules matched to get here. + self.open_as(L4::Quic) + } + + /// Tell the chain about this connection's datagrams, which are one flow of their own. + /// + /// One for the connection and not one per datagram: a datagram carries no identity a + /// flow could be keyed on beyond the connection it arrived in, which is the same + /// answer the plain UDP relay reaches from the other direction, where the flow is the + /// client address. Both directions share it, as they do there. + fn open_datagrams(&self) -> ConnectionId { + self.open_as(L4::QuicDatagram) + } + + fn open_as(&self, l4: L4) -> ConnectionId { + let connection = next_connection_id(); + self.chain.current().connection_opened( + connection, + &ConnectionMeta { + client: self.client, + server: self.upstream, + l4, + }, + ); + connection + } + + /// This connection's datagrams, in both directions. + /// + /// A refused datagram is **dropped, and the connection carries on**. That is a block + /// meaning something different from what it means one function up, and the difference + /// belongs to the traffic rather than to a preference: a stream is a conversation that + /// can be ended, and a datagram is already complete by the time it is judged — ending + /// the connection over one would cost every stream riding on it. The plain UDP relay + /// answers the same way. + /// + /// What a datagram does *not* lose is the filter's memory of the ones before it: the + /// flow keeps its sessions, so a pattern split across two datagrams a millisecond + /// apart is still found, which is the relay's rule again. What it loses is everything + /// built on a stream, and that is [`L4::QuicDatagram`]'s doing rather than this + /// function's. + async fn carry_datagrams(&self) { + // Nothing is opened for a connection that cannot carry them. A flow costs a + // filter its state — for Python, a whole set of module globals — and the flow is + // built on the first datagram rather than here, so a connection that negotiates + // the extension and never uses it costs nothing at all. + // Datagrams go from one QUIC connection to another and nowhere else: there is no + // HTTP/1.1 spelling of one, so a service reached that way carries none. + let Some(service) = self.service.quic().cloned() else { + return; + }; + if self.peer.max_datagram_size().is_none() && service.max_datagram_size().is_none() { + return; + } + let flow = tokio::sync::OnceCell::new(); + tokio::join!( + self.datagrams(&service, Direction::ClientToServer, &flow), + self.datagrams(&service, Direction::ServerToClient, &flow), + ); + // Released once, by whichever direction opened it: the two share one flow, and a + // filter told twice that a connection ended would be asked to forget it twice. + if let Some(&connection) = flow.get() { + self.chain.current().connection_closed(connection); + } + } + + /// One direction of the datagram flow. + async fn datagrams( + &self, + service: &Connection, + direction: Direction, + flow: &tokio::sync::OnceCell, + ) { + let (from, to) = match direction { + Direction::ClientToServer => (&self.peer, service), + Direction::ServerToClient => (service, &self.peer), + }; + let mut sessions: Option = None; + let mut complained = false; + loop { + // The error is the connection ending, which the stream carrier is already + // watching and reporting; saying it twice would put two lines in the log for + // one event. + let Ok(datagram) = from.read_datagram().await else { + return; + }; + // A datagram is something said, exactly as a byte on a stream is. + self.spoken.store(true, Ordering::Relaxed); + let connection = *flow.get_or_init(|| async { self.open_datagrams() }).await; + let sessions = sessions.get_or_insert_with(|| ChainSessions::new(connection)); + // Re-read every time, so a ruleset pushed mid-flow takes effect without + // anybody losing a connection. + if let Verdict::Reject(_) = self + .chain + .current() + .run(direction, &datagram, sessions) + .await + { + continue; + } + if let Err(e) = to.send_datagram(datagram) { + // Once per direction per connection: a datagram too large for the far end + // is a property of the two peers, so the second one fails for the same + // reason as the first and the thousandth would too. + if !complained { + complained = true; + eprintln!("[info] [quic] {}: cannot carry a datagram: {e}", self.client); + } + } + } + } + + /// A reconstruction of one stream, for whoever is watching the capture interface. + /// + /// Per stream and not per connection, because a stream is the thing that has a + /// beginning, an order and an end — everything a TCP conversation is made of. What it + /// does not have is a port of its own, so [`Tap::open_stream`] invents one; the + /// documentation says so wherever this interface is offered. + pub(crate) fn tap(&self) -> Option> { + Tap::open_stream(self.capture.clone(), self.client, self.upstream) + } + + /// A bidirectional stream: both directions, inspected independently. + fn spawn_bi(&self, near_recv: quinn::RecvStream, near_send: quinn::SendStream, origin: Origin) { + let connection = self.open(); + let tap = self.tap(); + let peer = self.peer.clone(); + // As in `carry_streams`: opaque bytes have nowhere but another QUIC connection + // to go, so this arm cannot be reached with an HTTP/1.1 service behind. + let Some(service) = self.service.quic().cloned() else { + return; + }; + let chain = self.chain.clone(); + let stats = Arc::clone(&self.stats); + let spoken = Arc::clone(&self.spoken); + tokio::spawn(async move { + // Opened here rather than in the accept loop: the far end's flow control can + // make this wait, and waiting must cost this stream and no other. + let far = match origin { + Origin::Client => service.open_bi().await, + Origin::Service => peer.open_bi().await, + }; + let (far_send, far_recv) = match far { + Ok(pair) => pair, + Err(e) => { + eprintln!("[info] [quic] cannot carry a stream through: {e}"); + chain.current().connection_closed(connection); + return; + } + }; + + // Which end the bytes came *from* is what names the direction, whoever opened + // the stream: the service answering on a stream it opened itself is still the + // service talking. + let (client_rd, client_wr, service_rd, service_wr) = match origin { + Origin::Client => (near_recv, near_send, far_recv, far_send), + Origin::Service => (far_recv, far_send, near_recv, near_send), + }; + + let (up, down) = spawn_pumps( + client_rd, client_wr, service_rd, service_wr, &chain, &stats, connection, + tap.clone(), spoken, + // The deadline is the connection's, and its watchdog is already running. + None, + ); + if join_pumps(up, down).await { + // A refusal ends the connection, not the stream. Killing one stream of + // many would leave the client free to ask again on the next one, which is + // not what a block means anywhere else in firegex — and on HTTP/3 it is + // the same thing keep-alive does on HTTP/1.1, where a refused request + // takes the connection with it. + peer.close(VarInt::from_u32(REFUSED_CODE), b"blocked by firegex"); + service.close(VarInt::from_u32(REFUSED_CODE), b"blocked by firegex"); + } + if let Some(tap) = tap { + tap.closed(); + } + chain.current().connection_closed(connection); + }); + } + + /// A unidirectional stream: one pump, and nothing coming back. + fn spawn_uni(&self, near_recv: quinn::RecvStream, origin: Origin) { + let connection = self.open(); + let tap = self.tap(); + let peer = self.peer.clone(); + // As in `carry_streams`: opaque bytes have nowhere but another QUIC connection + // to go, so this arm cannot be reached with an HTTP/1.1 service behind. + let Some(service) = self.service.quic().cloned() else { + return; + }; + let chain = self.chain.clone(); + let stats = Arc::clone(&self.stats); + let spoken = Arc::clone(&self.spoken); + tokio::spawn(async move { + let far = match origin { + Origin::Client => service.open_uni().await, + Origin::Service => peer.open_uni().await, + }; + let far_send = match far { + Ok(send) => send, + Err(e) => { + eprintln!("[info] [quic] cannot carry a stream through: {e}"); + chain.current().connection_closed(connection); + return; + } + }; + let direction = match origin { + Origin::Client => Direction::ClientToServer, + Origin::Service => Direction::ServerToClient, + }; + // Its own stop channel, which nothing else holds: there is no other direction + // of this stream to wind down, and the refusal below closes the connection + // that every other stream is riding on anyway. + let (stop_tx, stop_rx) = tokio::sync::watch::channel(false); + let outcome = pump( + near_recv, far_send, direction, chain.clone(), stats, connection, tap.clone(), + spoken, stop_tx, stop_rx, + ) + .await; + if matches!(outcome, Ok(PumpOutcome::Rejected)) { + refuse(&peer, &service); + } + if let Some(tap) = tap { + tap.closed(); + } + chain.current().connection_closed(connection); + }); + } +} + +/// What the service agreed to speak. +fn negotiated_protocol(connection: &Connection) -> Option> { + let data = connection.handshake_data()?; + let data = data.downcast::().ok()?; + data.protocol +} + +/// Binds and tracks the QUIC endpoints of one service, one per protected address. +/// +/// The same shape as [`crate::udp::UdpManager`] and answering the same question, because +/// the backend asks it in the same words: a QUIC service is UDP as far as the rules are +/// concerned, so what it wants back is a port per address to point them at. +#[derive(Clone)] +pub struct QuicManager { + chain: ChainHandle, + cfg: QuicConfig, + stats: Arc, + relays: Arc>>, +} + +impl QuicManager { + pub fn new(chain: ChainHandle, cfg: QuicConfig, stats: Arc) -> Self { + Self { + chain, + cfg, + stats, + relays: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + } + } + + pub async fn add_relay(&self, upstream: SocketAddr, onward: Onward) -> io::Result { + let mut map = self.relays.lock().await; + if let Some(&port) = map.get(&upstream) { + return Ok(port); + } + let bind: SocketAddr = if upstream.is_ipv6() { + "[::]:0".parse().unwrap() + } else { + "0.0.0.0:0".parse().unwrap() + }; + // The relay's own copy, because what the service behind it speaks is this + // address's answer and not the process's. + let mut cfg = self.cfg.clone(); + cfg.upstream = onward; + let relay = QuicRelay::bind( + bind, + upstream, + self.chain.clone(), + cfg, + Arc::clone(&self.stats), + )?; + let port = relay.local_addr()?.port(); + map.insert(upstream, port); + tokio::spawn(Arc::new(relay).serve()); + Ok(port) + } +} diff --git a/backend/proxysrc/src/relays.rs b/backend/proxysrc/src/relays.rs new file mode 100644 index 0000000..ce4d8ca --- /dev/null +++ b/backend/proxysrc/src/relays.rs @@ -0,0 +1,39 @@ +//! The per-address relays a UDP-family service is made of. +//! +//! Two kinds, one question. `SO_ORIGINAL_DST` is TCP and SCTP only, so neither a +//! datagram relay nor a QUIC endpoint can recover where traffic was headed: both are +//! bound one per protected address with the upstream already known, and both answer the +//! backend with the port it should point that address's rule at. Which of the two a +//! service gets is decided once, by what the service speaks. +//! +//! An enum rather than a trait: there are two, the set is not open, and the control +//! channel only ever asks them the one thing. + +use std::io; +use std::net::SocketAddr; + +use crate::quic::QuicManager; +use crate::udp::UdpManager; + +#[derive(Clone)] +pub enum Relays { + /// Datagrams relayed as they are, each client's flow filtered on its own. + Datagram(UdpManager), + /// QUIC, terminated: the streams inside are what the filters see. + Quic(QuicManager), +} + +impl Relays { + /// Bind a relay for one protected address, or report the one already bound. + pub async fn add_relay( + &self, + upstream: SocketAddr, + onward: crate::proxy::Onward, + ) -> io::Result { + match self { + // A plain datagram relay forwards bytes; there is nothing for it to speak. + Relays::Datagram(manager) => manager.add_relay(upstream).await, + Relays::Quic(manager) => manager.add_relay(upstream, onward).await, + } + } +} diff --git a/backend/proxysrc/src/tls.rs b/backend/proxysrc/src/tls.rs index b83b4f7..6266439 100644 --- a/backend/proxysrc/src/tls.rs +++ b/backend/proxysrc/src/tls.rs @@ -103,6 +103,24 @@ pub fn client_config() -> Result, String> { Ok(Arc::new(config)) } +/// The same thing for QUIC, where TLS 1.3 is not a preference. +/// +/// QUIC carries its handshake in 1.3 and only 1.3, and a config still offering 1.2 is +/// refused outright by `QuicClientConfig` rather than quietly negotiating down — so the +/// version list is narrowed here, where the reason can be written next to it, instead of +/// at the call site as a conversion error nobody can act on. +pub fn quic_client_config() -> Result { + let provider = provider(); + Ok( + ClientConfig::builder_with_provider(Arc::clone(&provider)) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|e| format!("cannot configure QUIC: {e}"))? + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServer(provider))) + .with_no_client_auth(), + ) +} + /// The process-wide crypto provider, installed once. fn provider() -> Arc { if let Some(installed) = rustls::crypto::CryptoProvider::get_default() { diff --git a/backend/proxysrc/src/transparent.rs b/backend/proxysrc/src/transparent.rs index d258ede..b3d0793 100644 --- a/backend/proxysrc/src/transparent.rs +++ b/backend/proxysrc/src/transparent.rs @@ -196,12 +196,18 @@ pub async fn connect_plain(upstream: SocketAddr, self_mark: Option) -> io:: socket.connect(upstream).await } -/// Dial the service as the client over UDP: same source address, kernel-chosen source port. -pub async fn connect_as_udp( +/// A UDP socket wearing the client's address, with nothing dialled yet. +/// +/// Split out from [`connect_as_udp`] for QUIC: quinn owns its socket and sends to an +/// address per packet, which a *connected* socket refuses (`EISCONN`). Everything that +/// makes the address someone else's — `IP_TRANSPARENT`, the mark the intercept rules +/// skip, and the bind to a foreign address — happens here; who it then talks to is the +/// caller's business. +pub fn bind_as_udp( client: IpAddr, upstream: SocketAddr, self_mark: Option, -) -> io::Result { +) -> io::Result { if client.is_ipv6() != upstream.is_ipv6() { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -222,7 +228,16 @@ pub async fn connect_as_udp( socket.set_reuse_address(true)?; let bind_addr: socket2::SockAddr = SocketAddr::new(client, 0).into(); socket.bind(&bind_addr)?; - let std_sock: std::net::UdpSocket = socket.into(); + Ok(socket.into()) +} + +/// Dial the service as the client over UDP: same source address, kernel-chosen source port. +pub async fn connect_as_udp( + client: IpAddr, + upstream: SocketAddr, + self_mark: Option, +) -> io::Result { + let std_sock = bind_as_udp(client, upstream, self_mark)?; let tokio_sock = UdpSocket::from_std(std_sock)?; tokio_sock.connect(upstream).await?; Ok(tokio_sock) diff --git a/backend/proxysrc/src/udp.rs b/backend/proxysrc/src/udp.rs index 449c024..73f8dc9 100644 --- a/backend/proxysrc/src/udp.rs +++ b/backend/proxysrc/src/udp.rs @@ -29,7 +29,7 @@ use tokio::net::UdpSocket; use crate::proxy::ProxyStats; use crate::filter::{ next_connection_id, ChainHandle, ChainSessions, ConnectionId, ConnectionMeta, Direction, - Verdict, + Verdict, L4, }; /// The largest datagram this relay will carry. Comfortably past the practical MTU and @@ -251,7 +251,7 @@ impl UdpRelay { &ConnectionMeta { client, server: self.upstream, - tcp: false, + l4: L4::Udp, }, ); diff --git a/backend/proxysrc/tests/h2.rs b/backend/proxysrc/tests/h2.rs new file mode 100644 index 0000000..1d03b90 --- /dev/null +++ b/backend/proxysrc/tests/h2.rs @@ -0,0 +1,959 @@ +//! HTTP/2 terminated by the engine, and shown to the filters as HTTP/1.1. +//! +//! The point of the rendering is that a filter written once works on every version: a +//! pattern that blocks a path over HTTP/1.1 has to block the same path over HTTP/2, where +//! on the wire that path was an HPACK-compressed header block. These tests put a real h2 +//! client and a real h2 service either side of the proxy and check what the chain was +//! shown — and, where the question is what reached the *service*, what arrived there. +//! +//! They are the h3 suite's cases, asked again of the other protocol, plus the two that +//! only HTTP/2 can get wrong: a head that is the whole message has to go out as one, and +//! concurrent streams on one connection must not share a filter's state. + +use std::net::SocketAddr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use fgex_proxy::filter::{ChainHandle, FilterChain}; +use fgex_proxy::proxy::{Proxy, ProxyConfig, TlsSetup}; +use fgex_proxy::spec::parse_filters; +use fgex_proxy::tls; +use tokio::net::{TcpListener, TcpStream}; + +fn self_signed() -> (String, String) { + let cert = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .unwrap(); + (cert.cert.pem(), cert.key_pair.serialize_pem()) +} + +/// What a service did with one exchange, as the tests need to see it. +#[derive(Default)] +struct Seen { + /// Every trailer the service was handed, as `name=value`. + trailers: Mutex>, + /// How many requests it answered. + requests: AtomicUsize, +} + +/// How the stand-in service behaves. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Service { + /// Answers with what it was asked, body and all. + Echo, + /// Answers the moment the head arrives, without waiting for a body — the shape of a + /// bidirectional RPC and of every service that replies before an upload finishes. + Eager, + /// Answers with a head and nothing else, the way a gRPC status-only reply does. + TrailersOnly, +} + +/// A real HTTP/2 service, over TLS, standing where the protected service would be. +async fn spawn_service(cert: &str, key: &str, how: Service) -> (SocketAddr, Arc) { + let seen = Arc::new(Seen::default()); + let mut config = (*tls::server_config(cert, key).unwrap()).clone(); + config.alpn_protocols = vec![b"h2".to_vec()]; + let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let recorded = Arc::clone(&seen); + tokio::spawn(async move { + loop { + let Ok((sock, _)) = listener.accept().await else { + return; + }; + let acceptor = acceptor.clone(); + let recorded = Arc::clone(&recorded); + tokio::spawn(async move { + let Ok(tls) = acceptor.accept(sock).await else { + return; + }; + let Ok(connection) = h2::server::handshake(tls).await else { + return; + }; + serve(connection, how, recorded).await; + }); + } + }); + (addr, seen) +} + +/// What the stand-in service does with the exchanges on one connection. +/// +/// Shared by the TLS and the cleartext service so the two cannot answer differently: the +/// h2c tests are asking whether the *proxy* behaves the same without TLS in the way, and a +/// service that differed would make that unanswerable. +async fn serve(mut connection: h2::server::Connection, how: Service, seen: Arc) +where + T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, +{ + while let Some(Ok((request, mut respond))) = connection.accept().await { + let seen = Arc::clone(&seen); + tokio::spawn(async move { + seen.requests.fetch_add(1, Ordering::Relaxed); + let path = request.uri().path().to_string(); + let (_, mut body) = request.into_parts(); + + if how == Service::Eager { + let answer = "the service spoke first"; + let response = http::Response::builder() + .status(200) + .header("content-length", answer.len().to_string()) + .body(()) + .unwrap(); + let mut out = respond.send_response(response, false).unwrap(); + let _ = out.send_data(Bytes::from(answer), true); + return; + } + + let mut collected = Vec::new(); + while let Some(Ok(chunk)) = body.data().await { + let len = chunk.len(); + collected.extend_from_slice(&chunk); + let _ = body.flow_control().release_capacity(len); + } + if let Ok(Some(trailers)) = body.trailers().await { + for (name, value) in trailers.iter() { + seen.trailers + .lock() + .unwrap() + .push(format!("{name}={}", value.to_str().unwrap_or("?"))); + } + } + + if how == Service::TrailersOnly { + // HEADERS with END_STREAM and nothing else — what a gRPC status-only + // answer is on the wire. + let response = http::Response::builder() + .status(200) + .header("content-type", "application/grpc") + .header("grpc-status", "5") + .body(()) + .unwrap(); + let _ = respond.send_response(response, true); + return; + } + + // One path answers with something a rule is looking for, in bytes that appear + // nowhere in the request: that is what makes a test of the *answer* a test of + // the answer. + let answer = if path == "/secret" { + "here is FLAG{only-in-the-answer}".to_string() + } else { + format!( + "you asked for {path} and sent {} bytes: {}", + collected.len(), + String::from_utf8_lossy(&collected) + ) + }; + let response = http::Response::builder() + .status(200) + .header("content-length", answer.len().to_string()) + .body(()) + .unwrap(); + let mut out = respond.send_response(response, false).unwrap(); + let _ = out.send_data(Bytes::from(answer), true); + }); + } +} + +/// The same service, speaking HTTP/2 in the clear: no TLS, prior knowledge, which is how +/// most gRPC is actually deployed behind a load balancer. +async fn spawn_cleartext_service(how: Service) -> (SocketAddr, Arc) { + let seen = Arc::new(Seen::default()); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let recorded = Arc::clone(&seen); + tokio::spawn(async move { + loop { + let Ok((sock, _)) = listener.accept().await else { + return; + }; + let recorded = Arc::clone(&recorded); + tokio::spawn(async move { + let Ok(connection) = h2::server::handshake(sock).await else { + return; + }; + serve(connection, how, recorded).await; + }); + } + }); + (addr, seen) +} + +/// A plain relay with no TLS on either edge, which is what a `tcp` service is. +async fn spawn_plain_proxy(upstream: SocketAddr, filters: &str) -> SocketAddr { + let chain = ChainHandle::new(FilterChain::new( + parse_filters(filters).unwrap(), + Duration::from_millis(500), + )); + let cfg = ProxyConfig::fixed("127.0.0.1:0".parse().unwrap(), upstream); + let proxy = Proxy::bind(cfg, chain).await.unwrap(); + let addr = proxy.local_addr().unwrap(); + tokio::spawn(proxy.serve()); + addr +} + +/// An h2 connection in the clear, the way a prior-knowledge client opens one. +async fn connect_cleartext( + addr: SocketAddr, +) -> Result<(h2::client::SendRequest, tokio::task::JoinHandle<()>), String> { + let sock = TcpStream::connect(addr).await.map_err(|e| e.to_string())?; + let (sender, connection) = h2::client::handshake(sock).await.map_err(|e| e.to_string())?; + let driving = tokio::spawn(async move { + let _ = connection.await; + }); + Ok((sender, driving)) +} + +/// The engine in front of it, terminating TLS on both edges as a `tls` service does. +async fn spawn_proxy(upstream: SocketAddr, cert: &str, key: &str, filters: &str) -> SocketAddr { + let chain = ChainHandle::new(FilterChain::new( + parse_filters(filters).unwrap(), + Duration::from_millis(500), + )); + let mut cfg = ProxyConfig::fixed("127.0.0.1:0".parse().unwrap(), upstream); + cfg.tls = TlsSetup { + server: Some(tls::server_config(cert, key).unwrap()), + upstream: Some(tls::client_config().unwrap()), + optional: false, + }; + let proxy = Proxy::bind(cfg, chain).await.unwrap(); + let addr = proxy.local_addr().unwrap(); + tokio::spawn(proxy.serve()); + addr +} + +/// An h2 connection to the proxy, offering `h2` so the service's answer is `h2`. +async fn connect(addr: SocketAddr) -> Result<(h2::client::SendRequest, tokio::task::JoinHandle<()>), String> { + let mut config = (*tls::client_config().unwrap()).clone(); + config.alpn_protocols = vec![b"h2".to_vec()]; + let connector = tokio_rustls::TlsConnector::from(Arc::new(config)); + let sock = TcpStream::connect(addr).await.map_err(|e| e.to_string())?; + let name = tls::server_name("localhost").map_err(|e| e.to_string())?; + let tls = connector.connect(name, sock).await.map_err(|e| e.to_string())?; + let (sender, connection) = h2::client::handshake(tls).await.map_err(|e| e.to_string())?; + let driving = tokio::spawn(async move { + let _ = connection.await; + }); + Ok((sender, driving)) +} + +/// How a request's body is framed on the way out. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Body { + /// No body at all: HEADERS with END_STREAM. + None, + /// A body whose length is declared up front. + Declared(&'static [u8]), + /// A body whose length is not declared, so the rendering has to frame it. + Undeclared(&'static [u8]), +} + +/// One request through the proxy, and what came back. +async fn request(addr: SocketAddr, method: &str, path: &str, body: Body) -> Result { + request_with_trailer(addr, method, path, body, None).await +} + +/// The same, optionally ending with a trailer section. +async fn request_with_trailer( + addr: SocketAddr, + method: &str, + path: &str, + body: Body, + trailer: Option<&str>, +) -> Result { + let (mut sender, driving) = connect(addr).await?; + let answer = async { + let mut builder = http::Request::builder() + .method(method) + .uri(format!("https://localhost{path}")); + let payload = match body { + Body::None => None, + Body::Declared(bytes) => { + builder = builder.header("content-length", bytes.len().to_string()); + Some(bytes) + } + Body::Undeclared(bytes) => Some(bytes), + }; + let outgoing = builder.body(()).map_err(|e| e.to_string())?; + let ends_here = payload.is_none() && trailer.is_none(); + let (response, mut out) = sender + .send_request(outgoing, ends_here) + .map_err(|e| e.to_string())?; + if let Some(bytes) = payload { + out.send_data(Bytes::from_static(bytes), trailer.is_none()) + .map_err(|e| e.to_string())?; + } + if let Some(trailer) = trailer { + let mut trailers = http::HeaderMap::new(); + trailers.insert("x-note", http::HeaderValue::from_str(trailer).unwrap()); + out.send_trailers(trailers).map_err(|e| e.to_string())?; + } + let response = response.await.map_err(|e| e.to_string())?; + let mut got = Vec::new(); + let mut incoming = response.into_body(); + while let Some(chunk) = incoming.data().await { + let chunk = chunk.map_err(|e| e.to_string())?; + let len = chunk.len(); + got.extend_from_slice(&chunk); + let _ = incoming.flow_control().release_capacity(len); + } + Ok(String::from_utf8_lossy(&got).into_owned()) + } + .await; + driving.abort(); + answer +} + +/// A chain with a rule in it that nothing will ever match. +/// +/// Not the same as no chain at all, and the difference is the whole of what these tests +/// are about: a service with no filters is **bypassed**, and a bypassed connection is +/// carried as bytes rather than terminated, so nothing below would be exercised. Spelling +/// it `""` here passed every assertion while proving nothing about HTTP/2 — which is the +/// class of false green this engine's tests exist to avoid, so it is spelled out. +const LIVE: &str = "block:ZZ-NO-SUCH-BYTES-ZZ"; + +/// The whole arrangement, for the cases that only need to ask one question. +async fn relay(how: Service, filters: &str) -> (SocketAddr, Arc) { + let (cert, key) = self_signed(); + let (upstream, seen) = spawn_service(&cert, &key, how).await; + let addr = spawn_proxy(upstream, &cert, &key, filters).await; + (addr, seen) +} + +#[tokio::test] +async fn an_exchange_goes_through() { + let (addr, _) = relay(Service::Echo, LIVE).await; + let answer = request(addr, "GET", "/hello", Body::None).await.unwrap(); + assert_eq!(answer, "you asked for /hello and sent 0 bytes: "); +} + +/// The whole reason for rendering the exchange: a pattern written against an HTTP request +/// matches one that arrived as an HPACK header block. +#[tokio::test] +async fn a_pattern_matches_the_request_line() { + let (addr, _) = relay(Service::Echo, "block:GET /admin").await; + + let allowed = request(addr, "GET", "/public", Body::None).await.unwrap(); + assert!(allowed.contains("/public"), "an unrelated request was affected: {allowed}"); + + let refused = request(addr, "GET", "/admin", Body::None).await; + assert!( + refused.is_err(), + "a request the chain refused was answered anyway: {refused:?}" + ); +} + +/// A header the request did not carry must not appear in what the filters are shown. +/// +/// A filter looking for `transfer-encoding` is usually looking for request smuggling, so +/// one invented by the proxy in front of it is the worst possible answer. +#[tokio::test] +async fn a_request_with_no_body_is_shown_no_framing_header() { + let (addr, _) = relay(Service::Echo, "block:transfer-encoding").await; + let answer = request(addr, "GET", "/plain", Body::None).await.unwrap(); + assert!(answer.contains("/plain"), "a framing header was invented: {answer}"); +} + +/// And a body whose length was never declared is shown chunked, because that is what that +/// message is in HTTP/1.1 — so a parser reading the rendering finds a body. +#[tokio::test] +async fn a_body_of_unknown_length_is_shown_chunked() { + let (addr, _) = relay(Service::Echo, "block:transfer-encoding").await; + let refused = request(addr, "POST", "/upload", Body::Undeclared(b"anything")).await; + assert!( + refused.is_err(), + "a body with no declared length was shown unframed: {refused:?}" + ); +} + +/// A declared length is passed through as it was, and the body shown raw beneath it. +#[tokio::test] +async fn a_declared_length_is_left_alone() { + let (addr, _) = relay(Service::Echo, "block:transfer-encoding").await; + let answer = request(addr, "POST", "/upload", Body::Declared(b"declared")) + .await + .unwrap(); + assert!(answer.contains("sent 8 bytes"), "the body did not arrive: {answer}"); +} + +/// Both directions are inspected, and a refused answer does not reach the client. +#[tokio::test] +async fn the_answer_is_inspected_too() { + let (addr, _) = relay(Service::Echo, "block:FLAG{").await; + + let answer = request(addr, "GET", "/harmless", Body::None).await; + assert!(answer.is_ok(), "an unrelated exchange was affected: {answer:?}"); + + // Nothing in this request carries the needle; everything that does is in the reply. + let refused = request(addr, "GET", "/secret", Body::None).await; + match refused { + Err(_) => {} + Ok(body) => panic!("the answer reached the client anyway: {body}"), + } +} + +/// A trailer section on a message the rendering can frame is shown to the chain, which is +/// the only reason it may then be forwarded. +/// +/// The bodiless case is the one that matters most: a message of HEADERS and TRAILERS and +/// nothing in between is exactly a gRPC trailers-only request, and without a chunked body +/// to hang a trailer section off the rendering would end at the head and the trailers +/// would travel to the far side unread. +#[tokio::test] +async fn a_trailer_section_is_shown_to_the_chain() { + for (body, what) in [ + (Body::None, "no body at all"), + (Body::Undeclared(b"harmless"), "a body of undeclared length"), + ] { + let (addr, _) = relay(Service::Echo, "block:SNEAKY").await; + let refused = request_with_trailer(addr, "POST", "/up", body, Some("SNEAKY")).await; + assert!( + refused.is_err(), + "a needle in the trailers of a message with {what} was not seen: {refused:?}" + ); + } +} + +/// And where the rendering cannot carry one, it is dropped rather than forwarded unread. +/// +/// A declared length means the HTTP/1.1 view of the message ends where the header said it +/// would, so there is nowhere to put a trailer section — and forwarding a piece of a +/// message no filter was shown is the one thing this engine does not do anywhere else. +#[tokio::test] +async fn a_trailer_the_rendering_cannot_carry_is_not_forwarded() { + let (addr, seen) = relay(Service::Echo, LIVE).await; + let answer = + request_with_trailer(addr, "POST", "/up", Body::Declared(b"harmless"), Some("SNEAKY")).await; + assert!(answer.is_ok(), "the exchange was disturbed: {answer:?}"); + assert!( + seen.trailers.lock().unwrap().is_empty(), + "a trailer section no filter was shown reached the service: {:?}", + seen.trailers.lock().unwrap() + ); + + // Without the declared length the same trailers are renderable, so they are shown and + // then carried: the rule is what the chain saw, not a dislike of trailers. + let (addr, seen) = relay(Service::Echo, LIVE).await; + let answer = request_with_trailer( + addr, + "POST", + "/up", + Body::Undeclared(b"harmless"), + Some("SNEAKY"), + ) + .await; + assert!(answer.is_ok(), "the exchange was disturbed: {answer:?}"); + assert_eq!( + seen.trailers.lock().unwrap().as_slice(), + ["x-note=SNEAKY"], + "a trailer section the chain accepted was dropped anyway" + ); +} + +/// An exchange the service is supposed to speak first in is not held waiting for a body. +/// +/// The head is held only until the framing is known, and a client that sends a head and +/// then waits to be answered — a bidirectional RPC, gRPC's among them — never sends one. +/// Holding it there deadlocks the exchange: the proxy waiting for a body, the client +/// waiting for the answer that would tell it what to send. +#[tokio::test] +async fn a_service_that_speaks_first_is_not_waited_out() { + let (addr, _) = relay(Service::Eager, LIVE).await; + let (mut sender, driving) = connect(addr).await.unwrap(); + + let outgoing = http::Request::builder() + .method("POST") + .uri("https://localhost/stream") + .body(()) + .unwrap(); + // Deliberately no body and no end of stream: this client is waiting to be spoken to. + let (response, _out) = sender.send_request(outgoing, false).unwrap(); + let answered = tokio::time::timeout(Duration::from_secs(5), response).await; + driving.abort(); + assert!( + answered.is_ok(), + "the head never reached a service that answers without waiting for a body" + ); + assert_eq!(answered.unwrap().unwrap().status(), 200); +} + +/// A head that is the whole message goes out as one, and does not grow a body on the way. +/// +/// This is the case HTTP/3 could not get wrong and HTTP/2 can: the end of a message rides +/// on its last frame, so sending the head open and closing it with an empty DATA frame +/// turns a *trailers-only* answer into a message with a body. That is precisely the shape +/// of a gRPC status-only reply, and gRPC clients refuse the mangled version. +#[tokio::test] +async fn a_head_that_ends_the_message_stays_that_way() { + let (addr, _) = relay(Service::TrailersOnly, LIVE).await; + let (mut sender, driving) = connect(addr).await.unwrap(); + + let outgoing = http::Request::builder() + .method("POST") + .uri("https://localhost/rpc") + .header("content-type", "application/grpc") + .body(()) + .unwrap(); + let (response, _out) = sender.send_request(outgoing, true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("the answer never arrived") + .expect("the answer was an error"); + assert_eq!(response.headers().get("grpc-status").unwrap(), "5"); + let mut body = response.into_body(); + assert!( + body.is_end_stream(), + "a head-only answer reached the client with a body attached to it" + ); + assert!( + body.data().await.is_none(), + "an empty DATA frame was invented on the way through" + ); + driving.abort(); +} + +/// Concurrent streams on one connection are separate connections to the chain. +/// +/// On HTTP/1.1 a keep-alive connection carries its requests one after another, so one set +/// of filter state is the honest answer. HTTP/2 interleaves them, and a filter whose state +/// was shared between two streams would be one client's bytes deciding another client's +/// verdict — which is also a way to smuggle a pattern past a filter by splitting it across +/// two streams. Here half a needle goes up each of two concurrent streams; neither may +/// block. +#[tokio::test] +async fn concurrent_streams_do_not_share_filter_state() { + let (addr, _) = relay(Service::Echo, "block:NEEDLEHALVES").await; + let (mut sender, driving) = connect(addr).await.unwrap(); + + let mut waiting = Vec::new(); + for half in ["NEEDLE", "HALVES"] { + let outgoing = http::Request::builder() + .method("POST") + .uri("https://localhost/split") + .body(()) + .unwrap(); + let (response, mut out) = sender.send_request(outgoing, false).unwrap(); + out.send_data(Bytes::from(half), true).unwrap(); + waiting.push(response); + } + for response in waiting { + let answered = tokio::time::timeout(Duration::from_secs(5), response).await; + assert!( + matches!(&answered, Ok(Ok(r)) if r.status() == 200), + "a pattern split across two streams was matched as if they were one: {answered:?}" + ); + } + driving.abort(); +} + +/// And one connection really does carry them at the same time. +#[tokio::test] +async fn one_connection_carries_many_streams() { + let (addr, seen) = relay(Service::Echo, LIVE).await; + let (mut sender, driving) = connect(addr).await.unwrap(); + + let mut waiting = Vec::new(); + for n in 0..8 { + let outgoing = http::Request::builder() + .method("GET") + .uri(format!("https://localhost/n{n}")) + .body(()) + .unwrap(); + let (response, _) = sender.send_request(outgoing, true).unwrap(); + waiting.push((n, response)); + } + for (n, response) in waiting { + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("an answer never arrived") + .expect("an answer was an error"); + let mut body = response.into_body(); + let mut got = Vec::new(); + while let Some(Ok(chunk)) = body.data().await { + let len = chunk.len(); + got.extend_from_slice(&chunk); + let _ = body.flow_control().release_capacity(len); + } + let got = String::from_utf8_lossy(&got).into_owned(); + assert!( + got.contains(&format!("/n{n}")), + "two streams of one connection were mixed up: {got}" + ); + } + driving.abort(); + assert_eq!(seen.requests.load(Ordering::Relaxed), 8); +} + +/// A refusal ends the connection carrying it, not just the stream. +/// +/// Resetting one stream would leave the client free to ask again on the next, which is not +/// what a block means anywhere else in firegex — and on HTTP/1.1 keep-alive a refused +/// request already takes the connection with it. +#[tokio::test] +async fn a_refusal_ends_the_connection() { + let (addr, _) = relay(Service::Echo, "block:GET /admin").await; + let (mut sender, driving) = connect(addr).await.unwrap(); + + let refused = http::Request::builder() + .method("GET") + .uri("https://localhost/admin") + .body(()) + .unwrap(); + let (response, _) = sender.send_request(refused, true).unwrap(); + let _ = tokio::time::timeout(Duration::from_secs(5), response).await; + + // The connection is gone, so a second request on it cannot be answered — whether it is + // refused outright or simply never answered. + let again = http::Request::builder() + .method("GET") + .uri("https://localhost/public") + .body(()) + .unwrap(); + let answered = match sender.send_request(again, true) { + Err(_) => None, + Ok((response, _)) => tokio::time::timeout(Duration::from_secs(2), response) + .await + .ok() + .and_then(|r| r.ok()), + }; + driving.abort(); + assert!( + answered.is_none(), + "a connection whose stream was refused went on answering: {answered:?}" + ); +} + +/// A service with no filters is carried as bytes, and that is deliberate. +/// +/// Rendering an exchange for a chain that has nothing to say is work paid for no answer, +/// so a bypassed chain — a service with no filters, or one whose filters have all been +/// disabled, or a connection admitted past the limit with `over_limit_forwards` — takes +/// the byte pump instead. The visible consequence is that nothing is dropped, filtered or +/// rendered, which this pins with the one case that can tell the two paths apart: a +/// trailer section on a message of declared length, which a terminated connection drops +/// and a forwarded one carries. +/// +/// The cost of the trade is that a connection admitted while the chain was empty stays a +/// byte pump for its whole life, even if a filter is pushed a moment later. +#[tokio::test] +async fn an_empty_chain_is_carried_as_bytes() { + let (addr, seen) = relay(Service::Echo, "").await; + let answer = + request_with_trailer(addr, "POST", "/up", Body::Declared(b"harmless"), Some("SNEAKY")).await; + assert!(answer.is_ok(), "the exchange was disturbed: {answer:?}"); + assert_eq!( + seen.trailers.lock().unwrap().as_slice(), + ["x-note=SNEAKY"], + "an unfiltered connection was terminated and rendered anyway" + ); +} + +/// HTTP/2 in the clear is rendered too, which is how most gRPC is actually deployed. +/// +/// A `tcp` service with no TLS anywhere: the client opens with the connection preface and +/// the proxy recognises it. Without this the exchange is HPACK on the wire and nothing in +/// the chain can read it — the same silence the TLS path had, on the layer where a gRPC +/// service behind a load balancer usually lives. +#[tokio::test] +async fn cleartext_http2_is_rendered_as_well() { + let (upstream, _) = spawn_cleartext_service(Service::Echo).await; + let addr = spawn_plain_proxy(upstream, "block:GET /admin").await; + + let (mut sender, driving) = connect_cleartext(addr).await.unwrap(); + let outgoing = http::Request::builder() + .method("GET") + .uri("http://localhost/public") + .body(()) + .unwrap(); + let (response, _) = sender.send_request(outgoing, true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("no answer came back") + .expect("the answer was an error"); + assert_eq!(response.status(), 200); + driving.abort(); + + // And the pattern matches a request line that was HPACK on the wire. + let (mut sender, driving) = connect_cleartext(addr).await.unwrap(); + let outgoing = http::Request::builder() + .method("GET") + .uri("http://localhost/admin") + .body(()) + .unwrap(); + let (response, _) = sender.send_request(outgoing, true).unwrap(); + let answered = tokio::time::timeout(Duration::from_secs(5), response).await; + driving.abort(); + assert!( + !matches!(&answered, Ok(Ok(_))), + "a cleartext HTTP/2 request the chain refused was answered anyway: {answered:?}" + ); +} + +/// Looking for the preface must not make a service that speaks first wait for it. +/// +/// This is the trap the whole sniff is arranged around: SMTP, SSH and most game protocols +/// send a banner before the client says anything, and a proxy that holds the connection +/// until the client speaks breaks all of them in a way that looks random. The service +/// having spoken is proof this is not a prior-knowledge HTTP/2 client, so the banner is +/// what ends the wait. +#[tokio::test] +async fn a_banner_service_is_not_held_by_the_preface_check() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + use tokio::io::AsyncWriteExt; + let _ = sock.write_all(b"220 service ready\r\n").await; + let _ = sock.flush().await; + // Held open, as a real banner protocol would, so the test is about the + // banner arriving and not about the connection ending. + tokio::time::sleep(Duration::from_secs(30)).await; + }); + } + }); + let addr = spawn_plain_proxy(upstream, LIVE).await; + + let mut sock = TcpStream::connect(addr).await.unwrap(); + let mut buf = [0u8; 19]; + let banner = tokio::time::timeout(Duration::from_millis(200), async { + use tokio::io::AsyncReadExt; + sock.read_exact(&mut buf).await + }) + .await; + assert!( + matches!(banner, Ok(Ok(_))), + "a service that speaks first was held waiting for an HTTP/2 preface: {banner:?}" + ); + assert_eq!(&buf[..], b"220 service ready\r\n"); +} + +/// And a plain HTTP/1.1 client is not mistaken for one, nor delayed by the check. +#[tokio::test] +async fn cleartext_http1_is_not_mistaken_for_http2() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = vec![0u8; 1024]; + let n = sock.read(&mut buf).await.unwrap_or(0); + let seen = String::from_utf8_lossy(&buf[..n]).into_owned(); + let body = format!("saw {} bytes", seen.len()); + let _ = sock + .write_all( + format!( + "HTTP/1.1 200 OK\r\ncontent-length: {}\r\n\r\n{body}", + body.len() + ) + .as_bytes(), + ) + .await; + }); + } + }); + let addr = spawn_plain_proxy(upstream, LIVE).await; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut sock = TcpStream::connect(addr).await.unwrap(); + sock.write_all(b"GET /plain HTTP/1.1\r\nhost: x\r\n\r\n") + .await + .unwrap(); + let mut buf = vec![0u8; 1024]; + let n = tokio::time::timeout(Duration::from_secs(5), sock.read(&mut buf)) + .await + .expect("an HTTP/1.1 client was left waiting by the preface check") + .unwrap(); + let answer = String::from_utf8_lossy(&buf[..n]).into_owned(); + assert!( + answer.starts_with("HTTP/1.1 200"), + "a plain HTTP/1.1 exchange was disturbed: {answer}" + ); +} + +/// One relay, one certificate, one chain — carrying the clear and the encrypted alike. +/// +/// This is what an `http` service is: a daemon answering in the clear on one port and +/// under TLS on another used to need two firegex services with their filter chains copied +/// between them by hand, and a chain kept in step by hand is one that stops protecting one +/// of them silently. The edge is decided per connection, from what the client actually +/// sent, which is the same rule the ALPN mirroring follows — firegex carries what the two +/// ends are doing rather than deciding it for them. +#[tokio::test] +async fn optional_tls_carries_both_edges_through_one_chain() { + let (cert, key) = self_signed(); + // Two services, because a real daemon listens twice: one socket in the clear and one + // under TLS. One relay in front of both. + let (encrypted, _) = spawn_service(&cert, &key, Service::Echo).await; + let (cleartext, _) = spawn_cleartext_service(Service::Echo).await; + + for (upstream, tls_on_the_wire) in [(encrypted, true), (cleartext, false)] { + let chain = ChainHandle::new(FilterChain::new( + parse_filters("block:GET /admin").unwrap(), + Duration::from_millis(500), + )); + let mut cfg = ProxyConfig::fixed("127.0.0.1:0".parse().unwrap(), upstream); + cfg.tls = TlsSetup { + server: Some(tls::server_config(&cert, &key).unwrap()), + upstream: Some(tls::client_config().unwrap()), + optional: true, + }; + let proxy = Proxy::bind(cfg, chain).await.unwrap(); + let addr = proxy.local_addr().unwrap(); + tokio::spawn(proxy.serve()); + + let open = |addr| async move { + if tls_on_the_wire { + connect(addr).await + } else { + connect_cleartext(addr).await + } + }; + + let (mut sender, driving) = open(addr).await.unwrap(); + let outgoing = http::Request::builder() + .method("GET") + .uri("https://localhost/public") + .body(()) + .unwrap(); + let (response, _) = sender.send_request(outgoing, true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .unwrap_or_else(|_| panic!("no answer on the {} edge", edge(tls_on_the_wire))) + .unwrap_or_else(|e| panic!("the {} edge answered an error: {e}", edge(tls_on_the_wire))); + assert_eq!(response.status(), 200); + driving.abort(); + + // And the one chain refuses on both, which is the whole point of one service. + let (mut sender, driving) = open(addr).await.unwrap(); + let outgoing = http::Request::builder() + .method("GET") + .uri("https://localhost/admin") + .body(()) + .unwrap(); + let (response, _) = sender.send_request(outgoing, true).unwrap(); + let answered = tokio::time::timeout(Duration::from_secs(5), response).await; + driving.abort(); + assert!( + !matches!(&answered, Ok(Ok(_))), + "the {} edge answered a request the chain refused: {answered:?}", + edge(tls_on_the_wire) + ); + } +} + +fn edge(tls: bool) -> &'static str { + if tls { + "TLS" + } else { + "cleartext" + } +} + +/// A cleartext **HTTP/1.1** service, which is what most of the web is. +async fn spawn_http1_service(seen: Arc>>) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let seen = Arc::clone(&seen); + tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = vec![0u8; 8192]; + let read = match socket.read(&mut buf).await { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + seen.lock() + .unwrap() + .push(String::from_utf8_lossy(&buf[..read]).to_string()); + let _ = socket + .write_all( + b"HTTP/1.1 200 OK\r\ncontent-type: text/plain\r\n\ + content-length: 5\r\n\r\nhello", + ) + .await; + }); + } + }); + addr +} + +/// The engine terminating HTTP/2 from the client in front of an HTTP/1.1 service. +async fn spawn_proxy_to_http1( + upstream: SocketAddr, + cert: &str, + key: &str, + filters: &str, +) -> SocketAddr { + use fgex_proxy::proxy::{Edge, Onward, Published}; + let chain = ChainHandle::new(FilterChain::new( + parse_filters(filters).unwrap(), + Duration::from_millis(500), + )); + let mut cfg = ProxyConfig::fixed("127.0.0.1:0".parse().unwrap(), upstream); + cfg.tls = TlsSetup { + server: Some(tls::server_config(cert, key).unwrap()), + upstream: Some(tls::client_config().unwrap()), + optional: true, + }; + // What the address says: the service behind it speaks HTTP/1.1 in the clear, whatever + // the client brought. That is the whole of what makes this edge different. + cfg.targets.publish( + upstream, + Published { target: None, edge: Edge::Whatever, upstream: Onward::Plain }, + ); + let proxy = Proxy::bind(cfg, chain).await.unwrap(); + let addr = proxy.local_addr().unwrap(); + tokio::spawn(proxy.serve()); + addr +} + +#[tokio::test] +async fn http2_is_carried_to_a_service_that_speaks_http1() { + // The deployment an operator actually has: an ordinary cleartext web service, reached + // over HTTP/2 with only firegex holding a certificate. It was reachable over HTTP/3 + // long before it was reachable over HTTP/2 — same rendering underneath, and the TLS + // edge simply had nobody to copy an ALPN from, so it answered nothing and every + // client fell back to HTTP/1.1. + let (cert, key) = self_signed(); + let seen = Arc::new(Mutex::new(Vec::new())); + let service = spawn_http1_service(Arc::clone(&seen)).await; + let proxy = spawn_proxy_to_http1(service, &cert, &key, LIVE).await; + + let answer = request(proxy, "GET", "/files/ok", Body::None).await.unwrap(); + assert!(answer.contains("hello"), "the answer did not come back: {answer}"); + + // And what the service was handed is HTTP/1.1 — the same bytes the chain was shown, + // which is what makes this a forward rather than a translation invented here. + let arrived = seen.lock().unwrap().clone(); + assert_eq!(arrived.len(), 1, "the service saw {} requests", arrived.len()); + assert!( + arrived[0].starts_with("GET /files/ok HTTP/1.1\r\n"), + "the service was not spoken to in HTTP/1.1: {:?}", + arrived[0] + ); +} + +#[tokio::test] +async fn a_pattern_blocks_over_http2_in_front_of_http1() { + // The promise the rendering exists for, on the edge that just gained it: one filter, + // written once, refusing the same request whatever version carried it. + let (cert, key) = self_signed(); + let seen = Arc::new(Mutex::new(Vec::new())); + let service = spawn_http1_service(Arc::clone(&seen)).await; + let proxy = spawn_proxy_to_http1(service, &cert, &key, "block:/secret").await; + + let refused = request(proxy, "GET", "/secret", Body::None).await; + assert!(refused.is_err(), "the request was carried: {refused:?}"); + assert!( + seen.lock().unwrap().is_empty(), + "the refused request reached the service anyway" + ); +} diff --git a/backend/proxysrc/tests/h3.rs b/backend/proxysrc/tests/h3.rs new file mode 100644 index 0000000..0e6d940 --- /dev/null +++ b/backend/proxysrc/tests/h3.rs @@ -0,0 +1,703 @@ +//! HTTP/3 carried by the engine, and shown to the filters as HTTP/1.1. +//! +//! The point of the translation is that a filter written once works on both: a pattern +//! that blocks a path over TCP has to block the same path over QUIC, where on the wire +//! that path was a QPACK-compressed header block. These tests put a real HTTP/3 client and +//! a real HTTP/3 service either side of the relay and check what the chain was shown. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use bytes::{Buf, Bytes}; +use fgex_proxy::filter::{ChainHandle, FilterChain}; +use fgex_proxy::proxy::ProxyStats; +use fgex_proxy::proxy::Onward; +use fgex_proxy::quic::{QuicConfig, QuicRelay, QuicSetup}; +use fgex_proxy::spec::parse_filters; +use fgex_proxy::tls; +use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig}; + +fn self_signed() -> (String, String) { + let cert = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .unwrap(); + (cert.cert.pem(), cert.key_pair.serialize_pem()) +} + +/// An HTTP/3 service that answers with what it was asked, body and all. +async fn spawn_h3_echo(cert: &str, key: &str) -> SocketAddr { + let mut config = (*tls::server_config(cert, key).unwrap()).clone(); + config.alpn_protocols = vec![b"h3".to_vec()]; + let endpoint = quinn::Endpoint::server( + quinn::ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(config).unwrap())), + "127.0.0.1:0".parse().unwrap(), + ) + .unwrap(); + let addr = endpoint.local_addr().unwrap(); + tokio::spawn(async move { + while let Some(incoming) = endpoint.accept().await { + tokio::spawn(async move { + let Ok(connection) = incoming.await else { + return; + }; + let Ok(mut h3) = h3::server::builder() + .build(h3_quinn::Connection::new(connection)) + .await + else { + return; + }; + while let Ok(Some(resolver)) = h3.accept().await { + tokio::spawn(async move { + let Ok((request, mut stream)) = resolver.resolve_request().await else { + return; + }; + let mut body = Vec::new(); + while let Ok(Some(mut chunk)) = stream.recv_data().await { + let len = chunk.remaining(); + body.extend_from_slice(&chunk.copy_to_bytes(len)); + } + // One path answers with something a rule is looking for, and + // says it in bytes that appear nowhere in the request: that is + // what makes a test of the *answer* a test of the answer. + let answer = if request.uri().path() == "/secret" { + "here is FLAG{only-in-the-answer}".to_string() + } else { + format!( + "you asked for {} and sent {} bytes: {}", + request.uri().path(), + body.len(), + String::from_utf8_lossy(&body) + ) + }; + let response = http::Response::builder() + .status(http::StatusCode::OK) + .header("content-length", answer.len().to_string()) + .body(()) + .unwrap(); + let _ = stream.send_response(response).await; + let _ = stream.send_data(Bytes::from(answer)).await; + let _ = stream.finish().await; + }); + } + }); + } + }); + addr +} + +/// A service that speaks HTTP/1.1 and nothing else, which is what most of the web is. +/// +/// Deliberately hand-written rather than a crate: what these tests are about is what +/// leaves this engine, so the far end has to be something that reads exactly what an +/// HTTP/1.1 service reads and nothing more forgiving. +async fn spawn_h1_echo() -> SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = Vec::new(); + let mut chunk = [0u8; 4096]; + // The head, then whatever the framing says the body is. + let head_end = loop { + let n = match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + buf.extend_from_slice(&chunk[..n]); + if let Some(at) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break at + 4; + } + }; + let head = String::from_utf8_lossy(&buf[..head_end]).to_lowercase(); + let path = head.split_whitespace().nth(1).unwrap_or("/").to_string(); + let declared = head + .split("content-length:") + .nth(1) + .and_then(|rest| rest.split("\r\n").next()) + .and_then(|v| v.trim().parse::().ok()); + let mut body = buf[head_end..].to_vec(); + match declared { + Some(want) => { + while body.len() < want { + let n = match socket.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + body.extend_from_slice(&chunk[..n]); + } + body.truncate(want.min(body.len())); + } + // Chunked, which is what an undeclared body is in HTTP/1.1 — and what + // the rendering shows a filter, so it is what has to arrive here. + None if head.contains("transfer-encoding: chunked") => { + let mut decoded = Vec::new(); + loop { + while !body.windows(2).any(|w| w == b"\r\n") { + let n = match socket.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + body.extend_from_slice(&chunk[..n]); + } + let Some(at) = body.windows(2).position(|w| w == b"\r\n") else { + break; + }; + let size = usize::from_str_radix( + String::from_utf8_lossy(&body[..at]).trim(), + 16, + ) + .unwrap_or(0); + if size == 0 { + break; + } + body.drain(..at + 2); + while body.len() < size + 2 { + let n = match socket.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + body.extend_from_slice(&chunk[..n]); + } + decoded.extend_from_slice(&body[..size.min(body.len())]); + body.drain(..(size + 2).min(body.len())); + } + body = decoded; + } + None => body.clear(), + } + let answer = format!( + "you asked for {path} and sent {} bytes: {}", + body.len(), + String::from_utf8_lossy(&body) + ); + let _ = socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{answer}", + answer.len() + ) + .as_bytes(), + ) + .await; + let _ = socket.flush().await; + }); + } + }); + addr +} + +async fn spawn_relay(upstream: SocketAddr, filters: &str) -> SocketAddr { + spawn_relay_to(upstream, filters, Onward::Same).await +} + +async fn spawn_relay_to( + upstream: SocketAddr, + filters: &str, + protocol: Onward, +) -> SocketAddr { + let (cert, key) = self_signed(); + let chain = ChainHandle::new(FilterChain::new( + parse_filters(filters).unwrap(), + Duration::from_millis(500), + )); + let cfg = QuicConfig { + setup: Arc::new(QuicSetup::build(&cert, &key, vec![b"h3".to_vec()]).unwrap()), + self_mark: None, + spoof_source: false, + max_connections: 0, + over_limit_forwards: false, + first_byte_timeout: None, + connect_timeout: Duration::from_secs(5), + upstream: protocol, + // No interface to reconstruct anything onto, which is every host without the + // capture device — what it would carry is pinned by the integration suite, which + // has a real one. + capture: None, + }; + let relay = QuicRelay::bind( + "127.0.0.1:0".parse().unwrap(), + upstream, + chain, + cfg, + Arc::new(ProxyStats::default()), + ) + .unwrap(); + let addr = relay.local_addr().unwrap(); + tokio::spawn(Arc::new(relay).serve()); + addr +} + +/// One request through the relay, and what came back. +async fn request( + addr: SocketAddr, + method: &str, + path: &str, + body: Option<&[u8]>, + declare_length: bool, +) -> Result { + let mut config = tls::quic_client_config().unwrap(); + config.alpn_protocols = vec![b"h3".to_vec()]; + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new( + QuicClientConfig::try_from(config).unwrap(), + ))); + let connection = endpoint + .connect(addr, "localhost") + .map_err(|e| e.to_string())? + .await + .map_err(|e| e.to_string())?; + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .map_err(|e| e.to_string())?; + let driving = tokio::spawn(async move { + let _ = driver.wait_idle().await; + }); + + let answer = async { + let mut builder = http::Request::builder() + .method(method) + .uri(format!("https://localhost{path}")); + if declare_length { + builder = builder.header("content-length", body.map(|b| b.len()).unwrap_or(0).to_string()); + } + let outgoing = builder.body(()).map_err(|e| e.to_string())?; + let mut stream = sender.send_request(outgoing).await.map_err(|e| e.to_string())?; + if let Some(body) = body { + stream + .send_data(Bytes::copy_from_slice(body)) + .await + .map_err(|e| e.to_string())?; + } + stream.finish().await.map_err(|e| e.to_string())?; + stream.recv_response().await.map_err(|e| e.to_string())?; + let mut got = Vec::new(); + while let Some(mut chunk) = stream.recv_data().await.map_err(|e| e.to_string())? { + let len = chunk.remaining(); + got.extend_from_slice(&chunk.copy_to_bytes(len)); + } + Ok(String::from_utf8_lossy(&got).into_owned()) + } + .await; + driving.abort(); + answer +} + +#[tokio::test] +async fn an_exchange_goes_through() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_echo(&cert, &key).await; + let addr = spawn_relay(upstream, "").await; + + let answer = request(addr, "GET", "/hello", None, false).await.unwrap(); + assert_eq!(answer, "you asked for /hello and sent 0 bytes: "); +} + +/// The whole reason for rendering the exchange: a pattern written against an HTTP +/// request matches one that arrived as a QPACK header block. +#[tokio::test] +async fn a_pattern_matches_the_request_line() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_echo(&cert, &key).await; + let addr = spawn_relay(upstream, "block:GET /admin").await; + + let allowed = request(addr, "GET", "/public", None, false).await.unwrap(); + assert!(allowed.contains("/public"), "an unrelated request was affected: {allowed}"); + + let refused = request(addr, "GET", "/admin", None, false).await; + assert!( + refused.is_err(), + "a request the chain refused was answered anyway: {refused:?}" + ); +} + +/// A header the request did not carry must not appear in what the filters are shown. +/// +/// This is what the head is held back for. A bodiless request rendered as +/// `transfer-encoding: chunked` would be a lie in front of every filter looking for that +/// header — and a filter looking for it is usually looking for request smuggling, which +/// makes a fabricated one the worst possible answer. +#[tokio::test] +async fn a_request_with_no_body_is_shown_no_framing_header() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_echo(&cert, &key).await; + let addr = spawn_relay(upstream, "block:transfer-encoding").await; + + let answer = request(addr, "GET", "/plain", None, false).await.unwrap(); + assert!(answer.contains("/plain"), "a framing header was invented: {answer}"); +} + +/// And a body whose length was never declared is shown chunked, because that is what +/// that message is in HTTP/1.1 — so a parser reading the rendering finds a body. +#[tokio::test] +async fn a_body_of_unknown_length_is_shown_chunked() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_echo(&cert, &key).await; + let addr = spawn_relay(upstream, "block:transfer-encoding").await; + + let refused = request(addr, "POST", "/upload", Some(b"anything"), false).await; + assert!( + refused.is_err(), + "a body with no declared length was shown unframed: {refused:?}" + ); +} + +/// A declared length is passed through as it was, and the body shown raw beneath it. +#[tokio::test] +async fn a_declared_length_is_left_alone() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_echo(&cert, &key).await; + let addr = spawn_relay(upstream, "block:transfer-encoding").await; + + let answer = request(addr, "POST", "/upload", Some(b"declared"), true) + .await + .unwrap(); + assert!(answer.contains("sent 8 bytes"), "the body did not arrive: {answer}"); +} + +/// Both directions are inspected, and a refused answer does not reach the client. +#[tokio::test] +async fn the_answer_is_inspected_too() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_echo(&cert, &key).await; + let addr = spawn_relay(upstream, "block:FLAG{").await; + + let answer = request(addr, "GET", "/harmless", None, false).await; + assert!(answer.is_ok(), "an unrelated exchange was affected: {answer:?}"); + + // Nothing in this request carries the needle; everything that does is in the reply. + let refused = request(addr, "GET", "/secret", None, false).await; + match refused { + Err(_) => {} + Ok(body) => panic!("the answer reached the client anyway: {body}"), + } +} + +/// A service that answers a fixed body and remembers the trailer section it was sent. +/// +/// Separate from the echo above on purpose: what these tests are asking is what reached +/// the *service*, which an answer that reflects the request cannot tell them apart from +/// what reached the client. +async fn spawn_h3_recorder( + cert: &str, + key: &str, +) -> (SocketAddr, Arc>>) { + let seen: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut config = (*tls::server_config(cert, key).unwrap()).clone(); + config.alpn_protocols = vec![b"h3".to_vec()]; + let endpoint = quinn::Endpoint::server( + quinn::ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(config).unwrap())), + "127.0.0.1:0".parse().unwrap(), + ) + .unwrap(); + let addr = endpoint.local_addr().unwrap(); + let recorded = Arc::clone(&seen); + tokio::spawn(async move { + while let Some(incoming) = endpoint.accept().await { + let recorded = Arc::clone(&recorded); + tokio::spawn(async move { + let Ok(connection) = incoming.await else { return }; + let Ok(mut h3) = h3::server::builder() + .build(h3_quinn::Connection::new(connection)) + .await + else { + return; + }; + while let Ok(Some(resolver)) = h3.accept().await { + let recorded = Arc::clone(&recorded); + tokio::spawn(async move { + let Ok((_request, mut stream)) = resolver.resolve_request().await else { + return; + }; + while let Ok(Some(_)) = stream.recv_data().await {} + if let Ok(Some(trailers)) = stream.recv_trailers().await { + for (name, value) in trailers.iter() { + recorded.lock().unwrap().push(format!( + "{name}={}", + value.to_str().unwrap_or("?") + )); + } + } + let answer = "recorded"; + let response = http::Response::builder() + .status(http::StatusCode::OK) + .header("content-length", answer.len().to_string()) + .body(()) + .unwrap(); + let _ = stream.send_response(response).await; + let _ = stream.send_data(Bytes::from(answer)).await; + let _ = stream.finish().await; + }); + } + }); + } + }); + (addr, seen) +} + +/// A service that answers the moment the head arrives, without waiting for a body. +/// +/// The shape of a bidirectional RPC and of every service that replies before an upload +/// finishes — a `413`, a redirect, a refusal. +async fn spawn_h3_eager(cert: &str, key: &str) -> SocketAddr { + let mut config = (*tls::server_config(cert, key).unwrap()).clone(); + config.alpn_protocols = vec![b"h3".to_vec()]; + let endpoint = quinn::Endpoint::server( + quinn::ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(config).unwrap())), + "127.0.0.1:0".parse().unwrap(), + ) + .unwrap(); + let addr = endpoint.local_addr().unwrap(); + tokio::spawn(async move { + while let Some(incoming) = endpoint.accept().await { + tokio::spawn(async move { + let Ok(connection) = incoming.await else { return }; + let Ok(mut h3) = h3::server::builder() + .build(h3_quinn::Connection::new(connection)) + .await + else { + return; + }; + while let Ok(Some(resolver)) = h3.accept().await { + tokio::spawn(async move { + let Ok((_request, mut stream)) = resolver.resolve_request().await else { + return; + }; + let answer = "the service spoke first"; + let response = http::Response::builder() + .status(http::StatusCode::OK) + .header("content-length", answer.len().to_string()) + .body(()) + .unwrap(); + let _ = stream.send_response(response).await; + let _ = stream.send_data(Bytes::from(answer)).await; + let _ = stream.finish().await; + }); + } + }); + } + }); + addr +} + +/// One request that ends with a trailer section, and what came back. +async fn request_with_trailer( + addr: SocketAddr, + body: &[u8], + declare_length: bool, + trailer: &str, +) -> Result { + let mut config = tls::quic_client_config().unwrap(); + config.alpn_protocols = vec![b"h3".to_vec()]; + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new( + QuicClientConfig::try_from(config).unwrap(), + ))); + let connection = endpoint + .connect(addr, "localhost") + .map_err(|e| e.to_string())? + .await + .map_err(|e| e.to_string())?; + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .map_err(|e| e.to_string())?; + let driving = tokio::spawn(async move { + let _ = driver.wait_idle().await; + }); + + let answer = async { + let mut builder = http::Request::builder() + .method("POST") + .uri("https://localhost/up"); + if declare_length { + builder = builder.header("content-length", body.len().to_string()); + } + let outgoing = builder.body(()).map_err(|e| e.to_string())?; + let mut stream = sender.send_request(outgoing).await.map_err(|e| e.to_string())?; + if !body.is_empty() { + stream + .send_data(Bytes::copy_from_slice(body)) + .await + .map_err(|e| e.to_string())?; + } + let mut trailers = http::HeaderMap::new(); + trailers.insert("x-note", http::HeaderValue::from_str(trailer).unwrap()); + stream.send_trailers(trailers).await.map_err(|e| e.to_string())?; + let _ = stream.finish().await; + stream.recv_response().await.map_err(|e| e.to_string())?; + let mut got = Vec::new(); + while let Some(mut chunk) = stream.recv_data().await.map_err(|e| e.to_string())? { + let len = chunk.remaining(); + got.extend_from_slice(&chunk.copy_to_bytes(len)); + } + Ok(String::from_utf8_lossy(&got).into_owned()) + } + .await; + driving.abort(); + answer +} + +/// A trailer section on a message whose body was never framed by a declared length is +/// shown to the chain, which is the only reason it may then be forwarded. +/// +/// The bodiless case is the one that matters most and the one that used to slip past: a +/// message of HEADERS and TRAILERS and nothing in between — which is exactly a gRPC +/// trailers-only answer — had no chunked body to hang a trailer section off, so the +/// rendering ended at the head and the trailers travelled to the far side unread. They +/// decide the framing now: in HTTP/1.1 a message with trailers *is* a chunked one. +#[tokio::test] +async fn a_trailer_section_is_shown_to_the_chain() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_echo(&cert, &key).await; + let addr = spawn_relay(upstream, "block:SNEAKY").await; + + for (body, declared, what) in [ + (&b""[..], false, "no body at all"), + (&b"harmless"[..], false, "a body of undeclared length"), + ] { + let refused = request_with_trailer(addr, body, declared, "SNEAKY").await; + assert!( + refused.is_err(), + "a needle in the trailers of a message with {what} was not seen: {refused:?}" + ); + } +} + +/// And where the rendering cannot carry one, it is dropped rather than forwarded unread. +/// +/// A declared length means the HTTP/1.1 view of the message ends where the header said it +/// would, so there is nowhere to put a trailer section — and forwarding a piece of a +/// message no filter was shown is the one thing this engine does not do anywhere else. +/// Dropping is what an intermediary is allowed to do with a trailer section it cannot +/// carry; the exchange itself is untouched. +#[tokio::test] +async fn a_trailer_the_rendering_cannot_carry_is_not_forwarded() { + let (cert, key) = self_signed(); + let (upstream, seen) = spawn_h3_recorder(&cert, &key).await; + let addr = spawn_relay(upstream, "").await; + + let answer = request_with_trailer(addr, b"harmless", true, "SNEAKY").await; + assert_eq!(answer.as_deref(), Ok("recorded"), "the exchange was disturbed"); + assert!( + seen.lock().unwrap().is_empty(), + "a trailer section no filter was shown reached the service: {:?}", + seen.lock().unwrap() + ); + + // Without the declared length the same trailers are renderable, so they are shown and + // then carried: the rule is what the chain saw, not a dislike of trailers. + let (upstream, seen) = spawn_h3_recorder(&cert, &key).await; + let addr = spawn_relay(upstream, "").await; + let answer = request_with_trailer(addr, b"harmless", false, "SNEAKY").await; + assert_eq!(answer.as_deref(), Ok("recorded")); + assert_eq!( + seen.lock().unwrap().as_slice(), + ["x-note=SNEAKY"], + "a trailer section the chain accepted was dropped anyway" + ); +} + +/// An exchange the service is supposed to speak first in is not held waiting for a body. +/// +/// The head is held only until the framing is known, and a client that sends a head and +/// then waits to be answered — a bidirectional RPC, gRPC's among them — never sends one. +/// Holding it there deadlocked the exchange: the proxy waiting for a body, the client +/// waiting for the answer that would tell it what to send. +#[tokio::test] +async fn a_service_that_speaks_first_is_not_waited_out() { + let (cert, key) = self_signed(); + let upstream = spawn_h3_eager(&cert, &key).await; + let addr = spawn_relay(upstream, "").await; + + let mut config = tls::quic_client_config().unwrap(); + config.alpn_protocols = vec![b"h3".to_vec()]; + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new( + QuicClientConfig::try_from(config).unwrap(), + ))); + let connection = endpoint.connect(addr, "localhost").unwrap().await.unwrap(); + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .unwrap(); + let driving = tokio::spawn(async move { + let _ = driver.wait_idle().await; + }); + + let outgoing = http::Request::builder() + .method("POST") + .uri("https://localhost/stream") + .body(()) + .unwrap(); + let mut stream = sender.send_request(outgoing).await.unwrap(); + // Deliberately no body and no finish: this client is waiting to be spoken to. + let answered = tokio::time::timeout(Duration::from_secs(5), stream.recv_response()).await; + driving.abort(); + assert!( + answered.is_ok(), + "the head never reached a service that answers without waiting for a body" + ); + assert_eq!(answered.unwrap().unwrap().status(), http::StatusCode::OK); +} + +// --- a service that speaks HTTP/1.1, behind an HTTP/3 edge ---------------------------- +// +// The one place in this engine where what leaves is not the version that arrived. It is +// the operator's choice (`FGEX_PROXY_UPSTREAM`) and it exists because most of the web is +// an HTTP/1.1 service that will never speak QUIC: firegex terminates HTTP/3 in front of +// it and sends on the rendering the filters were already being shown. + +#[tokio::test] +async fn an_http1_service_answers_an_http3_client() { + let upstream = spawn_h1_echo().await; + let addr = spawn_relay_to(upstream, "", Onward::Plain).await; + + let answer = request(addr, "GET", "/hello", None, false).await.unwrap(); + assert_eq!(answer, "you asked for /hello and sent 0 bytes: "); +} + +/// A body, declared — which is the framing that travels as a `content-length` and is +/// read back by the far end as one. +#[tokio::test] +async fn a_declared_body_reaches_an_http1_service() { + let upstream = spawn_h1_echo().await; + let addr = spawn_relay_to(upstream, "", Onward::Plain).await; + + let answer = request(addr, "POST", "/upload", Some(b"twelve bytes"), true) + .await + .unwrap(); + assert_eq!(answer, "you asked for /upload and sent 12 bytes: twelve bytes"); +} + +/// And one that declared nothing, which is chunked in HTTP/1.1 — the framing the chain +/// is shown, so it had better be the framing the service is sent. +#[tokio::test] +async fn an_undeclared_body_reaches_an_http1_service_chunked() { + let upstream = spawn_h1_echo().await; + let addr = spawn_relay_to(upstream, "", Onward::Plain).await; + + let answer = request(addr, "POST", "/upload", Some(b"twelve bytes"), false) + .await + .unwrap(); + assert_eq!(answer, "you asked for /upload and sent 12 bytes: twelve bytes"); +} + +/// The promise that makes the whole thing worth having: one filter file, and it does not +/// care which version either end speaks. The same pattern that blocks this request +/// against an HTTP/3 service blocks it against an HTTP/1.1 one. +#[tokio::test] +async fn a_pattern_matches_with_an_http1_service_behind() { + let upstream = spawn_h1_echo().await; + let addr = spawn_relay_to(upstream, "block:GET /admin", Onward::Plain).await; + + assert!( + request(addr, "GET", "/admin", None, false).await.is_err(), + "the request reached an HTTP/1.1 service that a pattern should have refused" + ); + let answer = request(addr, "GET", "/public", None, false).await.unwrap(); + assert_eq!(answer, "you asked for /public and sent 0 bytes: "); +} diff --git a/backend/proxysrc/tests/pyworker.rs b/backend/proxysrc/tests/pyworker.rs index c549abc..5394ed9 100644 --- a/backend/proxysrc/tests/pyworker.rs +++ b/backend/proxysrc/tests/pyworker.rs @@ -14,7 +14,7 @@ use std::io::Write; use std::time::{Duration, Instant}; use fgex_proxy::filter::{ - next_connection_id, ChainSessions, ConnectionMeta, Direction, FilterChain, Verdict, + next_connection_id, ChainSessions, ConnectionMeta, Direction, FilterChain, Verdict, L4, }; use fgex_proxy::rules::parse_ruleset; @@ -391,7 +391,7 @@ def only_from_that_client(packet: RawPacket): let meta = ConnectionMeta { client: "10.0.0.9:51000".parse().unwrap(), server: "10.0.0.1:8080".parse().unwrap(), - tcp: true, + l4: L4::Tcp, }; assert_eq!( feed_from(&c, &[b"hello"], Some(meta)).await, diff --git a/backend/proxysrc/tests/quic.rs b/backend/proxysrc/tests/quic.rs new file mode 100644 index 0000000..8e3b5f3 --- /dev/null +++ b/backend/proxysrc/tests/quic.rs @@ -0,0 +1,386 @@ +//! QUIC terminated by the engine, and the three things that has to get right. +//! +//! A client and a service, both speaking real QUIC, either side of the relay. What is +//! being proved is not that bytes arrive — it is that a rule can see them at all, which +//! on this transport is only true because the connection is terminated: on the wire the +//! stream boundaries and the frames are encrypted along with the payload. +//! +//! The protocol negotiated here is deliberately **not** `h3`. These are the tests for a +//! QUIC service that speaks something of its own, where a stream is a stream of bytes and +//! nothing reads them but the chain; HTTP/3 is carried by a module that understands it, +//! and is tested in `h3.rs`. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use fgex_proxy::filter::{ChainHandle, FilterChain}; +use fgex_proxy::proxy::ProxyStats; +use fgex_proxy::proxy::Onward; +use fgex_proxy::quic::{QuicConfig, QuicRelay, QuicSetup}; +use fgex_proxy::spec::parse_filters; +use fgex_proxy::tls; +use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig}; + +/// Generated per run: no key material belongs in the repository. +fn self_signed() -> (String, String) { + let cert = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .unwrap(); + (cert.cert.pem(), cert.key_pair.serialize_pem()) +} + +fn alpn(list: &[&str]) -> Vec> { + list.iter().map(|p| p.as_bytes().to_vec()).collect() +} + +/// A service that speaks QUIC and echoes every stream, standing in for the real one. +async fn spawn_quic_echo(cert: &str, key: &str, speaks: &[&str]) -> SocketAddr { + let mut config = (*tls::server_config(cert, key).unwrap()).clone(); + config.alpn_protocols = alpn(speaks); + let crypto = QuicServerConfig::try_from(config).unwrap(); + let endpoint = quinn::Endpoint::server( + quinn::ServerConfig::with_crypto(Arc::new(crypto)), + "127.0.0.1:0".parse().unwrap(), + ) + .unwrap(); + let addr = endpoint.local_addr().unwrap(); + tokio::spawn(async move { + while let Some(incoming) = endpoint.accept().await { + tokio::spawn(async move { + let Ok(connection) = incoming.await else { + return; + }; + // Datagrams, echoed the same way the streams are: their own task, so a + // service that is answering one is still accepting the other. + let datagrams = connection.clone(); + tokio::spawn(async move { + while let Ok(datagram) = datagrams.read_datagram().await { + if datagrams.send_datagram(datagram).is_err() { + return; + } + } + }); + while let Ok((mut send, mut recv)) = connection.accept_bi().await { + tokio::spawn(async move { + let mut buf = vec![0u8; 16 * 1024]; + while let Ok(Some(n)) = recv.read(&mut buf).await { + if send.write_all(&buf[..n]).await.is_err() { + return; + } + } + let _ = send.finish(); + }); + } + }); + } + }); + addr +} + +/// The relay, in front of that service. +async fn spawn_relay(upstream: SocketAddr, filters: &str, offers: &[&str]) -> SocketAddr { + let (cert, key) = self_signed(); + let chain = ChainHandle::new(FilterChain::new( + parse_filters(filters).unwrap(), + Duration::from_millis(500), + )); + let cfg = QuicConfig { + setup: Arc::new(QuicSetup::build(&cert, &key, alpn(offers)).unwrap()), + // No mark and no impersonation: `SO_MARK` and `IP_TRANSPARENT` both need + // CAP_NET_ADMIN, which a test run has no business requiring. What they change is + // which address the service sees, and nothing here is asking about that. + self_mark: None, + spoof_source: false, + max_connections: 0, + over_limit_forwards: false, + first_byte_timeout: None, + connect_timeout: Duration::from_secs(5), + upstream: Onward::Same, + // Nothing to write a reconstruction to, which is the ordinary case on any host + // without the capture interface and the case every test here runs in. + capture: None, + }; + let relay = QuicRelay::bind( + "127.0.0.1:0".parse().unwrap(), + upstream, + chain, + cfg, + Arc::new(ProxyStats::default()), + ) + .unwrap(); + let addr = relay.local_addr().unwrap(); + tokio::spawn(Arc::new(relay).serve()); + addr +} + +/// One message, and every message underneath it. +/// +/// quinn nests the reason: a read failing because the connection went away says only +/// "connection lost", and *why* it was lost — the application close, with the reason +/// firegex put in it — is one `source()` further down. A test that looked at the top +/// line alone could not tell a block from a crash. +fn because(error: &dyn std::error::Error) -> String { + let mut parts = vec![error.to_string()]; + let mut current = error.source(); + while let Some(inner) = current { + parts.push(inner.to_string()); + current = inner.source(); + } + parts.join(": ") +} + +struct Exchange { + /// What the client ended up speaking — which is what the *service* chose. + protocol: Option, + echoed: Result, String>, +} + +/// One client, one stream, one answer. +async fn roundtrip(addr: SocketAddr, payload: &[u8], speaks: &[&str]) -> Exchange { + let mut config = tls::quic_client_config().unwrap(); + config.alpn_protocols = alpn(speaks); + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new( + QuicClientConfig::try_from(config).unwrap(), + ))); + + let connecting = endpoint.connect(addr, "localhost").unwrap(); + let connection = match tokio::time::timeout(Duration::from_secs(5), connecting).await { + Ok(Ok(connection)) => connection, + Ok(Err(e)) => { + return Exchange { + protocol: None, + echoed: Err(e.to_string()), + } + } + Err(_) => { + return Exchange { + protocol: None, + echoed: Err("the handshake timed out".to_string()), + } + } + }; + let protocol = connection + .handshake_data() + .and_then(|d| d.downcast::().ok()) + .and_then(|d| d.protocol) + .map(|p| String::from_utf8_lossy(&p).into_owned()); + + let echoed = async { + let (mut send, mut recv) = connection.open_bi().await.map_err(|e| because(&e))?; + send.write_all(payload).await.map_err(|e| because(&e))?; + send.finish().map_err(|e| because(&e))?; + recv.read_to_end(64 * 1024).await.map_err(|e| because(&e)) + } + .await; + + Exchange { protocol, echoed } +} + +/// The whole point: a stream goes through, and it went through something that decrypted +/// it on the way. +/// +/// It also exercises the Retry this relay answers a new client with — every first +/// connection from an address arrives unvalidated, so a malformed token would show up +/// here as a handshake that never completes rather than as a subtle weakness. +#[tokio::test] +async fn carries_a_stream_through() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "", &["fgex-raw"]).await; + + let exchange = roundtrip(addr, b"hello over quic", &["fgex-raw"]).await; + assert_eq!(exchange.echoed.unwrap(), b"hello over quic"); +} + +/// The reason terminating it is worth the trouble: a rule matches bytes that on the wire +/// were inside an encrypted packet, with the frame headers encrypted too. +#[tokio::test] +async fn rules_see_the_plaintext_inside_the_streams() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:FLAG{", &["fgex-raw"]).await; + + let harmless = roundtrip(addr, b"harmless request", &["fgex-raw"]).await; + assert_eq!(harmless.echoed.unwrap(), b"harmless request"); + + let blocked = roundtrip(addr, b"give me FLAG{secret}", &["fgex-raw"]).await; + let error = blocked + .echoed + .expect_err("a rule let the payload through, or the connection was left open"); + // Closed on purpose, with a reason the client can read — not a stream reset and not + // a silence to time out on. + assert!( + error.contains("blocked by firegex"), + "the client was not told why: {error}" + ); +} + +/// A refusal ends the connection, not just the stream that carried it. +/// +/// The alternative — resetting one stream — would leave the client free to ask again on +/// the next one, which is not what a block means anywhere else in firegex. +#[tokio::test] +async fn a_refusal_takes_the_whole_connection() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:FLAG{", &["fgex-raw"]).await; + + let mut config = tls::quic_client_config().unwrap(); + config.alpn_protocols = alpn(&["fgex-raw"]); + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new( + QuicClientConfig::try_from(config).unwrap(), + ))); + let connection = endpoint.connect(addr, "localhost").unwrap().await.unwrap(); + + // One stream says something forbidden... + let (mut send, mut recv) = connection.open_bi().await.unwrap(); + send.write_all(b"here is a FLAG{x}").await.unwrap(); + let _ = send.finish(); + let _ = recv.read_to_end(4096).await; + + // ...and a second one, on the same connection, has nowhere to go. + let opened = tokio::time::timeout(Duration::from_secs(2), connection.open_bi()).await; + assert!( + matches!(opened, Ok(Err(_))), + "the connection outlived the block" + ); +} + +/// The service picks the protocol, and the client is told that and nothing else. +/// +/// This is the ordering that had to change from the TLS path: there the ClientHello is +/// held open and the service is asked what the *client* offered, which QUIC gives no +/// opportunity to do. So the relay offers its candidates to the service and passes the +/// answer on — and a service speaking something unusual is carried without firegex +/// having to guess. +#[tokio::test] +async fn the_service_chooses_the_protocol() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-test"]).await; + let addr = spawn_relay(upstream, "", &["h3", "fgex-test"]).await; + + let exchange = roundtrip(addr, b"ping", &["h3", "fgex-test"]).await; + assert_eq!(exchange.protocol.as_deref(), Some("fgex-test")); + assert_eq!(exchange.echoed.unwrap(), b"ping"); +} + +/// A client that cannot speak what the service chose is refused by the handshake, which +/// is the honest failure this ordering costs. +#[tokio::test] +async fn a_client_that_speaks_something_else_is_refused() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-test"]).await; + let addr = spawn_relay(upstream, "", &["fgex-test"]).await; + + let exchange = roundtrip(addr, b"ping", &["h3"]).await; + assert!( + exchange.echoed.is_err(), + "a client was told a protocol the service never agreed to" + ); +} + +/// One client connection, for the tests that need more of it than `roundtrip` gives. +async fn connect(addr: SocketAddr, speaks: &[&str]) -> quinn::Connection { + let mut config = tls::quic_client_config().unwrap(); + config.alpn_protocols = alpn(speaks); + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new( + QuicClientConfig::try_from(config).unwrap(), + ))); + endpoint.connect(addr, "localhost").unwrap().await.unwrap() +} + +/// A datagram goes through, and comes back. +#[tokio::test] +async fn datagrams_are_carried_both_ways() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "", &["fgex-raw"]).await; + + let connection = connect(addr, &["fgex-raw"]).await; + assert!( + connection.max_datagram_size().is_some(), + "the extension was not advertised to a client that could have used it" + ); + connection + .send_datagram(bytes::Bytes::from_static(b"ping")) + .unwrap(); + let back = tokio::time::timeout(Duration::from_secs(5), connection.read_datagram()) + .await + .expect("the datagram never came back") + .unwrap(); + assert_eq!(&back[..], b"ping"); +} + +/// A rule refusing a datagram drops that datagram, and nothing else. +/// +/// The opposite of what a refusal does to a stream, on purpose: there is no conversation +/// to end, so ending the connection would take a hundred other streams with it over one +/// message that was already complete when it was judged. +#[tokio::test] +async fn a_refused_datagram_is_dropped_and_the_connection_lives() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:forbidden", &["fgex-raw"]).await; + + let connection = connect(addr, &["fgex-raw"]).await; + connection + .send_datagram(bytes::Bytes::from_static(b"forbidden")) + .unwrap(); + // Nothing comes back, because nothing was forwarded. + assert!( + tokio::time::timeout(Duration::from_millis(500), connection.read_datagram()) + .await + .is_err(), + "a refused datagram reached the service anyway" + ); + + // And the connection is still there to be used. + connection + .send_datagram(bytes::Bytes::from_static(b"allowed")) + .unwrap(); + let back = tokio::time::timeout(Duration::from_secs(5), connection.read_datagram()) + .await + .expect("the connection was closed by a datagram that should only have been dropped") + .unwrap(); + assert_eq!(&back[..], b"allowed"); +} + +/// The datagrams of one connection are one flow, and a filter keeps its state across +/// them. +/// +/// The same answer the plain UDP relay gives — its sessions live on the flow, not on the +/// datagram — and the reason is that the alternative is a way through: a pattern split +/// over two datagrams would otherwise be two halves that match nothing, sent a +/// millisecond apart. What a datagram does *not* get is the stream models built on top of +/// a session, which is `L4::QuicDatagram`'s job and not this one's. +#[tokio::test] +async fn a_datagram_flow_keeps_its_filter_state() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:secret", &["fgex-raw"]).await; + + let connection = connect(addr, &["fgex-raw"]).await; + connection + .send_datagram(bytes::Bytes::from_static(b"sec")) + .unwrap(); + let back = tokio::time::timeout(Duration::from_secs(5), connection.read_datagram()) + .await + .expect("half a needle is not a needle, and this one was blocked") + .unwrap(); + assert_eq!(&back[..], b"sec"); + + // The half that completes it is refused, though it arrived in a datagram of its own. + connection + .send_datagram(bytes::Bytes::from_static(b"ret")) + .unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(500), connection.read_datagram()) + .await + .is_err(), + "a needle split across two datagrams went through" + ); +} diff --git a/backend/proxysrc/tests/tls.rs b/backend/proxysrc/tests/tls.rs index 3915204..6c51f6a 100644 --- a/backend/proxysrc/tests/tls.rs +++ b/backend/proxysrc/tests/tls.rs @@ -124,6 +124,7 @@ async fn terminates_tls_and_relays_to_a_plain_service() { TlsSetup { server: Some(tls::server_config(&cert, &key).unwrap()), upstream: None, + optional: false, }, ) .await; @@ -145,6 +146,7 @@ async fn decrypts_inspects_and_re_encrypts() { TlsSetup { server: Some(tls::server_config(&cert, &key).unwrap()), upstream: Some(tls::client_config().unwrap()), + optional: false, }, ) .await; @@ -165,6 +167,7 @@ async fn rules_see_the_plaintext() { TlsSetup { server: Some(tls::server_config(&cert, &key).unwrap()), upstream: Some(tls::client_config().unwrap()), + optional: false, }, ) .await; diff --git a/backend/routers/services.py b/backend/routers/services.py index e66520d..0af750f 100644 --- a/backend/routers/services.py +++ b/backend/routers/services.py @@ -34,7 +34,7 @@ from modules.services.firewall import ( from modules.services import stats from modules.services.logs import log_for from modules.services.models import (KIND, L4, MODE, PROTO, STATUS, Service, - TRANSPORT) + TRANSPORT, UPSTREAM, upstream_refusal) from modules.services.nftables import FiregexTables from modules.services import transports from modules.services.transports import PROXY_ENGINE, PYWORKER, UnsupportedChain @@ -59,8 +59,9 @@ db = SQLite( "name": "VARCHAR(100) NOT NULL UNIQUE", "status": 'VARCHAR(100) NOT NULL CHECK (status IN ("active", "stop"))', # What the service speaks, which is not always what the kernel matches on: - # `tls` is TCP on the wire. The address carries the second answer. - "proto": 'VARCHAR(4) NOT NULL CHECK (proto IN ("tcp", "udp", "tls"))', + # `tls` is TCP on the wire and `quic` is UDP. The address carries the second + # answer. + "proto": 'VARCHAR(4) NOT NULL CHECK (proto IN ("tcp", "udp", "tls", "quic", "http"))', "transport": 'VARCHAR(20) NOT NULL CHECK (transport IN ("nfqueue", "proxy", "external"))', "fail_open": "BOOLEAN NOT NULL CHECK (fail_open IN (0, 1)) DEFAULT 1", # How many connections (or UDP flows) this service may carry at once, and @@ -102,6 +103,31 @@ db = SQLite( "service_id": "VARCHAR(100) NOT NULL", "ip_int": "VARCHAR(100) NOT NULL", "port": "INT NOT NULL CHECK(port > 0 and port < 65536)", + # What clients speak at this address, in the same words a service speaks + # them: `tls` is TCP with TLS on it, `quic` is UDP with QUIC. For every + # protocol but `http` it is the service's own and nobody is asked; `http` is + # the one that means "each address says", which is what it has always meant. + "edge": 'VARCHAR(4) NOT NULL CHECK (edge IN ("tcp", "udp", "tls", "quic")) ' + "DEFAULT 'tcp'", + # Where this address's traffic actually goes, when that is not the port it + # arrived on. NULL is transparent — the address is the service — which is + # what firegex did and only did until an address could be a *publication*: + # a service answering in the clear on :80 reached over TLS on :443 without + # moving. Only the port: the same host, because what is being published is a + # service this instance is already in front of. + "target_port": "INT CHECK(target_port IS NULL OR " + "(target_port > 0 AND target_port < 65536))", + # What the service behind *this address* speaks, which used to be assumed to + # be whatever the client spoke. `same` is that assumption and the default; + # `tcp` and `tls` say the service speaks HTTP/1.1 — in the clear or under its + # own TLS — whatever arrived at the front, which is what puts an ordinary web + # service behind a TLS or HTTP/3 edge with firegex terminating on its behalf. + # + # On the address rather than on the service because that is where it has an + # answer: one daemon reached over TLS on one port and in the clear on another + # is re-encrypted for the first and handed the plaintext for the second. + "upstream": 'VARCHAR(8) NOT NULL CHECK (upstream IN ("same", "tcp", "tls")) ' + "DEFAULT 'same'", # The transport the *kernel* sees, derived from the service's own: `tls` is # TCP on the wire. Kept here so `(ip, port, proto)` is a usable uniqueness # key — a TCP service and a UDP one may share an address exactly as the @@ -203,6 +229,13 @@ class AddressModel(BaseModel): ip_int: str port: PortType proto: str + #: What clients speak here, which is what the operator chose. `proto` is what the + #: kernel matches, and is this with the hat taken off. + edge: str = L4.TCP + #: Where the service is, when this address is a publication rather than the service. + target_port: int | None = None + #: What the service behind this address speaks: `same`, `tcp` or `tls`. + upstream: str = UPSTREAM.SAME proxy_ip: str | None = None proxy_port: int | None = None @@ -210,6 +243,18 @@ class AddressModel(BaseModel): class AddressForm(BaseModel): ip_int: str port: PortType + #: What clients speak here — `tcp`, `tls`, `udp` or `quic`. Only ever set on an + #: `http` service, which is the one whose addresses are not all the same; elsewhere + #: the service's own protocol decides and a value here is refused rather than + #: allowed to disagree with it. + edge: str | None = None + #: Where the service actually is, when it is not on this port. Absent is the + #: transparent case: the address *is* the service. + target_port: int | None = None + #: What the service behind this address speaks. Absent is `same` — whatever arrived + #: at the front goes back out the same way, which is what every address did before + #: there was a choice. + upstream: str | None = None #: `external` only: where your own proxy listens for this address. Two addresses #: cannot share one — the return rule tells them apart by it. proxy_ip: str | None = None @@ -725,31 +770,139 @@ async def get_service(service_id: str): return _with_addresses(res[0]) -def _insert_address(service_id: str, proto: str, form: AddressForm) -> str: +def _insert_address(service_id: str, proto: str, transport: str, form: AddressForm) -> str: """Add one address, carrying the transport the kernel will match on. Not the service's own protocol: `tls` is TCP on the wire, and an address storing it verbatim would let a TCP service and a TLS one claim one `ip:port` between them. + + On an `http` service the address says what is spoken at it, because that service is + reached in more than one way at once — `:80` in the clear, `:443` under TLS, a UDP + port over HTTP/3. What it says is kept beside the transport the kernel matches: the + two are the same answer with the hat on and off. """ try: parsed_ip = parse_ip_or_int(form.ip_int) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + edge = _address_edge(proto, form) + target_port = _address_target(transport, form) + upstream = _address_upstream(proto, form) address_id = gen_id() db.query( "INSERT INTO service_addresses (address_id, service_id, ip_int, port, proto, " - "proxy_ip, proxy_port) VALUES (?, ?, ?, ?, ?, ?, ?);", + "edge, target_port, upstream, proxy_ip, proxy_port) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", address_id, service_id, parsed_ip, form.port, - L4.l4_of(proto), + L4.l4_of(edge), + edge, + target_port, + upstream, form.proxy_ip or None, form.proxy_port, ) return address_id +def _address_edge(proto: str, form: AddressForm) -> str: + """What clients speak at one address of a service. + + Every protocol but `http` has one answer for the whole service, so an address of one + is not asked and a value sent anyway is refused rather than quietly kept: a row whose + edge disagrees with its service is a row that would be intercepted on one transport + and served on another. + + `http` is the protocol that means "each address says", which is what it meant when + the choice was TCP or UDP and what it still means now that the same address can say + whether it is encrypted. The default is `tcp`, which is what an HTTP address is + unless somebody says otherwise. + """ + asked = str(form.edge) if form.edge else None + if str(proto) != L4.HTTP: + if asked and asked != str(proto): + raise HTTPException( + status_code=400, + detail=f"A {str(proto).upper()} service speaks {str(proto).upper()} at " + f"every address, so this one cannot be {asked!r}. Only an HTTPS " + f"service is reached in more than one way at once.", + ) + return str(proto) + if asked is None: + return L4.TCP + if asked not in (L4.TCP, L4.TLS, L4.QUIC): + raise HTTPException( + status_code=400, + detail=f"An address of an HTTPS service is reached in the clear, under TLS " + f"or over QUIC — not {asked!r}.", + ) + return asked + + +def _address_target(transport: str, form: AddressForm) -> int | None: + """Where the service reached through one address is, when it is not on that port. + + Every protocol can say it, because **publishing is being reachable one more way than + the service is** and nothing about that is HTTP's. One daemon on `:8080` is reached + at `:80` in the clear, at `:443` under TLS and at a UDP port over HTTP/3 — and also, + with nothing HTTP about it, at whatever port the round's scoreboard was told about. + It was refused outside `http` while `http` was the only protocol whose addresses were + not all the same, which confused the shape of one protocol with the shape of the + feature. + + What it does need is something that **dials**, and that is the layer rather than the + protocol. The proxy terminates the connection and opens the one to the service, so it + is free to open it somewhere else. NFQUEUE hands the kernel a verdict on packets + already on their way and opens nothing; the hand-off layer rewrites the destination to + your own proxy, which then decides for itself where the service is. On both, a stored + port would be read by nothing at all — refused rather than ignored, because an + operator told their service is published and finding it merely intercepted is exactly + the trap this file keeps arguing against. + """ + if not form.target_port: + return None + if str(transport) != TRANSPORT.PROXY: + raise HTTPException( + status_code=400, + detail=f"Only the proxy layer can send an address's traffic to a port other " + f"than the one it arrived on: it terminates the connection and opens " + f"the one to the service, so it is free to open it elsewhere. " + + ("NFQUEUE passes judgement on packets already on their way and opens " + "nothing." + if str(transport) == TRANSPORT.NFQUEUE else + "This layer hands the traffic to your own proxy, which decides for " + "itself where the service is."), + ) + if not 0 < form.target_port < 65536: + raise HTTPException(status_code=400, detail="Invalid port for the service") + return form.target_port + + +def _address_upstream(proto: str, form: AddressForm) -> str: + """What the service behind one address speaks. + + Only a service that is decrypted has an upstream leg of its own to decide: on + anything else firegex terminates nothing, so there is nothing to put back or leave + off, and a stored answer would be a setting that does nothing. Refused rather than + ignored for that reason. + """ + asked = str(form.upstream) if form.upstream else UPSTREAM.SAME + if asked == UPSTREAM.SAME: + return UPSTREAM.SAME + if asked not in UPSTREAM.ALL: + raise HTTPException(status_code=400, detail=f"Unknown upstream {asked!r}") + if str(proto) not in (L4.TLS, L4.QUIC, L4.HTTP): + raise HTTPException( + status_code=400, + detail=f"A {str(proto).upper()} service is carried as it arrives: firegex " + f"terminates nothing there, so there is no upstream leg to decide. " + f"Only a service firegex decrypts has one.", + ) + return asked + + def _address_taken(e: sqlite3.IntegrityError) -> str: """Say which uniqueness rule was broken, because they mean different things.""" if "unique_hijack_target" in str(e): @@ -812,13 +965,33 @@ async def add_service(form: ServiceAddForm): raise HTTPException( status_code=400, detail="A service needs at least one address to protect" ) - if form.proto == L4.TLS: + if form.proto in (L4.TLS, L4.QUIC, L4.HTTP): if not (form.tls_cert and form.tls_key): raise HTTPException( status_code=400, - detail="A service that speaks TLS needs both a certificate and a private key", + detail=f"A service that speaks {form.proto.upper()} needs both a " + f"certificate and a private key" + + (". An HTTP service covers the TLS and HTTP/3 edges as well as " + "the cleartext one, and neither exists without a certificate — " + "a service that only answers in the clear is a `tcp` one, where " + "HTTP/1.1 and HTTP/2 are already filtered." + if form.proto == L4.HTTP else ""), ) _check_tls_material(form.tls_cert, form.tls_key) + # Each address answers for itself what the service behind it speaks, and is refused + # here rather than at the first start if it cannot. Everything else an address says + # is checked in the same breath, and for a second reason: the rows are written after + # the service's own, so a refusal arriving then would leave a service behind that + # nobody asked for. + for address in form.addresses: + _address_target(form.transport, address) + refusal = upstream_refusal( + form.proto, + _address_upstream(form.proto, address), + L4.l4_of(_address_edge(form.proto, address)) == L4.UDP, + ) + if refusal: + raise HTTPException(status_code=400, detail=refusal) # The transport gets a look before the row exists. A chain it could not host is # caught later, when there is one; what is caught here is what the *service* makes # impossible on its own — a protocol this layer cannot carry — and it is worth @@ -852,7 +1025,8 @@ async def add_service(form: ServiceAddForm): db.query( "INSERT INTO services (service_id, name, status, proto, transport, " "fail_open, max_connections, over_limit_forwards, first_byte_timeout, " - "tls_cert, tls_key) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", + "tls_cert, tls_key) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", service_id, refactor_name(form.name), STATUS.STOP, @@ -865,11 +1039,17 @@ async def add_service(form: ServiceAddForm): form.tls_cert, form.tls_key, ) - except sqlite3.IntegrityError: + except sqlite3.IntegrityError as e: + # The name is the only thing an operator can collide with by accident, so it is + # the answer worth giving — but it is not the only constraint on this row, and + # answering "the name exists" to a CHECK that failed sends whoever reads it + # looking for a service that is not there. Ask the exception which it was. + if "name" not in str(e): + return {"status": f"The service was refused by the database: {e}"} return {"status": "A service with this name already exists"} try: for address in form.addresses: - _insert_address(service_id, form.proto, address) + _insert_address(service_id, form.proto, form.transport, address) except sqlite3.IntegrityError as e: # All of it or none: a service that came up on half the addresses the operator # listed is one they would believe is protecting the other half. @@ -921,8 +1101,8 @@ async def edit_service(service_id: str, form: ServiceSettingsForm): # operator has by then attributed to something else. So the stored material counts # towards it, and a service already holding a certificate can be switched on without # pasting it again. - lands_on_tls = (form.proto if form.proto is not None else row["proto"]) == L4.TLS - if lands_on_tls: + lands_on = form.proto if form.proto is not None else row["proto"] + if lands_on in (L4.TLS, L4.QUIC, L4.HTTP): cert = form.tls_cert if form.tls_cert is not None else row["tls_cert"] key = form.tls_key if form.tls_key is not None else row["tls_key"] if not (cert and cert.strip()) or not (key and key.strip()): @@ -930,9 +1110,10 @@ async def edit_service(service_id: str, form: ServiceSettingsForm): else "a certificate" if not (cert and cert.strip()) else "a private key" raise HTTPException( status_code=400, - detail=f"This service terminates TLS, so it needs {missing}. Give it one " - f"here — the engine would refuse to start without it, and the service " - f"would fail on its next start rather than now.", + detail=f"This service terminates {str(lands_on).upper()}, so it needs " + f"{missing}. Give it one here — the engine would refuse to start " + f"without it, and the service would fail on its next start rather " + f"than now.", ) if form.tls_cert is not None: fields["tls_cert"] = form.tls_cert @@ -948,14 +1129,24 @@ async def edit_service(service_id: str, form: ServiceSettingsForm): ) except sqlite3.IntegrityError: raise HTTPException(status_code=400, detail="That name is already taken") - if "proto" in fields: + if "proto" in fields and fields["proto"] != L4.HTTP: # The addresses carry the transport the kernel matches on, so that # `(ip, port, proto)` can be unique. Kept in step here rather than being a second # thing the operator sets — and derived, because `tls` is TCP on the wire. + # + # Not for `http`, which is the one protocol whose addresses already hold the + # answer: rewriting them all to `tcp` would quietly turn its HTTP/3 address into a + # TCP one and leave the QUIC relay pointing at nothing. Switching *away* from + # `http` does flatten them, which is honest — the service now speaks one + # transport — and can legitimately collide, which is the 400 below. try: db.query( - "UPDATE service_addresses SET proto = ? WHERE service_id = ?;", + # Both, because they are one answer: a service that now speaks one thing + # everywhere has addresses that speak it too, and an `edge` left behind + # would be a row claiming to be reached a way the service no longer is. + "UPDATE service_addresses SET proto = ?, edge = ? WHERE service_id = ?;", L4.l4_of(fields["proto"]), + fields["proto"], service_id, ) except sqlite3.IntegrityError: @@ -1006,8 +1197,18 @@ async def add_address(service_id: str, form: AddressForm): detail="The external transport hands traffic to your own proxy and rewrites the " "source address on return, which requires a concrete IP address rather than an interface.", ) + # Before the row exists, because this is the address's own answer and it can be one + # this instance cannot honour. `ProxyTransport.check` says the same thing at start, + # and would leave the address behind to be undone. + refusal = upstream_refusal( + row["proto"], + _address_upstream(row["proto"], form), + L4.l4_of(_address_edge(row["proto"], form)) == L4.UDP, + ) + if refusal: + raise HTTPException(status_code=400, detail=refusal) try: - address_id = _insert_address(service_id, row["proto"], form) + address_id = _insert_address(service_id, row["proto"], row["transport"], form) except sqlite3.IntegrityError as e: raise HTTPException(status_code=400, detail=_address_taken(e)) def undo(): @@ -1031,7 +1232,18 @@ async def add_address(service_id: str, form: AddressForm): @app.put("/{service_id}/addresses/{address_id}", response_model=StatusMessageModel) async def edit_address(service_id: str, address_id: str, form: AddressForm): - """Move an address. The rules for it are taken back and reinstalled.""" + """Change one address: where it is, and everything it says. + + The rules for it are taken back and reinstalled, which is what makes the whole row + rewritable rather than only its address. It used to touch `ip_int`, `port` and the + proxy endpoint alone, so the edge, the port the service is on and what that service + speaks were add-only — and the operator who set one of them while creating the + service had no way back to it except deleting the address and adding it again. That + is a worse answer than reinstalling rules the edit already reinstalls. + + Only this address stops being steered while that happens; the rest of the service + keeps its connections. A refusal puts the row back exactly as it was. + """ row = _service_or_404(service_id) was = _address_or_404(service_id, address_id) if row["transport"] == TRANSPORT.EXTERNAL and not form.proxy_port: @@ -1047,11 +1259,34 @@ async def edit_address(service_id: str, address_id: str, form: AddressForm): "source address on return, which requires a concrete IP address rather than an interface.", ) - def write(ip_int, port, proxy_ip, proxy_port): + # A field the caller did not mention keeps what the row already had. This is the + # endpoint that *moves* an address as well as the one that changes what it says, and + # a move sending only an address and a port would otherwise quietly unpublish it. + # Taking one off is a value, not an absence: `0` for the port, `same` for the + # upstream — which is what the interface sends when it is emptied. + if form.edge is None: + form.edge = was["edge"] + if form.target_port is None: + form.target_port = was["target_port"] + if form.upstream is None: + form.upstream = was["upstream"] + + # Everything the row says is worked out *before* the old rules come back, so a + # refusal leaves an address that is still steered exactly as it was. + edge = _address_edge(row["proto"], form) + target_port = _address_target(row["transport"], form) + upstream = _address_upstream(row["proto"], form) + refusal = upstream_refusal(row["proto"], upstream, L4.l4_of(edge) == L4.UDP) + if refusal: + raise HTTPException(status_code=400, detail=refusal) + + def write(ip_int, port, proto, edge, target_port, upstream, proxy_ip, proxy_port): db.query( - "UPDATE service_addresses SET ip_int = ?, port = ?, proxy_ip = ?, " - "proxy_port = ? WHERE address_id = ?;", - ip_int, port, proxy_ip, proxy_port, address_id, + "UPDATE service_addresses SET ip_int = ?, port = ?, proto = ?, edge = ?, " + "target_port = ?, upstream = ?, proxy_ip = ?, proxy_port = ? " + "WHERE address_id = ?;", + ip_int, port, proto, edge, target_port, upstream, proxy_ip, proxy_port, + address_id, ) async def restore(): @@ -1061,7 +1296,8 @@ async def edit_address(service_id: str, address_id: str, form: AddressForm): alternative to trying is an address left unprotected because the *new* one was impossible. A failure here is reported by the exception that caused it. """ - write(was["ip_int"], was["port"], was["proxy_ip"], was["proxy_port"]) + write(was["ip_int"], was["port"], was["proto"], was["edge"], + was["target_port"], was["upstream"], was["proxy_ip"], was["proxy_port"]) try: await firewall.get(service_id).address_added(address_id) except Exception: @@ -1075,7 +1311,8 @@ async def edit_address(service_id: str, address_id: str, form: AddressForm): manager = firewall.get(service_id) await manager.address_removed(address_id) try: - write(parsed_ip, form.port, form.proxy_ip or None, form.proxy_port) + write(parsed_ip, form.port, L4.l4_of(edge), edge, target_port, upstream, + form.proxy_ip or None, form.proxy_port) except sqlite3.IntegrityError as e: await restore() raise HTTPException(status_code=400, detail=_address_taken(e)) diff --git a/docs/pyfilter.md b/docs/pyfilter.md index c5a07d7..72cc09c 100644 --- a/docs/pyfilter.md +++ b/docs/pyfilter.md @@ -124,16 +124,41 @@ That is deliberate. The two network layers cannot honestly offer the same thing #### Which models work on UDP -`RawPacket` is the only model that does not need a connection underneath it. Everything -else — an assembled `TCPInputStream`, a parsed `HttpRequest` — reaches for something only -TCP has, and on a datagram the library declines to build it, which means **the filter is -never called**. +`RawPacket` is the only model that does not need a stream underneath it. Everything +else — an assembled `TCPInputStream`, a parsed `HttpRequest` — reaches for something a +datagram does not have, and on one the library declines to build it, which means **the +filter is never called**. So a Python filter runs on a UDP service, on either network layer, as long as it asks for a `RawPacket`. Saving one that asks for anything else against a UDP service is refused, naming the model: a filter sitting in the chain doing nothing, with nothing saying so, is the failure this whole module is arranged to prevent. +**HTTP/2 is not a gap any more, and it used to be the one that mattered.** A filter asking +for an `HttpRequest` works on HTTP/1.x, on HTTP/2 and on HTTP/3 alike. It did not work on +a connection that negotiated `h2`: those headers are HPACK-compressed, so the parser was +handed a compression format and the filter was never called — no block, no log, no +counter. The engine terminates HTTP/2 now and shows the chain the HTTP/1.1 each exchange +would have been, from the same code that does it for HTTP/3, so the same file covers every +version. Since gRPC is HTTP/2, that is also what makes gRPC filterable. + +**One thing about HTTP/2 to know if your filter keeps state.** Each HTTP/2 stream is its +own connection to a filter, with its own module globals — where an HTTP/1.1 keep-alive +connection carries many requests through one set of them. HTTP/2 interleaves its streams, +so sharing state between them would let one client's bytes decide another client's +verdict, and would let an attacker split a pattern across two streams to get past a +filter. One request, one state, is the honest mapping; it is the same rule an HTTP/3 +request stream follows. + +**QUIC is not that case, even though it is UDP on the wire.** A QUIC stream is ordered +and reliable, the engine terminates the connection to see inside it, and each stream +reaches the chain as its own connection — so every model works there exactly as it does +on TCP. On an HTTP/3 service the exchange is rendered as the HTTP/1.1 it would have been +before the chain sees it, so `HttpRequest` and the rest mean what they always meant; the +[services documentation](services.md#http3-is-shown-to-the-filters-as-http11) says what +that rendering does and does not promise. The question a model asks is therefore "is +there a stream", not "is this TCP", and `RawPacket` answers both separately. + Rewriting a datagram is **exact**, on both layers: there are no sequence numbers for a different length to desynchronise. @@ -152,7 +177,15 @@ One chunk of the connection, and what is known about it. The **only** data struc - `data_size: int` — how many bytes of payload this chunk carries. - `is_input: bool` — `True` for client → service, `False` for service → client. `is_output` is its inverse. - `is_ipv6: bool` — `True` for IPv6, `False` for IPv4. -- `is_tcp: bool` — `True` for TCP, `False` for UDP. +- `is_tcp: bool` — `True` for TCP. `False` for a datagram, and `False` for QUIC. +- `l4: str` — what carries this connection: `"tcp"`, `"udp"`, `"quic"` for a stream + inside a QUIC connection, or `"quic-datagram"` for a DATAGRAM frame in one. +- `is_stream: bool` — whether these bytes arrive in order, once each, as part of a + stream: true for TCP and for a QUIC stream, false for either kind of datagram. This is + what every model above `RawPacket` actually needs, and the two came apart when QUIC + arrived — it is a stream *and* it is UDP, so one flag answering both questions gets one + of them wrong. A filter asking for a stream model is simply not called for a datagram, + so one file can carry both. - `src_ip: str`, `dst_ip: str`, `src_port: int`, `dst_port: int` — where this chunk came from and where it is going (read-only). - `client_ip`, `client_port`, `server_ip`, `server_port` — the same two endpoints named by role instead of by direction, so a filter does not have to branch on `is_input` to find out who the client is (read-only). - `was_mangled: bool` — whether an earlier filter in the chain already rewrote this chunk. @@ -285,6 +318,58 @@ def check_previous_requests(resp: HttpResponse): The number of entries kept per stream is capped by the `FGEX_MAX_HISTORY_SIZE` global (default `100`) — see [Other global options](#other-global-options) below; once the cap is reached, the oldest entry is dropped as a new one is added. +### gRPC messages + +**HTTP only.** `GrpcMessage`, `GrpcRequest` and `GrpcResponse` hand you **one gRPC message +at a time**, with its length prefix taken off. + +gRPC is HTTP/2, and firegex renders every version of HTTP to the filters as HTTP/1.1 — so +everything *around* a call already arrives through the models above: the method is the +path on an `HttpRequest`, the metadata are its headers, the status is in the trailer +section. What did not arrive was the body. A gRPC body is a sequence of messages, each one +a flag byte, a four-byte length and then the protobuf; a filter reading `request.body` read +that framing glued to the payload, and a pattern could match across the boundary between +two messages — a false positive with nothing to point at. + +```python +from firegex.pyfilters import pyfilter, ACCEPT, REJECT +from firegex.pyfilters.models import GrpcMessage + +@pyfilter +def refuse_a_payload(message: GrpcMessage): + if b"../" in message.payload: + return REJECT + return ACCEPT +``` + +| Member | What it is | +|---|---| +| `payload` | This message's bytes, without the length prefix | +| `compressed` | Whether this message's own flag byte says it is compressed | +| `is_request` | Whether it went from the client towards the service | +| `method` | The gRPC method, which is the HTTP path: `/package.Service/Method` | +| `grpc_status` | The status, where the message carrying it stated one | + +`GrpcRequest` and `GrpcResponse` are the same thing narrowed to one direction, so you do +not have to check `is_request` yourself. + +**It is called per message, not per body**, and that is the part that matters: a +server-streaming or bidirectional call has a body that does not finish until the stream +closes, so a model that waited for a finished body would call your filter once, at the +end — a log entry rather than a block. Each message is handed over as it completes, and a +message split across two packets is held until it is whole: deciding on half a payload is +how a pattern gets defeated by splitting it. + +Two things it deliberately does **not** do. It does not decompress a message whose flag +byte says it is compressed — what it is compressed with is the two peers' agreement +(`grpc-encoding`), and guessing is how a filter comes to read something that is not there, +so you get `compressed` and the raw bytes. And it does not decode the protobuf: without +your `.proto` there is no schema, so `payload` is bytes and a pattern against them is what +a ruleset has. + +A filter asking for one of these is simply **not called** on traffic that is not gRPC, the +way every other model declines when what it needs is not there. + ## Stream limiter What happens if a single TCP stream carries a lot of data? Past a configurable size, a "full stream" action kicks in. First import the enum: diff --git a/docs/services.md b/docs/services.md index 4d311fd..3d294dc 100644 --- a/docs/services.md +++ b/docs/services.md @@ -26,6 +26,70 @@ without interrupting existing traffic. The alternative — one service per address, with the chains copied between them — is how one of them silently stops being protected the first time a pattern is added to the other. +### What one address can say + +An address is normally **where the service is**: firegex intercepts what the client +dialled and dials it back as the client, so the service sees the connection it would have +seen. That is what transparent means here, and it is what every address does unless you +say otherwise. + +Everything else lives behind the **⚙ button on the address**, and only what applies to +the service being configured is offered: + +| | When it is offered | What it does | +|---|---|---| +| **What clients speak here** — clear · TLS · HTTP/3 | HTTPS services, on the row itself | Not an extra: it is what the address *is* | +| **The service is on another port** | the Proxy layer | Clients arrive at this address; firegex dials the service on that port instead | +| **Send to the service** — what arrived · plaintext · TLS | any service firegex decrypts | What firegex speaks to the service, which need not be what the client spoke. *Plaintext* is what puts an ordinary cleartext site behind HTTPS, HTTP/2 and HTTP/3 with only firegex holding a certificate | +| **Hand it to your proxy at** | the hand-off layer | Where your own proxy is listening for this address | + +What is set is repeated back under the address as a **tag**, and clicking a tag opens the +panel it came from. Folded is not hidden: an option you cannot see you set is one that +will surprise you at the worst moment. The same tags are shown beside the address on the +service's page, in the same words, and open the same settings. + +**An address is a way in, and one more way in is one more address.** The option above is +not "also expose it on :443" — it says the opposite thing: *clients arrive here, and the +service is over there*. So a cleartext service on `:80` reached over TLS on `:443` is two +addresses, `:443` carrying the option that points at `:80`, and not one address on `:80` +with `443` typed into it. Written that way round it is intercepted where it already +listens and published nowhere, which is the one mistake this option makes easy. + +*The service is on another port* is the Proxy layer's rather than any protocol's: that layer +terminates the connection and opens the one to the service, so it is free to open it +somewhere else. NFQUEUE passes judgement on packets already on their way and opens +nothing; the hand-off layer gives the traffic to the proxy you run yourself, which decides +for itself where the service is. On both it is refused rather than stored, because a port +nothing reads would tell you your service is published while it is merely intercepted. + +So a service answering HTTP/1.1 in the clear on `:80`, and nothing else, is three rows: + +| Address | Clients speak | Sent on to | Sent as | +|---|---|---|---| +| `10.0.0.1:80` | clear | — | what arrived | +| `10.0.0.1:443` | TLS | `:80` | plaintext | +| `10.0.0.1:4433` | HTTP/3 | `:80` | plaintext | + +Firegex terminates what each client speaks, shows the filters the same HTTP/1.1 whichever +it was, runs **one chain** over all three, and hands the service the HTTP/1.1 it has always +understood. The service is not moved, reconfigured, or told about any of it — and only +firegex holds a certificate. + +Three things to know: + +- **On a published address firegex is not transparent**, which is the point of it. The + service sees a connection to `:80` — from the client's own address, as always — rather + than to the port the client dialled. +- **Declaring TLS is a promise**, and the engine keeps it: a client that opens that port + in the clear is refused rather than carried to a service expecting HTTPS. *In the clear* + is the permissive default and terminates a client that brings TLS anyway, decided per + connection, because refusing one at a port nobody promised would be a rule with nothing + behind it. +- **All of it can be changed afterwards**, from the pencil beside the address on the + service's page. Editing an address takes its rules back and reinstalls them, which is + what makes the whole row rewritable and not only the address; only that one address + stops being steered while it happens, and a refusal puts the row back untouched. + ### An address, or the interface it arrives on An address entry is a concrete IP (`192.168.1.10:80`, `::1:80`, CIDR `10.0.0.0/24:8080`) or @@ -43,15 +107,41 @@ edited when the address changes, and nothing silently stops being protected when The rules are built differently for each, so the layer matters: - **On NFQUEUE**: interface matching operates natively in nftables using `meta iifname ` (in prerouting) and `meta oifname ` (in postrouting). Packets arriving at or routed through that interface on the configured port are queued to userspace, regardless of the destination IP. -- **On Proxy (TCP & TLS)**: traffic entering through the interface (`meta iifname `) is redirected to the proxy listener. The engine connects upstream preserving the destination (`SO_ORIGINAL_DST`) and spoofing the client source IP (`IP_TRANSPARENT`), while policy routing diverts return packets (`meta oifname `). Local host output redirection is skipped for interfaces to prevent hijacking unrelated outbound host connections. +- **On Proxy (TCP & TLS)**: traffic entering through the interface (`meta iifname `) is redirected to the proxy listener. The engine connects upstream preserving the destination (`SO_ORIGINAL_DST`) and spoofing the client source IP (`IP_TRANSPARENT`), while policy routing diverts return packets (`meta oifname `). **Protecting a link protects everything that crosses it on that port** — including traffic that is only passing through. On a bridge like `docker0` that means the containers' own outgoing connections: a service protecting `docker0:443` takes every container's outbound HTTPS into firegex, which terminates it with that service's certificate, and the containers see a certificate that does not match the site they asked for. That is the interface doing what it says; it is not narrowed for you, because a link is also how you protect a service on **another machine** that this host routes for, which is the case the address is not yours to know. If you want only what is addressed to this host, name the address instead of the link. An interface stands for whatever addresses it carries, and firegex resolves them wherever it needs an address rather than a match — so an HTTPS edge, a published port and a client on this host all work the same whether you write `eth0` or the address itself. Traffic this host generates towards those addresses is redirected too, matched by address rather than by interface name — a connection from the host to one of its own addresses is routed through loopback, so there is no interface on it to match. That is what makes protecting `lo` also protect a client running beside the service. - **On Proxy (UDP)**: because UDP relays need a concrete destination to bind upstream forwarding, Firegex automatically resolves the interface's primary assigned IP. If the interface has no IP assigned, use NFQUEUE instead. - **External proxies**: the `external` transport takes an IP only. Its return rule recognises your proxy by one address and port to put the original port back, and an interface is not one address — so the interface half of the picker is not offered there at all. -## Transport protocol +## What the service speaks -TCP, UDP or TLS, chosen on the service. Every layer carries TCP and UDP; TLS is on the -proxy layer alone, because decrypting means terminating the connection — see -[TLS](#tls). +One choice per service: **TCP**, **UDP**, **TLS**, **QUIC** or **HTTP**. It tells firegex +what is on the wire, which is what decides what the filters can be shown. + +| | Pick it for | The filters are shown | It needs | +|---|---|---|---| +| **TCP** | anything unencrypted on TCP — a web service on `:80`, a game protocol, a shell | the bytes as they travel; HTTP/1.1 parsed, and HTTP/2 in the clear recognised from its preface and rendered as HTTP/1.1 | nothing but the addresses | +| **UDP** | anything unencrypted on UDP — a game server, DNS, something homemade | one datagram at a time, both directions, `RawPacket` only | nothing but the addresses | +| **TLS** | a port where *everything* arrives encrypted | the decrypted stream, the same models a cleartext service gives | a certificate and key; the proxy layer. [What the service behind speaks](#what-the-service-behind-speaks) is its own question | +| **QUIC** | a QUIC service, in practice HTTP/3 | one stream at a time with its own state; HTTP/3 rendered as HTTP/1.1 | a certificate and key; the proxy layer | +| **HTTP** | one daemon reached **more than one way at once** — `:80`, `:443` and `:443` over UDP | every version as the same HTTP/1.1, gRPC included | a certificate and key, including for the cleartext addresses; the proxy layer | + +**The interface asks this as two questions**, because the list of five looks like five +alternatives on one axis and is not: + +> **Is the traffic encrypted?** in the clear · encrypted 🔒 +> **Encrypted with what?** TLS · QUIC · HTTPS — or, in the clear: **TCP** · **UDP** + +The first answer is the one the certificate hangs off, which is why it wears the lock; the +second only ever offers protocols of that kind. `HTTP` is shown as **HTTPS** there, because +beside TLS and QUIC an option labelled "HTTP" reads as the cleartext web protocol — which +is the confusion this picker was rebuilt around. It is `http` in the API and in a backup. + +**A plain web service is `TCP`.** Filtering HTTP needs nothing special: HTTP/1.1 is parsed +as it goes past, HTTP/2 in the clear is recognised from its connection preface, and +patterns match the plaintext because the plaintext is what is on the wire. + +Every layer carries TCP and UDP; the three that have to be decrypted are on the proxy +layer alone, because decrypting means terminating the connection — see [TLS](#tls), +[QUIC](#quic) and [HTTP](#http-every-version-through-one-chain). **UDP on the proxy layer preserves the client's address** transparently using `IP_TRANSPARENT` source spoofing and policy routing, matching the behavior of TCP. See @@ -80,7 +170,7 @@ allowed to do to the traffic. Pick by those. | Measured ([how](../tests/bench/README.md#performance)) | **4035 MB/s** at 1 thread, **13 984** at 8 | 1820 at 1 thread, 2956 at 8 | | Short connections | the two are indistinguishable — see below | | | UDP | yes, fully transparent (source IP preserved, one relay per address) | yes, fully transparent | -| TLS termination | yes | no — decrypting means terminating | +| TLS and QUIC | yes, terminated here | no — decrypting means terminating | ### Proxy @@ -284,6 +374,51 @@ so the fields are empty when you come back to edit and leaving them that way kee is already there. Switching a service *to* TLS when it has never been given either is refused there and then. +#### What the service behind speaks + +**It is asked of each address**, under its ⚙ button, and not of the service: what leaves +towards the service is a property of the way in. One daemon reached over TLS on one port +and in the clear on another is re-encrypted for the first and handed the plaintext for the +second. + +| | What firegex does | +|---|---| +| **As it arrives** | It speaks what arrived. The default, and what every address did before there was a choice: a connection terminated here goes back out encrypted. Firegex is *carrying* its encryption. | +| **In the clear** | It is an ordinary HTTP/1.1 service. Firegex terminates what the client used and forwards the plaintext, so **only firegex needs a certificate**. | +| **TLS** | It speaks HTTP/1.1 under its own TLS, whatever the client used to get here. | + +The second one is why this exists. A service that speaks HTTP and nothing else can be put +behind an encrypted edge with firegex providing the encryption: clients reach it over TLS +— or over **HTTP/3**, see below — on the port it already listens on in the clear. + +Two things follow from choosing anything but *as it arrives*, and both are worth knowing +before a round: + +- **ALPN is no longer mirrored** on that address. The protocol a client is told is + normally the one the service picked, and a service reached over HTTP/1.1 picks nothing. + On TLS the client is told nothing and settles on HTTP/1.1; on QUIC, firegex answers `h3` + — for itself, not for the service, because it is firegex that is speaking HTTP/3. +- **Stopping firegex stops the encryption**, because the encryption was firegex's. A + client dialling that port with TLS then reaches a service answering in the clear. + +##### HTTP/3 in front of a service that has never heard of it + +This is the one place in the engine where what leaves is **not** the version that +arrived, and it is only ever an explicit choice. An HTTP/3 exchange is already rendered as +the HTTP/1.1 it would have been, because that is what the filters have to be shown — so +that same rendering can be sent on to a service that speaks HTTP/1.1. Nothing is invented: +what the service receives is byte for byte what the chain inspected. + +One QUIC stream is one request, and HTTP/1.1 has no multiplexing, so each exchange opens +its own connection to the service — dialled from the client's own address like every other +dial firegex makes. + +What cannot be done, and is a limit rather than a missing setting: a QUIC edge carrying +anything **other than HTTP/3**. Opaque bytes on a QUIC stream have no HTTP/1.1 form, so +there is nothing to send a service that speaks one. In practice that means an instance +whose `FGEX_PROXY_QUIC_ALPN` is not `h3`: the choice is refused there, on `QUIC` and +`HTTPS` services alike, with the ALPN named. + #### What is carried through, and what cannot be **ALPN is the service's answer, carried.** Terminating a connection means answering for @@ -298,6 +433,47 @@ That means HTTP/2 works if your service speaks it, and does not appear if it doe Before this, ALPN was dropped in both directions — every client fell back to HTTP/1.1 whatever it asked for, and a client that insists on `h2` failed. +> **HTTP/2 is terminated and rendered, exactly as HTTP/3 is.** HTTP/2 puts the method, +> the path and the headers in an **HPACK-compressed** HEADERS frame, so a firegex that +> only forwarded it showed a filter a compression format: a pattern written against a +> request line matched nothing, and a filter asking for an `HttpRequest` was handed the +> `PRI * HTTP/2.0` preface and then frames it could not read, so it was never called and +> nothing said so. Only a body travelled in the clear. Since gRPC *is* HTTP/2, gRPC could +> not be filtered at all. +> +> On a connection that negotiates `h2`, the engine now terminates it and shows the chain +> each exchange as the HTTP/1.1 it would have been — the same rendering HTTP/3 gets (see +> [below](#http3-is-shown-to-the-filters-as-http11)), from the same code. One pattern and +> one Python filter cover HTTP/1.1, HTTP/2 and HTTP/3. There is nothing to turn off on +> your service any more. +> +> Since gRPC is HTTP/2, that is also what makes gRPC filterable: the method name is the +> path, the metadata are headers and the status is a trailer section, all of them visible +> to a pattern and to a Python filter. For the message bodies themselves there is a +> `GrpcMessage` model that takes the length prefix off — see +> [the pyfilter documentation](pyfilter.md#grpc-messages). +> +> **HTTP/2 in the clear** (`h2c`, prior knowledge) is recognised too, on a plain `tcp` +> service with no certificate anywhere: such a client opens with a fixed 24-byte preface +> that no other protocol begins with. What is *not* handled is the old HTTP/1.1 +> `Upgrade: h2c` handshake — the upgrade request is filtered as the HTTP/1.1 request it +> is, but if your service answers `101` everything after that is HTTP/2 nobody rendered. +> RFC 9113 deprecated that mechanism and nginx dropped it in 1.25.1, so in practice no +> service accepts it. +> +> Two shapes are **refused rather than rendered**, because a tunnel has no HTTP exchange +> in it to show and inventing one would be worse than saying no: **server push**, which +> is turned off in the handshake with your service so it knows rather than watching its +> pushes disappear, and **CONNECT** streams, including the extended CONNECT that +> WebTransport uses. +> +> One difference to know about if you write stateful Python filters: **each HTTP/2 stream +> is its own connection to a filter**, with its own module globals, where an HTTP/1.1 +> keep-alive connection carries many requests through one. HTTP/2 interleaves its +> streams, so sharing state between them would let one client's bytes decide another's +> verdict — and would let an attacker split a pattern across two streams to get past a +> filter. + **TLS versions are negotiated per leg**, which is what terminating means: a client on TLS 1.2 can reach a service on TLS 1.3. Both legs do **TLS 1.2 and 1.3 only**. TLS 1.0 and 1.1 are not supported and cannot be turned on — the engine's TLS library does not @@ -323,7 +499,7 @@ equivalent switch. The plaintext never becomes a packet — that is exactly what removed the two ports — so the engine writes it out itself. **`firegex0` carries the decrypted traffic of every TLS -service and nothing else:** +and QUIC service and nothing else:** ```bash sudo tcpdump -i firegex0 -w decrypted.pcap @@ -332,8 +508,7 @@ sudo tcpdump -i firegex0 -w decrypted.pcap No filter expression, no port to look up, and one capture covering the whole instance rather than one per service. Wireshark takes the same interface live, or opens the file afterwards. Run it on the **host** running firegex: the container shares the host's -network namespace, so the interface is there. It exists only while some TLS service is -running. +network namespace, so the interface is there. What arrives there is a **reconstruction**. The bytes are real — exactly what the filters saw and exactly what was forwarded, after any rewriting — but the framing around them is @@ -349,6 +524,203 @@ so once at startup. What comes out is decrypted traffic, which makes the file exactly as sensitive as the private key that would have produced it. +### QUIC + +QUIC is the fourth thing a service can speak, and the one with the least room for +argument about where it lives. + +TLS over TCP can at least be carried past unopened: the bytes are framed by a transport +the kernel understands, so a layer that forwards still has a packet to match and a rule +to match it with, and filtering the ciphertext is a thing you can choose to do. QUIC +leaves not even that. Past the Initial packet it encrypts the **frames, the stream +boundaries and the packet number** along with the payload, so a packet queued to +userspace is a datagram of noise. Terminating it is not the better option, it is the only +one — which is why the QUIC option greys out on the other two layers, exactly as TLS +does, and says so. + +On the wire QUIC is **UDP**, and that is what the rules match: a QUIC service's addresses +are steered the same way a datagram service's are, one endpoint bound per protected +address. What changes is what is behind the port — something that terminates rather than +something that forwards. + +Give it a certificate and a private key, as for TLS. QUIC carries TLS 1.3 inside itself, +so there is no unencrypted QUIC to fall back to and no version to negotiate. + +**A stream is what the filters see.** A QUIC connection carries many streams at once, and +each one is handed to the chain as its own connection: its own filter state, its own set +of module globals for a Python filter, released when that stream ends. That is the honest +mapping — a filter's state follows a stream of bytes from its start to its end, which is +what a QUIC stream is and what a QUIC connection is not — and on HTTP/3 it lands where +you would expect, one request to a stream. + +**A refusal ends the connection, not the stream.** Blocking one stream would leave the +client free to ask again on the next one, which is not what blocking means anywhere else +in firegex; it is the same thing that happens on HTTP/1.1 with keep-alive, where a +refused request takes the connection with it. The client is told why, in the close +reason, rather than being left to time out. + +#### HTTP/3 is shown to the filters as HTTP/1.1 + +A filter is supposed to be written once. `HttpRequest` on a QUIC service has to mean what +it means on the TCP service beside it, and a hyperscan pattern written for one has to +match on the other — otherwise moving a service to QUIC silently switches its filters +off. + +But HTTP/3 puts the method, the path and the headers in a **QPACK-compressed HEADERS +frame**. A filter shown the bytes of the stream would be shown a compression format, and +every pattern would stop matching. So firegex terminates HTTP/3 too, and renders each +exchange as the HTTP/1.1 message it would have been. That is what the chain is shown, and +what goes on towards your service is HTTP/3 again, re-encoded. + +The same bargain the [decrypted capture](#watching-the-decrypted-traffic) makes: **the +bytes are real and the framing is reconstructed.** Concretely — + +- the request line says `HTTP/1.1`, because that is the only version an HTTP/1 parser + will read, and what was on the wire was HTTP/3; +- `Host` is rendered from the `:authority` pseudo-header, which is where HTTP/3 puts it; +- a body whose length the message declared is shown under its own `Content-Length`; +- a body whose length it did not declare is shown **chunked**, which is what that message + is in HTTP/1.1 — and the head is held back until it is known whether a body is coming + at all, so that a request without one is never shown a `Transfer-Encoding` it did not + carry. A filter looking for that header is usually looking for smuggling, and one + invented by the proxy in front of it would be the worst possible answer. Held *briefly*: + a declared length settles the question outright, a message with no body has already + ended its stream, and the only thing left waiting is a sender holding the stream open — + which after 100 ms is rendered chunked rather than held, because a bidirectional + exchange the service speaks first in (gRPC's among them) would otherwise deadlock, each + end waiting for the other; +- a **trailer section is always shown to the filters**, and that is what decides the + framing of a message that has one and no body — in HTTP/1.1 a trailer section belongs to + a chunked message and nowhere else, which is exactly a gRPC trailers-only answer. The + one message whose rendering cannot carry a trailer section is one that declared its + length, since its HTTP/1.1 body ends where the header said it would; there the trailers + are **dropped rather than forwarded**, because a piece of a message no filter was shown + is a piece that travelled unfiltered. The engine logs it when it happens; +- hop-by-hop headers do not appear, because HTTP/3 forbids sending them. + +#### What is carried through, and what cannot be + +**ALPN is the service's answer, carried — asked in the other order.** On TLS over TCP the +client's hello is held open, the service is asked with exactly the list the client +offered, and the client is told what came back. In QUIC the hello arrives inside an +encrypted Initial packet whose processing *is* the handshake, so there is nothing to hold +it at. The order is therefore reversed: firegex opens its connection to the service +first, offering the protocols it was told to offer, and tells the client the one thing +the service agreed to. The invariant that matters is unchanged — the client is never told +a protocol the service did not choose — and what is lost is knowing in advance whether +the client would have accepted it. When it would not, its handshake fails and the log +says which protocol the service picked. + +That list is `h3`, which is what a QUIC service speaks nine times in ten. An instance in +front of something else sets `FGEX_PROXY_QUIC_ALPN` as a comma-separated list in +preference order: + +```bash +python3 run.py restart --env FGEX_PROXY_QUIC_ALPN=h3,doq +``` + +`run.py` stores it and puts it back on every later start — editing the generated compose +file by hand does not survive one, because `run.py` rewrites that file each time. The +list is **per instance, not per service**, and it is a superset rather than a choice: +every QUIC service is offered all of it and each one picks what it actually speaks, so +listing every protocol the instance carries is the way to run more than one kind of QUIC +service at once. + +**0-RTT is not offered.** Early data is replayable by anyone who watched it go past, and +a filter that refused a request has no way to un-deliver the copy your service already +acted on. A round trip is the price of that not being true. + +**QUIC datagrams are carried, except on HTTP/3.** A datagram is unreliable and unordered, +which is to say it is not a stream — so a filter is handed a `RawPacket` for it and +nothing else, exactly as on a UDP service, and the stream and HTTP models stay out of the +way rather than being built on something that cannot support them. One connection's +datagrams are **one flow**: a filter keeping state sees them in order of arrival and can +still catch a pattern split across two of them, which is what the flow is for. A refused +datagram is **dropped and the connection carries on** — there is no conversation to end, +and taking the connection down would cost every stream on it for one message that was +already complete when it was judged. + +Over HTTP/3 they are not carried, and the handshake says so rather than letting a peer +find out by watching them disappear. An HTTP/3 datagram names the stream it belongs to, +and the request streams firegex opens towards your service are not the ones your client +opened, so the name would point at the wrong stream on the far side. **WebTransport is not +carried** for a further reason: its streams are bound to a session by a marker each of +them carries, which is a different shape from "one stream, one filter" and would need a +session model rather than a setting. + +**A new client is asked to prove its address.** The first Initial packet from an address +is answered with a Retry, so that nothing is opened towards your service on the word of a +source address nobody has checked — otherwise a forged source turns the relay into an +amplifier, which is the datagram flood the connection cap exists for with a handshake in +front of it. It costs an honest client one round trip. + +**The decrypted traffic is mirrored to `firegex0`, one TCP stream per QUIC stream** — and +over HTTP/3 that is one per request. What you read there is the same HTTP/1.1 the filters +were shown, which is the only honest thing it could be: what crossed the wire was a +compressed header block inside an encrypted packet, and there is no tool that would read +it back as an exchange. + +One thing to know before reading a port out of that capture: **the client port is +invented**. Every stream of one QUIC connection shares the client's real port, so each is +given one of its own — otherwise a hundred streams arrive as one conversation whose bytes +decode as nothing. The addresses are real and the service's port is real; the client's +names a stream, not a socket. + +**The connection limit is honoured; forwarding the excess unfiltered is not free here.** +On TCP a connection past the limit is relayed with no chain at all, costing nothing. A +QUIC connection past the limit is still terminated, decrypted and re-encrypted, because +there is no such thing as forwarding a QUIC connection unopened — what "unfiltered" +saves is the inspection, not the termination. + +### HTTP: every version through one chain + +`HTTP` is the one protocol on that list that does not say what is on the wire. It says +what the *service* is, and each of its addresses says how it is reached: + +| Address | What reaches it | +|---|---| +| **TCP** | HTTP/1.1 and HTTP/2 — in the clear or under TLS, whichever each client opens with | +| **UDP** | HTTP/3, over QUIC | + +One service, one filter chain, one certificate. Add `10.0.0.1:80` as TCP, `10.0.0.1:443` +as TCP and `10.0.0.1:443` as UDP, and every client that can reach your web service goes +through the same filters. + +**Why it exists.** HTTP/1.1 and HTTP/2 live on TCP and HTTP/3 lives on UDP, so covering +all three used to take two or three separate firegex services with the same patterns and +the same Python pasted into each of them. Keeping those in step by hand is exactly how one +of them quietly stops being protected — which is the same reason a service takes a *list* +of addresses rather than making you create one service per address. + +**Whether a connection is TLS is decided per connection**, from what the client actually +sent, not from a setting. So one `http` service in front of a daemon that answers in the +clear on `:80` and under TLS on `:443` carries both, and what leaves towards your service +matches what arrived: a cleartext connection is forwarded in the clear, an encrypted one +is re-encrypted. It is the same rule ALPN mirroring follows — firegex carries what the two +ends are doing rather than deciding it for them. + +**The service behind may answer in the clear too**, including on the HTTP/3 edge — see +[what the service behind speaks](#what-the-service-behind-speaks). Every edge is then +forwarded as HTTP/1.1, which is what an ordinary web service reached on `:80`, `:443` and +`:443/udp` at once looks like: three protocols in front, one behind, one chain over all +of them. + +**A certificate is required**, including for the cleartext addresses beside it. A service +that only ever answers in the clear should be a `TCP` one: HTTP/1.1 is filtered there as +it always was, and HTTP/2 in the clear is recognised from its connection preface. The TLS +and HTTP/3 edges are the whole of what `HTTP` adds, and neither exists without a +certificate. + +**Proxy layer only.** An `HTTP` service is reached over TLS on one port and QUIC on +another, and the NFQUEUE and hand-off layers can read neither — nor, in fact, the +cleartext edge, because HTTP/2 in the clear is HPACK-compressed and rendering it means +terminating the connection. + +Everything the [TLS](#tls) and [QUIC](#quic) sections say about what is and is not +supported applies here unchanged: the same TLS versions and ciphers, no client +certificates, no 0-RTT, no HTTP/3 datagrams and no WebTransport, and the same +`firegex0` capture of the decrypted traffic. + ## Filters ### Regex diff --git a/fgex-lib/firegex/pyfilters/internals/data.py b/fgex-lib/firegex/pyfilters/internals/data.py index 073f1de..e627b04 100644 --- a/fgex-lib/firegex/pyfilters/internals/data.py +++ b/fgex-lib/firegex/pyfilters/internals/data.py @@ -31,11 +31,17 @@ class RawPacket: dst_ip: str = "", src_port: int = 0, dst_port: int = 0, + l4: str | None = None, ): self.__data = bytes(data) self.__is_input = bool(is_input) self.__is_ipv6 = bool(is_ipv6) - self.__is_tcp = bool(is_tcp) + # Derived from `is_tcp` when nothing said otherwise, because one of the engines + # sending this has only ever had two answers to give: the NFQUEUE binaries carry + # TCP and UDP and nothing else, and asking them to learn a third word to say the + # same thing would be a change with no question behind it. + self.__l4 = str(l4) if l4 else ("tcp" if is_tcp else "udp") + self.__is_tcp = self.__l4 == "tcp" self.__src_ip = str(src_ip) self.__dst_ip = str(dst_ip) self.__src_port = int(src_port) @@ -58,9 +64,34 @@ class RawPacket: @property def is_tcp(self) -> bool: - "True if the connection is TCP, false if it is UDP" + "True if the connection is TCP. False for a datagram, and false for QUIC" return self.__is_tcp + @property + def l4(self) -> str: + """What carries this connection. + + `tcp`, `udp`, `quic` for a stream inside a QUIC connection, or `quic-datagram` + for a DATAGRAM frame in one — which is in a connection carrying streams and is + not one, the distinction `is_stream` is there to answer. + """ + return self.__l4 + + @property + def is_stream(self) -> bool: + """True if these bytes arrive in order, once each, as part of a stream. + + The question every model above `RawPacket` actually asks. It used to be spelled + `is_tcp`, and the two came apart the day QUIC arrived: a QUIC stream is ordered + and reliable, so an assembled stream and a parsed HTTP message mean exactly what + they mean on TCP — and it is carried by UDP, so a filter asking what is on the + wire has to be told UDP. One flag answering both got one of them wrong whichever + way it was set. A datagram — plain UDP or QUIC's own — answers false here, so a + filter asking for a stream model is simply not called for one and a single file + can carry both. + """ + return self.__l4 in ("tcp", "quic") + @property def src_ip(self) -> str: "Where this chunk came from" @@ -131,7 +162,7 @@ class RawPacket: way = "client -> service" if self.is_input else "service -> client" return ( f"RawPacket({way}, {self.src_ip}:{self.src_port} -> {self.dst_ip}:{self.dst_port}, " - f"{'tcp' if self.is_tcp else 'udp'}/{'ip6' if self.is_ipv6 else 'ip4'}, " + f"{self.l4}/{'ip6' if self.is_ipv6 else 'ip4'}, " f"{self.data_size} bytes)" ) diff --git a/fgex-lib/firegex/pyfilters/models/__init__.py b/fgex-lib/firegex/pyfilters/models/__init__.py index 6ee10d0..77fd4bc 100644 --- a/fgex-lib/firegex/pyfilters/models/__init__.py +++ b/fgex-lib/firegex/pyfilters/models/__init__.py @@ -13,6 +13,9 @@ from firegex.pyfilters.models.http import ( HttpFullResponse, HttpHistory, HttpStreamHistory, + GrpcMessage, + GrpcRequest, + GrpcResponse, ) from firegex.pyfilters.internals.data import RawPacket @@ -41,6 +44,13 @@ type_annotations_associations = { HttpFullResponse: HttpFullResponse._fetch_packet, HttpHistory: HttpHistory._fetch_packet, HttpStreamHistory: HttpStreamHistory._fetch_packet, + # gRPC is not a protocol beside HTTP, it is a body format inside it — so its + # models live here rather than under a protocol of their own. That is what keeps + # them free: a new protocol would mean a new value in `filters.proto`, and + # changing that schema deletes and recreates the services database. + GrpcMessage: GrpcMessage._fetch_packet, + GrpcRequest: GrpcRequest._fetch_packet, + GrpcResponse: GrpcResponse._fetch_packet, }, } @@ -58,6 +68,9 @@ __all__ = [ "HttpFullRequest", "HttpFullResponse", "HttpHistory", + "GrpcMessage", + "GrpcRequest", + "GrpcResponse", "HttpStreamHistory", ] diff --git a/fgex-lib/firegex/pyfilters/models/http.py b/fgex-lib/firegex/pyfilters/models/http.py index b1ed47e..b2ab54d 100644 --- a/fgex-lib/firegex/pyfilters/models/http.py +++ b/fgex-lib/firegex/pyfilters/models/http.py @@ -55,6 +55,12 @@ class InternalHTTPMessage: upgrading_to_h2: bool = field(default=False) upgrading_to_ws: bool = field(default=False) added_to_history: bool = field(default=False) + #: How much of `body` has already been handed to a model that consumes it. + #: + #: On the message rather than beside it because that is what it belongs to: a + #: keep-alive connection carries many messages through one parser, and a cursor kept + #: per connection would start the second message part-way in. + grpc_consumed: int = field(default=0) @dataclass @@ -81,6 +87,18 @@ class InternalCallbackHandler: _ws_extentions = None _ws_raised_error = False release_message_headers = True + #: Hand the message over as its body arrives, not only when it is finished. + #: + #: Off everywhere but the gRPC models, and each model gets a parser of its own + #: (`_parser_class`), so `HttpRequest` keeps its contract exactly: called twice, once + #: with the headers and once with the whole message. It exists because a gRPC stream + #: that is answered message by message — a server-streaming or bidirectional RPC — + #: has a body that does not finish until the stream does, so a filter shown only + #: finished bodies would be called once, at the end, which is too late to be a + #: firewall. + release_body_chunks = False + #: Whether a chunk has arrived that has not been handed over yet. + _body_chunk_pending = False def reset_data(self): self.msg = InternalHTTPMessage() @@ -91,6 +109,7 @@ class InternalCallbackHandler: self.buffers = InternalHttpBuffer() self.msg = InternalHTTPMessage() self.has_begun = True + self._body_chunk_pending = False def on_url(self, url): self.buffers._url_buffer += url @@ -168,6 +187,13 @@ class InternalCallbackHandler: if self.save_body: self.msg.total_size += len(body) self.buffers._body_buffer += body + if self.release_body_chunks: + # Visible as it arrives rather than only at the end. **Undecoded**, and + # that is not an oversight: `content-encoding` is undone once, in + # `on_message_complete`, and half a gzip stream is not half a body. The + # models that read this decline while an encoding is in the way. + self.msg.body = self.buffers._body_buffer + self._body_chunk_pending = True def on_message_complete(self): self.msg.body = self.buffers._body_buffer @@ -446,7 +472,7 @@ class HttpHistory: def _fetch_packet(cls, internal_data: DataStreamCtx): if ( internal_data.current_pkt is None - or internal_data.current_pkt.is_tcp is False + or internal_data.current_pkt.is_stream is False ): raise NotReadyToRun() @@ -585,6 +611,10 @@ class InternalBasicHttpMetaClass: def _should_release_message_headers(cls) -> bool: return True + @classmethod + def _should_release_body_chunks(cls) -> bool: + return False + @staticmethod def _before_fetch_callable_checks(internal_data: DataStreamCtx) -> bool: raise NotImplementedError() @@ -597,7 +627,7 @@ class InternalBasicHttpMetaClass: def _fetch_packet(cls, internal_data: DataStreamCtx): if ( internal_data.current_pkt is None - or internal_data.current_pkt.is_tcp is False + or internal_data.current_pkt.is_stream is False ): raise NotReadyToRun() @@ -614,6 +644,10 @@ class InternalBasicHttpMetaClass: internal_data.data_handler_context[parser_key] = parser parser.release_message_headers = cls._should_release_message_headers() + # Set here and not only in the constructor hook, for the same reason the line + # above is: parsing happens before any instance exists, and a flag applied + # afterwards would take effect one packet late. + parser.release_body_chunks = cls._should_release_body_chunks() if not internal_data.call_mem.get( @@ -714,6 +748,19 @@ class InternalBasicHttpMetaClass: # that counts did not. messages_tosend.append(parser.msg) + if ( + parser.release_body_chunks + and parser._body_chunk_pending + and not parser.msg.message_complete + and not any(msg is parser.msg for msg in messages_tosend) + ): + # A body still arriving, handed over as it does. Identity rather than + # equality on the guard: `InternalHTTPMessage` is a dataclass, so `in` would + # compare every field of every message to decide something that is a question + # about which object this is. + messages_tosend.append(parser.msg) + parser._body_chunk_pending = False + if parser._packet_to_stream(): messages_tosend.append( parser.msg @@ -905,3 +952,214 @@ class HttpResponseHeader(HttpResponse): @staticmethod def _parser_class() -> str: return "http_header" + + +#: The header in front of every gRPC message: one flag byte, then a big-endian length. +GRPC_HEADER_SIZE = 5 + + +@dataclass +class GrpcFrame: + """One length-prefixed message out of a gRPC body.""" + + #: Whether this message is compressed, which its own flag byte says. What it is + #: compressed *with* is `grpc-encoding` on the message that carried it. + compressed: bool + #: The bytes between the length prefix and the next one — a protobuf message, in + #: whatever schema the two ends agreed on out of band. + payload: bytes + + +def _grpc_frames(msg: InternalHTTPMessage) -> list[GrpcFrame]: + """Take every complete message the body has grown since the last look. + + Nothing is allocated on a claimed length: a frame is taken only once the bytes behind + it have actually arrived, so a header claiming four gigabytes costs nothing but the + wait — and the wait is already bounded by `FGEX_STREAM_MAX_SIZE`, which is what stops + a sender buying memory in this process. + """ + body = msg.body or b"" + taken: list[GrpcFrame] = [] + at = msg.grpc_consumed + while len(body) - at >= GRPC_HEADER_SIZE: + length = int.from_bytes(body[at + 1 : at + GRPC_HEADER_SIZE], "big") + end = at + GRPC_HEADER_SIZE + length + if len(body) < end: + break + taken.append( + GrpcFrame(compressed=body[at] != 0, payload=body[at + GRPC_HEADER_SIZE : end]) + ) + at = end + msg.grpc_consumed = at + return taken + + +def _speaks_grpc(msg: InternalHTTPMessage) -> bool: + return (msg.content_type or "").lower().startswith("application/grpc") + + +def _body_can_be_framed(msg: InternalHTTPMessage) -> bool: + """Whether the body as it stands can be read as gRPC frames at all. + + A `content-encoding` is undone once, when the message finishes, so while one is in + the way the bytes in hand are compressed and the framing is not in them. gRPC does not + use it — it compresses each message on its own and says so in the frame's flag byte — + so in practice this only ever declines for traffic that was never gRPC to begin with. + """ + if msg.message_complete: + return True + encodings = {e.strip() for e in (msg.content_encoding or "").lower().split(",")} + return not (encodings - {"", "identity"}) + + +class GrpcMessage(InternalBasicHttpMetaClass): + """One gRPC message, in either direction. + + gRPC is HTTP/2, so everything a filter could already ask about the exchange — the + method in the path, the headers, the trailer section carrying `grpc-status` — arrives + through `HttpRequest` and `HttpResponse`, because the engine renders every version of + HTTP as HTTP/1.1 before the chain sees it. What did not arrive was the *body*: a gRPC + body is a sequence of length-prefixed messages, so a filter reading `request.body` was + reading a five-byte header glued to a protobuf blob, and a pattern written against the + payload had to know to skip it. + + This hands over one message at a time, as each one completes. It is called **per + message and not per body**, which is what makes it usable on a streaming RPC: a + server-streaming or bidirectional call has a body that does not finish until the + stream does, and a filter shown only finished bodies would be called once, at the end. + + What it does **not** do is decompress a message whose own flag byte says it is + compressed, or decode the protobuf inside it. The first because what it is compressed + with is the peers' agreement (`grpc-encoding`) and guessing is how a filter comes to + read something that is not there; the second because without the `.proto` there is no + schema, and a pattern against the raw payload is what a ruleset has today. + """ + + @staticmethod + def _before_fetch_callable_checks(internal_data: DataStreamCtx) -> bool: + return True + + @staticmethod + def _parser_class() -> str: + return "http_grpc" + + @classmethod + def _should_release_message_headers(cls) -> bool: + # There is no message to show at the head: a gRPC exchange is its messages, and + # what is in the headers is what `HttpRequest` is for. + return False + + @classmethod + def _should_release_body_chunks(cls) -> bool: + return True + + def _contructor_hook(self): + self._parser.release_message_headers = False + self._parser.release_body_chunks = True + self._frame: GrpcFrame | None = None + self._is_input: bool = True + + @classmethod + def _fetch_packet(cls, internal_data: DataStreamCtx): + # The base drives the parser and decides which messages are ready; this only + # takes the gRPC framing out of the ones that have it. Written that way round so + # there is still exactly one HTTP parser in this library and this is not a second. + carried = super()._fetch_packet(internal_data) + if not isinstance(carried, list): + carried = [carried] + + pieces = [] + for one in carried: + msg = one._message + if not _speaks_grpc(msg) or not _body_can_be_framed(msg): + continue + for frame in _grpc_frames(msg): + piece = cls(one._parser, msg) + piece._history = one._history + piece._frame = frame + piece._is_input = internal_data.current_pkt.is_input + pieces.append(piece) + + if not pieces: + # Not "nothing matched" but "there is nothing to show yet" — a body half-way + # through a message, or an exchange that is not gRPC at all. The filter is not + # called, which is what `NotReadyToRun` means everywhere else. + raise NotReadyToRun() + + result = pieces[0] if len(pieces) == 1 else pieces + internal_data.call_mem[f"_fetched_obj_{cls._parser_class()}"] = result + return result + + @property + def payload(self) -> bytes: + """This message's bytes: the protobuf, without the length prefix.""" + return self._frame.payload if self._frame else b"" + + @property + def compressed(self) -> bool: + """Whether this message's own flag byte says it is compressed. + + What with is `grpc-encoding` on the message that carried it. Nothing here + decompresses it, so `payload` is the compressed bytes when this is true. + """ + return bool(self._frame and self._frame.compressed) + + @property + def is_request(self) -> bool: + """Whether this went from the client towards the service.""" + return self._is_input + + @property + def method(self) -> str | None: + """The gRPC method, which is the HTTP path: `/package.Service/Method`. + + Present on both directions, because both halves of an exchange belong to the same + stream and the engine renders the request that opened it. + """ + return self.url + + @property + def grpc_status(self) -> str | None: + """The status, where the message carrying this frame stated one. + + A gRPC status lives in the trailer section, and on a reply that refuses before + sending anything it lives in the headers instead — the *trailers-only* shape. Read + it where it is; on a reply that is still streaming it is not there yet, and this + is `None` rather than a guess. + """ + return self.get_header("grpc-status") + + def __repr__(self): + return ( + f"" + ) + + +class GrpcRequest(GrpcMessage): + """A gRPC message on its way to the service, and nothing coming back.""" + + @staticmethod + def _before_fetch_callable_checks(internal_data: DataStreamCtx) -> bool: + return internal_data.current_pkt.is_input + + @staticmethod + def _parser_class() -> str: + # Its own parser, as `HttpFullRequest` has one beside `HttpRequest`. Not tidiness: + # taking a frame *consumes* it, so two models sharing one parser would mean the + # second one is handed nothing and silently never runs. + return "http_grpc_req" + + +class GrpcResponse(GrpcMessage): + """A gRPC message on its way back to the client.""" + + @staticmethod + def _before_fetch_callable_checks(internal_data: DataStreamCtx) -> bool: + return not internal_data.current_pkt.is_input + + @staticmethod + def _parser_class() -> str: + return "http_grpc_res" diff --git a/fgex-lib/firegex/pyfilters/models/tcp.py b/fgex-lib/firegex/pyfilters/models/tcp.py index e0b1b1e..0f5ec36 100644 --- a/fgex-lib/firegex/pyfilters/models/tcp.py +++ b/fgex-lib/firegex/pyfilters/models/tcp.py @@ -32,7 +32,7 @@ class InternalTCPStream: @classmethod def _fetch_packet(cls, internal_data:DataStreamCtx, is_input:bool=False): - if internal_data.current_pkt is None or internal_data.current_pkt.is_tcp is False: + if internal_data.current_pkt is None or internal_data.current_pkt.is_stream is False: raise NotReadyToRun() if internal_data.current_pkt.is_input != is_input: raise NotReadyToRun() diff --git a/frontend/src/components/Services/AddEditService.tsx b/frontend/src/components/Services/AddEditService.tsx index adb3d00..677895d 100644 --- a/frontend/src/components/Services/AddEditService.tsx +++ b/frontend/src/components/Services/AddEditService.tsx @@ -1,19 +1,27 @@ -import { Accordion, ActionIcon, Alert, Box, Button, Code, Group, Modal, NumberInput, SegmentedControl, Space, Switch, Text, TextInput, Tooltip } from '@mantine/core'; +import { Accordion, ActionIcon, Alert, Box, Button, Code, Group, Modal, NumberInput, Space, Switch, Text, TextInput, Tooltip } from '@mantine/core'; import { useForm } from '@mantine/form'; import { useQueryClient } from '@tanstack/react-query'; import { useEffect, useState } from 'react'; import { BsPlusLg, BsTrashFill } from 'react-icons/bs'; import { errorNotify, isAddressOrInterface, isInterfaceName, okNotify } from '../../js/utils'; -import PortAndInterface from '../PortAndInterface'; +import { addressCapabilities } from './AddressOptions'; +import AddressRow from './AddressRow'; import LayerChoice from './LayerChoice'; import PemInput from './PemInput'; -import { AddressForm, carries, L4, Service, ServiceAddForm, serviceQueryKey, services, Transport } from './utils'; +import ProtocolChoice from './ProtocolChoice'; +import { AddressForm, decrypts, L4, Upstream, Service, ServiceAddForm, serviceQueryKey, services, Transport } from './utils'; type AddressValues = { ip_int: string, port: number, proxy_ip: string, proxy_port: number, + /** `http` only: what is spoken at this address. */ + edge: string, + /** Where the service is, when it is not on this port. Empty means "it is". */ + target_port: number | string, + /** What the service behind this address speaks. */ + upstream: string, } type FormValues = { @@ -32,6 +40,13 @@ type FormValues = { const emptyAddress = (): AddressValues => ({ ip_int: "127.0.0.1", port: 80, proxy_ip: "127.0.0.1", proxy_port: 8080, + // Only ever read on an `http` service, where each address says what is spoken at it; + // everywhere else the service's own protocol decides and this is ignored. + edge: L4.TCP, + // Both behind the row's own button, and both absent unless they were set: an address + // is where the service listens, forwarded as it arrived, until somebody says else. + target_port: "", + upstream: Upstream.SAME, }) /** @@ -77,13 +92,13 @@ export default function AddEditService({ opened, onClose, edit }: { // checked whenever something *was* typed is the PEM envelope, the same check // the backend makes — so the wrong field is named instead of the whole form. tls_cert: (v, values) => { - if (values.proto !== L4.TLS) return null + if (!decrypts(values)) return null if (v === "") return stored ? null : "A certificate is required" return v.includes("-----BEGIN CERTIFICATE-----") ? null : "Not a PEM certificate: no 'BEGIN CERTIFICATE' block in it" }, tls_key: (v, values) => { - if (values.proto !== L4.TLS) return null + if (!decrypts(values)) return null if (v === "") return stored ? null : "A private key is required" if (v.includes("ENCRYPTED PRIVATE KEY-----")) return "Passphrase-protected keys cannot be used: decrypt it first" @@ -124,9 +139,18 @@ export default function AddEditService({ opened, onClose, edit }: { const submit = async (values: FormValues) => { setSubmitting(true) const isExternal = values.transport === Transport.EXTERNAL + const caps = addressCapabilities(values.proto, values.transport) const addresses: AddressForm[] = values.addresses.map(a => ({ ip_int: a.ip_int, port: a.port, + // Sent only where it means something — the same rules the ⚙ on each row + // offers them by. Everywhere else the backend refuses them rather than + // letting a stored value sit there being read by nothing. + ...(values.proto === L4.HTTP ? { edge: a.edge } : {}), + ...(caps.canPublish && Number(a.target_port) && Number(a.target_port) !== a.port + ? { target_port: Number(a.target_port) } : {}), + ...(caps.canChooseUpstream && a.upstream !== Upstream.SAME + ? { upstream: a.upstream } : {}), ...(isExternal ? { proxy_ip: a.proxy_ip, proxy_port: a.proxy_port } : {}), })) @@ -140,8 +164,8 @@ export default function AddEditService({ opened, onClose, edit }: { max_connections: values.max_connections, over_limit_forwards: values.over_limit_forwards, first_byte_timeout: values.first_byte_timeout, - ...(values.proto === L4.TLS && values.tls_cert !== "" ? { tls_cert: values.tls_cert } : {}), - ...(values.proto === L4.TLS && values.tls_key !== "" ? { tls_key: values.tls_key } : {}), + ...(decrypts(values) && values.tls_cert !== "" ? { tls_cert: values.tls_cert } : {}), + ...(decrypts(values) && values.tls_key !== "" ? { tls_key: values.tls_key } : {}), }) if (err) { setError(err); setSubmitting(false); return } okNotify("Service updated", `${values.name} now uses the ${values.transport} transport`) @@ -153,8 +177,8 @@ export default function AddEditService({ opened, onClose, edit }: { over_limit_forwards: values.over_limit_forwards, first_byte_timeout: values.first_byte_timeout, } - if (values.proto === L4.TLS && values.tls_cert !== "") payload.tls_cert = values.tls_cert - if (values.proto === L4.TLS && values.tls_key !== "") payload.tls_key = values.tls_key + if (decrypts(values) && values.tls_cert !== "") payload.tls_cert = values.tls_cert + if (decrypts(values) && values.tls_key !== "") payload.tls_key = values.tls_key const res = await services.add(payload) if (res.status !== "ok" || !res.service_id) { setError(res.status); setSubmitting(false); return } if (values.autostart) { @@ -174,7 +198,10 @@ export default function AddEditService({ opened, onClose, edit }: { } const isProxy = form.values.transport === Transport.PROXY - const isTls = form.values.proto === L4.TLS + //: Whether the engine terminates and decrypts this service — TLS or QUIC. What the + // certificate fields, their validation and the payload all key off, because the two + // protocols need exactly the same thing from the operator. + const isEncrypted = decrypts(form.values) // What the fold would tell you if you opened it. Built from what *differs* rather // than listing every setting, so a service nobody has tuned says so in three words // and a service that has been says exactly how. @@ -193,17 +220,24 @@ export default function AddEditService({ opened, onClose, edit }: { // Leaving TLS selected while the layer that can decrypt it is not would send a // combination the backend refuses, and the operator would be looking at a protocol // the form is no longer offering. + //: The layer is not offered for a service that is decrypted, so it has to be set + // rather than assumed: a form that hides a control still submits its value. + useEffect(() => { + if (isEncrypted && form.values.transport !== Transport.PROXY) + form.setFieldValue('transport', Transport.PROXY) + }, [isEncrypted]) useEffect(() => { if (form.values.transport === Transport.PROXY) return // Neither survives leaving the layer that honours it. A stored limit that does - // nothing is the same trap as a stored TLS flag that does nothing. - if (isTls) form.setFieldValue('proto', L4.TCP) + // nothing is the same trap as a stored protocol nothing can decrypt. + if (isEncrypted) form.setFieldValue('proto', L4.TCP) if (form.values.max_connections !== 0) form.setFieldValue('max_connections', 0) if (form.values.first_byte_timeout !== 0) form.setFieldValue('first_byte_timeout', 0) }, [form.values.transport]) const isExternal = form.values.transport === Transport.EXTERNAL - const isUdp = form.values.proto === L4.UDP - + //: `http` is the one protocol whose addresses are not all on the same transport, so + // it is the one where each of them has to say which it is. + const isHttp = form.values.proto === L4.HTTP // The form is tall — addresses are a list, and the layer choice explains itself — so // the fields scroll and the actions do not. A submit button that falls below the // fold of a modal is a button that is not there. @@ -216,36 +250,33 @@ export default function AddEditService({ opened, onClose, edit }: { - Transport protocol - - What the service speaks on the wire. TLS is one of them rather than a - switch on top of TCP: the engine decrypts it, so the filters see the - plaintext and no extra port is used. - - - {/* Beside the control it is greyed out in, not further down beside the layer - that decides it: the question a reader has is "why can I not click that". */} - {!carries(form.values.transport, L4.TLS) ? - TLS is unavailable on this layer: it inspects packets as they pass rather - than terminating the connection, and decrypting means terminating. Choose - the proxy layer to use it. - : null} + form.setFieldValue('proto', v)} /> + {/* Directly under the choice that asks for them, not at the far end of the + form. They used to sit below the advanced settings, which put the reason + and the request a screenful apart: an operator who had just picked HTTP for + a service answering in the clear met the fields with nothing next to them + saying which choice had produced them, and read it as firegex wanting a + certificate for a service that has none. */} + {isEncrypted ? <> + + form.setFieldValue('tls_cert', v)} + error={form.errors.tls_cert} /> + + form.setFieldValue('tls_key', v)} + error={form.errors.tls_key} /> + {stored ? + Both are already stored: leave these empty to keep them. + : null} + : null} {edit ? null : <> @@ -255,7 +286,7 @@ export default function AddEditService({ opened, onClose, edit }: { {isExternal ? "One service, one hand-off, as many addresses as it answers on — IPv4 and IPv6 together if that is how it is reachable." - : "One service, one filter chain, as many places as it answers on — IPv4 and IPv6 together if that is how it is reachable, and an interface name where the address is not yours to know."} + : "One service, one filter chain, as many ways in as it has — IPv4 and IPv6 together if that is how it is reachable, and an interface name where the address is not yours to know. One more way in is one more address here."} @@ -266,28 +297,10 @@ export default function AddEditService({ opened, onClose, edit }: { - {form.values.addresses.map((_, index) => - - - - - form.removeListItem('addresses', index)}> - - - - {isExternal ? - - : null} - )} + {form.values.addresses.map((_, index) => + 1} />)} {/* Said once under the list rather than as a validation error per row: the operator is choosing between an address and an interface while they fill it in, and being told afterwards that half of what the picker @@ -299,16 +312,29 @@ export default function AddEditService({ opened, onClose, edit }: { An interface name — eth0, wg0, tun0 — protects whatever address that link currently carries, which is what you want when somebody else hands it out. An address protects that one alone. - {form.values.proto === L4.UDP && form.values.transport === Transport.PROXY - ? " On UDP this layer binds a relay to the interface's own address, so it has to have one; NFQUEUE needs none." + {(form.values.proto === L4.UDP || form.values.proto === L4.QUIC) + && form.values.transport === Transport.PROXY + ? " On UDP and QUIC this layer binds a relay to the interface's own address, so it has to have one; NFQUEUE needs none." : ""} } } - form.setFieldValue('transport', v)} /> + {/* Not a choice for a service that is decrypted: decrypting means + terminating the connection, and one layer does that. A picker with two + greyed-out options and one answer is a question nobody is being asked, so + it is said in a line instead — said rather than dropped, because the layer + is a real property of the service and an operator who has read about the + trade should not have to wonder which side of it they are on. */} + {isEncrypted + ? + Carried by the proxy layer, which is the only one that can + decrypt: the connection is terminated here and reopened towards + the service. + + : form.setFieldValue('transport', v)} />} {/* One fold for everything an operator sets once — after something went @@ -378,28 +404,6 @@ export default function AddEditService({ opened, onClose, edit }: { } - {isTls ? <> - - form.setFieldValue('tls_cert', v)} - error={form.errors.tls_cert} /> - - form.setFieldValue('tls_key', v)} - error={form.errors.tls_key} /> - - {stored - ? "Both are already stored. They are never sent back to the browser, so leaving these empty keeps them as they are." - : "The key is stored and used to configure nginx, and never sent back to the browser."} - - : null} - {edit ? null : <> diff --git a/frontend/src/components/Services/AddressList.tsx b/frontend/src/components/Services/AddressList.tsx index 54b9aa7..0553312 100644 --- a/frontend/src/components/Services/AddressList.tsx +++ b/frontend/src/components/Services/AddressList.tsx @@ -1,5 +1,5 @@ import { CodeHighlight } from '@mantine/code-highlight'; -import { ActionIcon, Badge, Box, Button, Card, Code, Collapse, Group, Modal, Space, Stack, Text, Tooltip } from '@mantine/core'; +import { ActionIcon, Badge, Box, Button, Card, Code, Collapse, Group, Modal, SegmentedControl, Space, Stack, Text, Tooltip } from '@mantine/core'; import { useForm } from '@mantine/form'; import { useQueryClient } from '@tanstack/react-query'; import { useEffect, useState } from 'react'; @@ -10,9 +10,32 @@ import { bareAddress, errorNotify, ipInterfacesQuery, isAddressOrInterface, isIn import { addressKind, BADGE_WIDTH } from '../InterfaceInput'; import PortAndInterface from '../PortAndInterface'; import YesNoModal from '../YesNoModal'; -import { Address, decrypts, Service, serviceQueryKey, services, Transport } from './utils'; +import AddressOptions, { addressCapabilities, addressTags, edgeHint, EDGE_OPTIONS } from './AddressOptions'; +import { Address, decrypts, L4, protoLabel, Service, serviceQueryKey, services, Transport, Upstream } from './utils'; -type Values = { ip_int: string, port: number, proxy_ip: string, proxy_port: number } +type Values = { + ip_int: string, port: number, proxy_ip: string, proxy_port: number, + /** `http` only: what is spoken at this address. */ + edge: string, + /** Where the service is, when it is not on this port. Empty means "it is". */ + target_port: number | string, + /** What the service behind this address speaks. */ + upstream: string, +} + +/** + * What one address of a service is reached over, in the words the operator chose it in. + * + * Every protocol but `http` has one answer for the whole service, and the header already + * says it. `http` is the one that does not: a TCP address there carries HTTP/1.1 and + * HTTP/2, and the UDP one beside it carries HTTP/3, so a row that said only "HTTP" would + * be describing two different things with one word. + */ +const edgeLabel = (service: Service, address: { edge?: string, proto: string }) => + service.proto === L4.HTTP + ? ({ [L4.TCP]: "HTTP", [L4.TLS]: "HTTPS", [L4.QUIC]: "HTTP/3" }[ + address.edge ?? address.proto] ?? "HTTP") + : protoLabel(service.proto) function AddressModal({ opened, onClose, service, edit }: { opened: boolean, onClose: () => void, service: Service, edit?: Address, @@ -21,9 +44,15 @@ function AddressModal({ opened, onClose, service, edit }: { const [busy, setBusy] = useState(false) const [error, setError] = useState(null) const isExternal = service.transport === Transport.EXTERNAL + const isHttp = service.proto === L4.HTTP + const caps = addressCapabilities(service.proto, service.transport) + const anyOption = caps.canPublish || caps.canChooseUpstream || caps.isExternal const form = useForm({ - initialValues: { ip_int: "127.0.0.1", port: 80, proxy_ip: "127.0.0.1", proxy_port: 8080 }, + initialValues: { + ip_int: "127.0.0.1", port: 80, proxy_ip: "127.0.0.1", proxy_port: 8080, + edge: L4.TCP, target_port: "", upstream: Upstream.SAME, + }, validate: { ip_int: v => !isAddressOrInterface(v, { cidr: !isExternal }) ? "Not an IP address, and not an interface name either" @@ -45,14 +74,25 @@ function AddressModal({ opened, onClose, service, edit }: { // which is a service quietly protecting a fraction of what it did before. ip_int: bareAddress(edit.ip_int), port: edit.port, proxy_ip: edit.proxy_ip ?? "127.0.0.1", proxy_port: edit.proxy_port ?? 8080, + edge: edit.edge ?? edit.proto, target_port: edit.target_port ?? "", + upstream: edit.upstream ?? Upstream.SAME, }) else form.reset() }, [opened, edit?.address_id]) const submit = async (values: Values) => { setBusy(true) + // Sent wherever it means something, and sent even when it is empty: this is a + // PUT over the whole row, and `0` is how an operator says the traffic goes back + // to arriving where it is sent. An absent field keeps whatever the row had. const payload = { ip_int: values.ip_int, port: values.port, + ...(isHttp ? { edge: values.edge } : {}), + ...(caps.canPublish ? { + target_port: Number(values.target_port) !== values.port + ? Number(values.target_port) || 0 : 0, + } : {}), + ...(caps.canChooseUpstream ? { upstream: values.upstream } : {}), ...(isExternal ? { proxy_ip: values.proxy_ip, proxy_port: values.proxy_port } : {}), } try { @@ -73,24 +113,39 @@ function AddressModal({ opened, onClose, service, edit }: { } return + title={edit ? "Change this address" : "Protect another address"}>
- {isExternal ? <> + {isHttp ? <> - + What clients speak here + - Its own port, not one another address already uses: the return rule puts - the original port back by recognising your proxy's, so two addresses - behind the same endpoint could not be told apart on the way out. + {edgeHint(form.values.edge)} + {" "}The filters see the same HTTP/1.1 whichever it is, so the chain + does not change. : null} + {/* The same options the ⚙ on a row of the creation form offers, in the same + words, and offered on an **edit** as well as on an add. + + They were add-only while this endpoint rewrote the address and nothing else, + which left an operator who had set one of them while creating the service with + no way back to it short of deleting the address. The edit already takes the + rules back and reinstalls them, so rewriting the rest of the row costs + nothing more than what it already spends. */} + {anyOption ? <> + + name} proto={service.proto} + transport={service.transport} /> + : null} {edit ? "Only this address stops being protected while the rules are replaced. The rest of the service keeps its connections." @@ -118,6 +173,7 @@ export default function AddressList({ service }: { service: Service }) { const [removing, setRemoving] = useState
(null) const addresses = service.addresses ?? [] const only = addresses.length === 1 + const caps = addressCapabilities(service.proto, service.transport) //: What each interface is carrying at the moment, so a row named `wg0` can say // where that actually is. An interface with nothing on it is worth seeing too: // on the proxy layer a UDP relay has to bind one of these. @@ -187,7 +243,15 @@ export default function AddressList({ service }: { service: Service }) { what is on it. No protocol badge — it is a property of the service, the header already says it once, and repeating it on every row is noise that grows with the list. */} - + {/* Grows to everything the buttons do not take, rather than to its + own contents. Sized to content, each row was as wide as its own + longest line — so whether the note beside the badges fitted was + decided by how long that note is, and the row carrying a full + IPv6 kept it inline while the one saying `now 127.0.0.1` pushed + it onto a second line. Two rows of one list, wrapping opposite + ways, for a reason nothing on screen explains. */} + }> - TLS + {edgeLabel(service, address)} : null} - {address.proxy_port ? - → {address.proxy_ip ?? "127.0.0.1"}:{address.proxy_port} - : null} + {/* Said on the row, and in the same words the panel that sets + them uses: this is where an operator comes back to check what + they chose, and a setting named differently here reads as a + different setting. Clicking one opens that panel. */} + {addressTags(address, caps).map(tag => + + setEditing(address)}> + {tag.label} + + )} + {/* The note takes a line of its own rather than the corner of + this one. Its basis is a full-length IPv6 with its prefix — when + that much room is not left beside the badges, the whole note wraps + down instead of being squeezed into the gap, which is what put + `fd07:…:fe4` and `0:be17` on two lines with the digits broken + between them. It shrank that far because the basis was 80px and + `anywhere` let it: a tag or two added to the row is all it took to + use the space up. The basis is the whole of the rule — a floor + beside it was tried and is what overflowed the card on a phone, + where the column is narrower than any floor worth setting, and it + bought nothing: an item only shares a line when its basis fitted + there, so on a line of its own it is already as wide as it can be. + `anywhere` stays as the last resort, so a phone wraps an address + rather than pushing it past the edge. */} {detail ? + style={{ flex: '1 1 320px', minWidth: 0, overflowWrap: 'anywhere' }}> {detail} : null} @@ -256,7 +344,7 @@ export default function AddressList({ service }: { service: Service }) { } /** - * Where the decrypted traffic of every TLS service can be watched. + * Where the decrypted traffic of every TLS and QUIC service can be watched. * * The engine decrypts inside the process that filters, so the plaintext never becomes * packets on any interface — which is exactly what removed the two loopback ports a TLS @@ -269,12 +357,19 @@ export default function AddressList({ service }: { service: Service }) { * as the TCP stream they were, because the stream that actually crossed the wire was * encrypted. Wireshark follows it normally; it is not the wire, and the interface says so. * + * A QUIC service is carried there too, one TCP stream per QUIC stream, which costs one + * more piece of invention: the streams of one connection share its four-tuple, so each is + * given a synthetic client port to be told apart by. An operator reading a port out of + * that capture has to know it names a stream and not a socket, which is why the panel + * says so rather than leaving it to be discovered. + * * Collapsed by default: it is a thing you go looking for while debugging a filter that is * not matching, not something that should sit in the way the rest of the time. */ function PlaintextCapture({ service }: { service: Service }) { const [open, setOpen] = useState(false) if (!decrypts(service)) return null + const quic = service.proto === L4.QUIC return @@ -286,7 +381,7 @@ function PlaintextCapture({ service }: { service: Service }) { - Every TLS service's plaintext is written to firegex0, and + Every decrypted service's traffic is written to firegex0, and nothing else is. Point Wireshark or tcpdump at it on the host running firegex — the container shares its network namespace, so the interface is there — and you get the decrypted traffic of the whole instance, both @@ -305,6 +400,18 @@ function PlaintextCapture({ service }: { service: Service }) { as sensitive as the private key that would have produced it, so treat the file the same way. + {quic ? <> + + + On QUIC each stream arrives as a TCP stream of its own, and over + HTTP/3 that is one per request — what you see is the same HTTP/1.1 + the filters were shown, because what crossed the wire was a + compressed header block inside an encrypted packet. The client + port is invented: the streams of one connection share the + client's real port, so each is given one of its own to be told apart + by. It names a stream, not a socket — the address beside it is real. + + : null} } diff --git a/frontend/src/components/Services/AddressOptions.tsx b/frontend/src/components/Services/AddressOptions.tsx new file mode 100644 index 0000000..a6c8826 --- /dev/null +++ b/frontend/src/components/Services/AddressOptions.tsx @@ -0,0 +1,155 @@ +import { Box, NumberInput, SegmentedControl, Stack, Text } from '@mantine/core'; +import PortAndInterface from '../PortAndInterface'; +import { decrypts, L4, Transport, Upstream } from './utils'; + +/** + * Everything an address can say beyond where it is, written once. + * + * Two places ask for it — the rows of the creation form, behind each row's ⚙, and the + * modal on the service's own page — and they used to ask in two sets of words. The same + * setting described twice reads as two settings, which is the whole reason this file + * exists; the capability rules below were written twice as well, so the form could offer + * something the backend then refused. + * + * The wording rule here: **say what firegex will do**, not what the field is called. + * "The service is on port" and "The service behind speaks" were both true and both left + * the operator to work out the effect for themselves. + */ + +/** Which of these options exist at all for the service being configured. */ +export function addressCapabilities(proto: string, transport: string) { + const isExternal = transport === Transport.EXTERNAL + return { + isExternal, + /** `http` is the one protocol whose addresses are not all the same. */ + isHttp: proto === L4.HTTP, + //: Sending traffic somewhere other than where it arrived takes something that + // *dials*, and that is the layer rather than the protocol: the proxy opens the + // connection to the service, NFQUEUE opens nothing, and the hand-off layer + // leaves the question to the proxy you run yourself. + canPublish: transport === Transport.PROXY, + //: Only a service firegex decrypts has a leg of its own towards the service. On + // a cleartext one it terminates nothing, so there is nothing to put back. + canChooseUpstream: decrypts({ proto }) && transport === Transport.PROXY, + } +} + +/** What clients speak at one address of an HTTPS service. Shared so the row, the modal + * and the badge on the service page cannot come to call them three different things. */ +export const EDGE_OPTIONS = [ + { label: 'Clear', value: L4.TCP }, + { label: 'TLS', value: L4.TLS }, + { label: 'HTTP/3', value: L4.QUIC }, +] + +export const edgeHint = (edge: string) => + edge === L4.QUIC + ? "A UDP port speaking HTTP/3." + : edge === L4.TLS + ? "Encrypted only: a client that arrives in the clear here is refused." + : "Cleartext HTTP/1.1 and HTTP/2 — and a client that brings TLS anyway is taken too." + +export const upstreamHint = (upstream: string) => + upstream === Upstream.TCP + ? "Firegex decrypts and sends plain HTTP/1.1. Only firegex holds a certificate." + : upstream === Upstream.TLS + ? "Firegex encrypts with TLS, even for a client that arrived over HTTP/3." + : "Firegex sends what the client sent, encrypted again — so the service holds a certificate of its own." + +type Extras = { + port: number, + target_port?: number | string | null, + upstream?: string | null, + proxy_ip?: string | null, + proxy_port?: number | null, +} + +/** + * What is switched on for one address, in a word each. + * + * Shown under the address in the form and beside it on the service page — the same + * labels in both, because this is where an operator comes back to check what they set, + * and finding it named differently is a reason to wonder whether it is the same thing. + */ +export function addressTags(values: Extras, caps: ReturnType) { + const tags: { label: string, hint: string }[] = [] + const target = Number(values.target_port) + if (caps.canPublish && target && target !== values.port) tags.push({ + label: `→ :${target}`, + hint: `Clients arrive on :${values.port} — firegex dials the service on :${target}, which is where it listens.`, + }) + if (caps.canChooseUpstream && values.upstream === Upstream.TCP) tags.push({ + label: "plaintext to the service", + hint: upstreamHint(Upstream.TCP), + }) + if (caps.canChooseUpstream && values.upstream === Upstream.TLS) tags.push({ + label: "TLS to the service", + hint: upstreamHint(Upstream.TLS), + }) + if (caps.isExternal && values.proxy_port) tags.push({ + label: `→ ${values.proxy_ip ?? "127.0.0.1"}:${values.proxy_port}`, + hint: "Traffic for this address is handed to your own proxy there, and put back on the way out.", + }) + return tags +} + +/** + * The controls themselves, over whichever form is asking. + * + * `field` is what turns `target_port` into the name this form knows it by — bare in the + * modal, `addresses.3.target_port` in a row of the creation form. + */ +export default function AddressOptions({ form, field, values, proto, transport }: { + form: any, + field: (name: string) => string, + /** This address's own values, which is what the descriptions read back. */ + values: Extras, + proto: string, + transport: string, +}) { + const caps = addressCapabilities(proto, transport) + const port = Number(values.port) + const target = Number(values.target_port) + const upstream = String(values.upstream ?? Upstream.SAME) + + return + {/* The label names the fact, and the description says what firegex does with + it, because an effect alone leaves the direction open when both ends are + ports. "Send it to another port" was read as "expose it on another port" and + filled in the other way round: the service's own port typed into the address, + and the port to publish on typed here — a service intercepted where it already + listens and published nowhere. Both descriptions name **both** ends for that + reason, and the empty one says where a second way in actually comes from. */} + {caps.canPublish ? + : null} + {caps.canChooseUpstream ? + Send to the service + {upstreamHint(upstream)} + + : null} + {caps.isExternal ? + + + Its own port: the way back is recognised by it, so two addresses cannot + share one. + + : null} + +} diff --git a/frontend/src/components/Services/AddressRow.tsx b/frontend/src/components/Services/AddressRow.tsx new file mode 100644 index 0000000..8c38054 --- /dev/null +++ b/frontend/src/components/Services/AddressRow.tsx @@ -0,0 +1,94 @@ +import { ActionIcon, Badge, Box, Group, Popover, SegmentedControl, Space, Text, Tooltip } from '@mantine/core'; +import { useState } from 'react'; +import { BsTrashFill } from 'react-icons/bs'; +import { IoSettingsSharp } from 'react-icons/io5'; +import PortAndInterface from '../PortAndInterface'; +import AddressOptions, { addressCapabilities, addressTags, edgeHint, EDGE_OPTIONS } from './AddressOptions'; + +/** + * One address being filled in, and everything that can be said about it. + * + * The row itself stays what it was — where the service answers, and on an HTTPS service + * what is spoken there — because that is what every address needs and most addresses need + * only that. Everything else is **behind one button**: traffic sent on to another port, + * what firegex speaks to the service, the endpoint of somebody else's proxy. Those were + * inline once, each of them, and a list of three addresses became a screenful of fields + * for cases most services never have. + * + * What is set is said back on the row as **tags**, and a tag opens the panel it came + * from: folded is not hidden, and an option an operator cannot see they set is an option + * that will surprise them at the worst moment. + * + * Which options exist at all, and how they are worded, live in `AddressOptions` — the + * modal on the service's own page asks for the same things and has to ask in the same + * words. + */ +export default function AddressRow({ form, index, proto, transport, canRemove }: { + form: any, + index: number, + /** What the service speaks, which decides which options exist at all. */ + proto: string, + transport: string, + canRemove: boolean, +}) { + const [open, setOpen] = useState(false) + const address = form.values.addresses[index] + const caps = addressCapabilities(proto, transport) + const tags = addressTags(address, caps) + const anything = caps.canPublish || caps.canChooseUpstream || caps.isExternal + + return + + + + + {/* What is spoken here — on the row rather than in the panel, because on an + HTTPS service it is not an extra: it is what this address *is*. */} + {caps.isHttp ? + + : null} + {anything ? + + + setOpen(o => !o)}> + + + + + + + Only this address. Everything here is off unless you set it. + + `addresses.${index}.${name}`} + proto={proto} transport={transport} /> + + : null} + form.removeListItem('addresses', index)}> + + + + {/* What was set, under the address it was set on. Clicking one opens the panel + it came from — a tag that says a thing is on and cannot take you to it is a + label, and the operator is left hunting for where they set it. */} + {tags.length > 0 ? + {tags.map(tag => + setOpen(true)}> + {tag.label} + + )} + : null} + + +} diff --git a/frontend/src/components/Services/LayerChoice.tsx b/frontend/src/components/Services/LayerChoice.tsx index de1ffcf..2154495 100644 --- a/frontend/src/components/Services/LayerChoice.tsx +++ b/frontend/src/components/Services/LayerChoice.tsx @@ -23,11 +23,17 @@ type Layer = { /** Folded in when the service speaks UDP, where the trade moves. */ udpGood?: string[], udpBad?: string[], + /** And when it speaks QUIC, where it moves again and further. */ + quicGood?: string[], + quicBad?: string[], + /** And when it speaks HTTP, which is every version of it at once. */ + httpGood?: string[], + httpBad?: string[], } const LAYERS: Record = { [Transport.PROXY]: { - line: "Terminates and reopens the connection — the faster of the two, and the only one that can decrypt TLS.", + line: "Terminates and reopens the connection — the faster of the two, and the only one that can decrypt TLS or QUIC.", good: [ "Carries bulk traffic 2-5× faster, and scales better with threads", "Python filters rewrite a payload exactly, at any length", @@ -43,6 +49,29 @@ const LAYERS: Record = { "One relay socket per address — no per-datagram state to recover", "Your service still sees the real client address (transparent IP spoofing)", ], + quicGood: [ + "The only layer that can see inside QUIC at all", + "Every stream is filtered on its own, with its own state", + "One endpoint per address, and the client address still reaches the service", + "Each stream reaches firegex0 as its own TCP stream, HTTP/3 as HTTP/1.1", + ], + quicBad: [ + "No 0-RTT: early data is replayable, and a refusal cannot be taken back", + "No datagrams on HTTP/3, and no WebTransport", + "A refused stream closes the connection carrying it", + ], + httpGood: [ + "HTTP/1.1, HTTP/2 and HTTP/3 through one chain, with one certificate", + "One filter file and one pattern cover all three: every version is shown as HTTP/1.1", + "Each address picks its own edge — cleartext, TLS, or HTTP/3", + "gRPC is filtered, request line and headers included, trailers included", + "TLS is decided per connection, so one port may serve both", + ], + httpBad: [ + "A certificate is needed even for the cleartext addresses beside it", + "Every HTTP/2 stream is its own connection to a filter, unlike keep-alive on HTTP/1.1", + "Server push and CONNECT tunnels are refused rather than rendered", + ], }, [Transport.NFQUEUE]: { line: "Verdicts on the real packets — nothing in the path, and the kernel keeps forwarding if a filter dies.", @@ -108,9 +137,13 @@ export default function LayerChoice({ value, onChange, proto }: { const [open, setOpen] = useState(false) const layer = LAYERS[value] ?? LAYERS[Transport.PROXY] const udp = proto === L4.UDP + const quic = proto === L4.QUIC + const http = proto === L4.HTTP const unavailable = Object.keys(LABELS).filter(t => !carries(t, proto)) - const good = [...layer.good, ...(udp ? layer.udpGood ?? [] : [])] - const bad = [...layer.bad, ...(udp ? layer.udpBad ?? [] : [])] + const good = [...layer.good, ...(udp ? layer.udpGood ?? [] : []), + ...(quic ? layer.quicGood ?? [] : []), ...(http ? layer.httpGood ?? [] : [])] + const bad = [...layer.bad, ...(udp ? layer.udpBad ?? [] : []), + ...(quic ? layer.quicBad ?? [] : []), ...(http ? layer.httpBad ?? [] : [])] return <> @@ -139,7 +172,11 @@ export default function LayerChoice({ value, onChange, proto }: { {LABELS[value]} - {udp ? UDP : null} + {udp || quic || http + ? + {proto.toUpperCase()} + + : null} {layer.line} @@ -162,11 +199,12 @@ export default function LayerChoice({ value, onChange, proto }: { /> {layer.line} + {/* One line. What is worth reading here is which layers are out and why, not the + whole argument for it — that is what the documentation is for. */} {unavailable.length > 0 ? - {unavailable.map(t => LABELS[t]).join(" and ")} - {unavailable.length > 1 ? " are" : " is"} not available for a service that speaks - TLS: decrypting means terminating the connection, and only this layer does that. - Set the protocol to TCP to use {unavailable.length > 1 ? "them" : "it"}. + {unavailable.map(t => LABELS[t]).join(" and ")} cannot carry encrypted + traffic: decrypting means terminating the connection, and only this layer + does that. : null} } diff --git a/frontend/src/components/Services/ProtocolChoice.tsx b/frontend/src/components/Services/ProtocolChoice.tsx new file mode 100644 index 0000000..93606aa --- /dev/null +++ b/frontend/src/components/Services/ProtocolChoice.tsx @@ -0,0 +1,233 @@ +import { ActionIcon, Badge, Box, Divider, Group, Popover, SegmentedControl, Space, Stack, Text, Tooltip } from '@mantine/core'; +import { useState } from 'react'; +import { IoInformationCircleOutline } from 'react-icons/io5'; +import { TbShieldLock } from 'react-icons/tb'; +import { carries, decrypts, L4, protoLabel } from './utils'; + +/** + * Choosing what the service speaks — two questions, not one list of five answers. + * + * The list of five was reported as confusing and it deserved to be. `TCP`, `UDP`, `TLS`, + * `QUIC` and `HTTP` side by side look like five alternatives on one axis, and they are + * not: an operator with a plain HTTP/1.1 service on `:80` had no way to tell whether + * `TCP` or `HTTP` was meant for them — both are true sentences about their service — and + * picking `HTTP` then asked for a certificate for something with nothing encrypted about + * it. + * + * So the first question is the one that actually divides them, and it is the one the + * certificate hangs off: **is the traffic encrypted?** The second question is then asked + * inside that answer, and only ever offers protocols of that kind — + * + * - in the clear: `TCP` · `UDP` + * - encrypted: `TLS` · `QUIC` · `HTTPS` + * + * Nothing is derived and nothing is hidden: every option is the protocol under its own + * name, which is what the service is stored as and what the rest of the interface calls + * it. An earlier attempt asked "on what transport?" and computed the protocol from the + * pair — the names then appeared only in a badge under the controls, which reads as the + * form having decided something on your behalf. + * + * **The text stays short on purpose.** What an operator needs here is which option is + * theirs, which is one clause each; what a protocol costs and what filters can be + * attached to it is documentation, and it is a click away in the popover rather than + * three paragraphs nobody reads twice. + */ + +type Protocol = { + /** One clause. The reason this option is yours. */ + line: string, + /** The service this is the answer for. */ + pick: string[], + /** What a filter is handed. */ + sees: string[], + /** What choosing it costs you in setup. */ + needs: string[], +} + +const PROTOCOLS: Record = { + [L4.TCP]: { + line: "A stream in the clear — HTTP/1.1 and HTTP/2 included.", + pick: [ + "Anything unencrypted on TCP: a web service on :80, a game protocol, a shell", + "The default, and the right answer unless the traffic is encrypted", + ], + sees: [ + "The bytes as they travel", + "A filter asking for an HttpRequest gets one — HTTP/1.1 parsed, and HTTP/2 in the clear recognised from its preface and rendered as the same HTTP/1.1", + ], + needs: ["Nothing but the addresses to protect"], + }, + [L4.UDP]: { + line: "Datagrams in the clear, one filter state per client flow.", + pick: ["Anything unencrypted on UDP: a game server, DNS, a custom protocol"], + sees: [ + "One datagram at a time, both directions", + "RawPacket only: every other model is built on a stream, and a datagram is not one", + ], + needs: ["Nothing but the addresses to protect"], + }, + [L4.TLS]: { + line: "TCP under TLS, decrypted here.", + pick: ["Everything reaching this port is encrypted — an HTTPS service, or any TLS protocol"], + sees: [ + "The decrypted stream, and the same models a cleartext service gives", + "The protocol the two ends agreed on: ALPN is mirrored, so h2 is rendered as HTTP/1.1 too", + ], + needs: [ + "A certificate and its key, in PEM", + "The proxy layer: decrypting means terminating the connection", + ], + }, + [L4.QUIC]: { + line: "UDP with TLS 1.3 and streams inside it, decrypted here.", + pick: ["A QUIC service — in practice HTTP/3, which is what the engine offers by default"], + sees: [ + "One stream at a time, with its own filter state from beginning to end", + "HTTP/3 rendered as HTTP/1.1, so one pattern covers it and the TCP service beside it", + ], + needs: [ + "A certificate and its key, in PEM", + "The proxy layer, with no alternative: past its first packet QUIC encrypts the frames and the stream boundaries too", + ], + }, + [L4.HTTP]: { + line: "Every version of HTTP through one chain — HTTP/1.1 and HTTP/2 on its TCP addresses, encrypted or in the clear, HTTP/3 on its UDP ones.", + pick: [ + "A web service reached more than one way: :80, :443 and :443 over UDP, which would otherwise be two or three services with the chain copied between them by hand", + "Its cleartext ports come with it — whether a connection is TLS is decided per connection, from what the client sent", + ], + sees: [ + "Every version as the same HTTP/1.1, so one pattern and one Python filter cover all three", + "gRPC included: the request line, the headers and the trailer section", + ], + needs: [ + "A certificate and its key, in PEM — including for the cleartext addresses beside them", + "The proxy layer, and each address saying whether it is reached over TCP or UDP", + ], + }, +} + +/** The first question. It is the one the certificate hangs off, and nothing else. */ +const CLEAR = "clear" +const ENCRYPTED = "encrypted" + +/** The second question, asked inside the first one's answer. */ +const OFFERED: Record = { + [CLEAR]: [L4.TCP, L4.UDP], + [ENCRYPTED]: [L4.TLS, L4.QUIC, L4.HTTP], +} + +const edgeOf = (proto: string) => decrypts({ proto }) ? ENCRYPTED : CLEAR + +/** + * The same service, said the other way. + * + * Switching the first answer has to land on something, and the honest something is the + * counterpart on the same transport: a TCP service that turns out to be encrypted is TLS, + * a UDP one is QUIC, and back again. `http` is several transports at once, so it comes + * back as the one its first address would be. + */ +const acrossEdge = (proto: string, edge: string) => edge === ENCRYPTED + ? (proto === L4.UDP ? L4.QUIC : L4.TLS) + : (proto === L4.QUIC ? L4.UDP : L4.TCP) + +function Section({ title, items }: { title: string, items: string[] }) { + return + {title} + {items.map(text => + {text} + )} + +} + +/** One question: its label, and the control that answers it. */ +function Question({ label, children }: { label: string, children: React.ReactNode }) { + return + {label} + {children} + +} + +export default function ProtocolChoice({ value, transport, onChange }: { + value: string, + /** The layer chosen below: it cannot carry anything that has to be decrypted. */ + transport: string, + onChange: (value: string) => void, +}) { + const [open, setOpen] = useState(false) + const proto = PROTOCOLS[value] ?? PROTOCOLS[L4.TCP] + const edge = edgeOf(value) + //: One rule, asked once: a layer that does not terminate cannot decrypt. + const canDecrypt = carries(transport, L4.TLS) + + return <> + + + What the service speaks + What is on the wire, and whether firegex has to decrypt it. + + + + + setOpen(o => !o)}> + + + + + + + : undefined}> + {protoLabel(value)} + + + {proto.line} + + +
+
+
+ + + + + + + {/* Disabled, not removed, the same way the layer picker below says no and from + the same `carries` rule: an option that vanishes leaves nothing to ask + about. The lock answers, before the click, which half of this form will + want a certificate. */} + + onChange(acrossEdge(value, e))} + data={[ + { value: CLEAR, label: "In the clear" }, + { + value: ENCRYPTED, + disabled: !canDecrypt, + label: + + Encrypted + , + }, + ]} + /> + + + + ({ value: p, label: protoLabel(p) }))} + /> + + + + + {proto.line} + {!canDecrypt ? + Encrypted traffic needs the proxy layer: decrypting means terminating the + connection, and the layer chosen below does not. + : null} + +} diff --git a/frontend/src/components/Services/ServiceRow.tsx b/frontend/src/components/Services/ServiceRow.tsx index cd8af82..1d9fb34 100644 --- a/frontend/src/components/Services/ServiceRow.tsx +++ b/frontend/src/components/Services/ServiceRow.tsx @@ -8,7 +8,7 @@ import { TbHexagon, TbShieldLock } from 'react-icons/tb'; import { bareAddress, errorNotify, isMediumScreen, okNotify } from '../../js/utils'; import YesNoModal from '../YesNoModal'; import AddEditService from './AddEditService'; -import { Address, decrypts, L4, Service, serviceQueryKey, services, Transport } from './utils'; +import { Address, decrypts, L4, protoLabel, Service, serviceQueryKey, services, Transport, Upstream } from './utils'; /** What the network layer means, in one line, where the operator is choosing. */ export const transportLabel = (transport: string) => ({ @@ -29,6 +29,10 @@ export const transportSummary = (transport: string, proto: string) => { return "Firegex only steers this traffic to a proxy you run; nothing here inspects it, so no filter can be attached." if (transport === Transport.NFQUEUE) return "Packets are inspected and a verdict handed back; nothing is terminated. Fully transparent, and the kernel keeps forwarding if a filter dies — at the cost of a userspace round trip per packet, userspace reassembly, and a process per filter." + if (proto === L4.HTTP) + return "Every version of HTTP through one chain: HTTP/1.1 and HTTP/2 on the TCP addresses — in the clear or under TLS, whichever each client opens with — and HTTP/3 on the UDP ones. All three are shown to the filters as the same HTTP/1.1, so one pattern and one Python filter cover them; without that, HTTP/2 and HTTP/3 put the request line and the headers in a compression format no filter could read. The service still sees the real client address." + if (proto === L4.QUIC) + return "QUIC is terminated here: the engine decrypts it, and each stream inside is filtered on its own with its own state. It has to be — past the first packet QUIC encrypts its frames and its stream boundaries as well as the payload, so no other layer can see anything. One endpoint per address, and the service still sees the real client address." return proto === L4.UDP ? "Each address is relayed by a dedicated socket. Filters keep per-flow state, source IP transparency is preserved, and a new address is relayed without restarting the service." : "The connection is terminated and reopened, so the kernel reassembles and the chain has no length limit. It also carries bulk traffic several times faster than NFQUEUE, which pays a userspace round trip per packet; what it costs is fail-open being rebuilt in userspace rather than guaranteed by the kernel. The service still sees the real client address." @@ -137,16 +141,28 @@ export default function ServiceRow({ service, onClick }: { service: Service, onC {transportLabel(service.transport)} - {decrypts(service) ? + {/* The plaintext leg is worth saying here and not only in + the form: a service whose backend speaks in the clear is one + firegex is *adding* encryption to, which changes what stopping + firegex does to it. */} + {decrypts(service) ? a.upstream === Upstream.TCP) + ? ". Behind at least one of its addresses the service answers in the clear, so firegex is what encrypts it." + : "")}> }>TLS + leftSection={}> + {protoLabel(service.proto)} + : null} `${bareAddress(a.ip_int)}:${a.port}`).join(", ")}> - {addressSummary(service.addresses)} ON {service.proto.toUpperCase()} + {addressSummary(service.addresses)} ON {protoLabel(service.proto)} diff --git a/frontend/src/components/Services/utils.ts b/frontend/src/components/Services/utils.ts index f770ab1..5a104b5 100644 --- a/frontend/src/components/Services/utils.ts +++ b/frontend/src/components/Services/utils.ts @@ -50,10 +50,81 @@ export enum L4 { UDP = "udp", /** TLS over TCP, decrypted by the engine. Only on the proxy layer. */ TLS = "tls", + /** + * QUIC: UDP on the wire, TLS 1.3 inside it, streams inside that. Decrypted by the + * engine, and only on the proxy layer — with less choice about it than TLS has. + * Past the first packet QUIC encrypts its frames and its stream boundaries too, so + * a layer that forwards packets has nothing to show a filter at all. + */ + QUIC = "quic", + /** + * Every version of HTTP at once: HTTP/1.1 and HTTP/2 on TCP — in the clear or under + * TLS, whichever each client opens with — and HTTP/3 on UDP beside them. + * + * The odd one out, because it says what the *service* is rather than what is on the + * wire, and lets each address say how it is reached. It exists because the + * alternative was two or three services with the same filter chain copied between + * them by hand, and a chain kept in step by hand is one that stops protecting one of + * them silently. Every version is shown to the filters as the same HTTP/1.1, so one + * pattern and one Python filter cover all of them. + */ + HTTP = "http", +} + +/** + * What a protocol is called in the interface. + * + * One place, because the form, the service row and its page all have to call a service + * the same thing: being offered `HTTPS` and then shown `HTTP` on the row is the kind of + * difference an operator reasonably reads as two different settings. + * + * Only `http` is not simply its own name in capitals. Stored and served as `http` — it + * *is* every version of HTTP — it is shown as **HTTPS** because of where it is chosen: + * among the protocols firegex has to decrypt, beside TLS and QUIC, where a thing labelled + * "HTTP" reads as the cleartext web protocol, which is the confusion this whole picker + * was rebuilt around. What it adds to TLS is the cleartext ports carried beside it, and + * that belongs in the sentence under the option rather than in its name. + */ +export const protoLabel = (proto: string) => proto === L4.HTTP ? "HTTPS" : proto.toUpperCase() + +/** + * What the service *behind one address* speaks. + * + * It used to be assumed rather than asked: whatever the client spoke, the service was + * taken to speak too — a TLS connection re-encrypted on the way out, a QUIC one + * re-encoded as QUIC. That is right when firegex is carrying somebody else's encryption + * and wrong in the case an operator most often has, which is a service that speaks + * neither and never will. + * + * It belongs to the address because that is where it has an answer: one daemon reached + * over TLS on one port and in the clear on another is re-encrypted for the first and + * handed the plaintext for the second. + */ +export enum Upstream { + /** It speaks what arrived. The default, and what every address used to do. */ + SAME = "same", + /** It answers in the clear, and firegex is what adds the encryption in front of it. */ + TCP = "tcp", + /** It speaks TLS, whatever the client used to get here. */ + TLS = "tls", } /** Whether a service's traffic is decrypted before the filters see it. */ -export const decrypts = (service: { proto: string }) => service.proto === L4.TLS +export const decrypts = (service: { proto: string }) => + service.proto === L4.TLS || service.proto === L4.QUIC || service.proto === L4.HTTP + +/** + * What an address of this service may be reached over. + * + * One answer for every protocol but `http`, which is the one that means "each address + * says" — and says it in the same words a service speaks: in the clear, under TLS, or + * over QUIC. + */ +export const addressEdges = (proto: string): string[] => + proto === L4.HTTP ? [L4.TCP, L4.TLS, L4.QUIC] : [proto] + +/** Whether an address of this service has to say which transport it is on. */ +export const addressPicksProto = (proto: string) => proto === L4.HTTP /** * Whether a network layer can carry a protocol at all. @@ -64,11 +135,12 @@ export const decrypts = (service: { proto: string }) => service.proto === L4.TLS * the backend refuses — which it does refuse, in `Transport.check()`, and being told no * after filling in a form is a worse way to learn it. * - * Only TLS constrains anything today: decrypting means terminating the connection, which - * is what the proxy layer does and the other two deliberately do not. + * Only the two protocols that have to be decrypted constrain anything: decrypting means + * terminating the connection, which is what the proxy layer does and the other two + * deliberately do not. */ export const carries = (transport: string, proto: string): boolean => - proto !== L4.TLS || transport === Transport.PROXY + !decrypts({ proto }) || transport === Transport.PROXY @@ -91,7 +163,17 @@ export type Address = { service_id: string, ip_int: string, port: number, + /** What the kernel matches: this is `edge` with the hat taken off. */ proto: string, + /** What clients speak here — `tcp`, `tls`, `udp` or `quic`. */ + edge: string, + /** What the service behind *this address* speaks. See {@link Upstream}. */ + upstream: string, + /** + * Where the service really is, when this address is a publication rather than the + * service itself. Null is the transparent case: firegex dials what the client dialled. + */ + target_port: number | null, /** `external` only: where your own proxy listens for this address. */ proxy_ip: string | null, proxy_port: number | null, @@ -100,6 +182,12 @@ export type Address = { export type AddressForm = { ip_int: string, port: number, + /** `http` only: what is spoken at this address. */ + edge?: string | null, + /** Where the service is, when it is not on this port. */ + target_port?: number | null, + /** What the service behind this address speaks. */ + upstream?: string | null, proxy_ip?: string | null, proxy_port?: number | null, } diff --git a/frontend/src/js/demo.ts b/frontend/src/js/demo.ts index 8cf608a..de05451 100644 --- a/frontend/src/js/demo.ts +++ b/frontend/src/js/demo.ts @@ -113,8 +113,11 @@ const DEMO_PYFILTER_API = { // ------------------------------------------------------------------ seed state -const svcShop = uuid(), svcScore = uuid(), svcVault = uuid() +const svcShop = uuid(), svcScore = uuid(), svcVault = uuid(), svcArena = uuid() const fltShopRegex = uuid(), fltShopPy = uuid(), fltScoreRegex = uuid(), fltVaultRegex = uuid() +const fltArenaRegex = uuid() +const svcWeb = uuid() +const fltWebPy = uuid() const svcHijack = uuid() /** The demo's services have been up for a while; a fresh one starts filtering now. */ @@ -132,6 +135,16 @@ const state = { { service_id: svcShop, name: "shop-api", status: "active", proto: "tcp", transport: "proxy", fail_open: true, max_connections: 0, over_limit_forwards: false, first_byte_timeout: 0, over_limit_hits: 0, over_limit_first: null, over_limit_last: null, filtering_since: demoStart }, { service_id: svcScore, name: "scoreboard", status: "active", proto: "tcp", transport: "nfqueue", fail_open: true, max_connections: 512, over_limit_forwards: false, first_byte_timeout: 0, over_limit_hits: 1847, over_limit_first: demoStart, over_limit_last: demoStart + 900, filtering_since: demoStart }, { service_id: svcVault, name: "vault", status: "stop", proto: "tls", transport: "proxy", fail_open: true, max_connections: 0, over_limit_forwards: false, first_byte_timeout: 0, over_limit_hits: 0, over_limit_first: null, over_limit_last: null, filtering_since: null }, + // QUIC: UDP on the wire, terminated here because nothing else can see inside it. + // Its filters are written against HTTP because its traffic is HTTP/3, which the + // engine shows the chain as the HTTP/1.1 it would have been. + { service_id: svcArena, name: "arena-h3", status: "active", proto: "quic", transport: "proxy", fail_open: true, max_connections: 0, over_limit_forwards: false, first_byte_timeout: 0, over_limit_hits: 0, over_limit_first: null, over_limit_last: null, filtering_since: demoStart }, + // One service for every version of HTTP: HTTP/1.1 and HTTP/2 on the TCP + // addresses, in the clear on one port and under TLS on the other, and HTTP/3 on + // the UDP one beside them. One certificate and one chain — which is the whole + // point, because covering the three used to take two or three services with the + // same filter copied between them by hand. + { service_id: svcWeb, name: "shop-web", status: "active", proto: "http", transport: "proxy", fail_open: true, max_connections: 0, over_limit_forwards: false, first_byte_timeout: 0, over_limit_hits: 0, over_limit_first: null, over_limit_last: null, filtering_since: demoStart }, // Handed to a proxy the operator wrote themselves: firegex steers, nothing here inspects. { service_id: svcHijack, name: "legacy-ftp", status: "active", proto: "tcp", transport: "external", fail_open: true, max_connections: 0, over_limit_forwards: false, first_byte_timeout: 0, over_limit_hits: 0, over_limit_first: null, over_limit_last: null, filtering_since: demoStart }, ] as Json[], @@ -140,12 +153,20 @@ const state = { // `wg0` by name, the other half of the same idea: an address somebody else hands // out is still one place this service answers. addresses: [ - { address_id: uuid(), service_id: svcShop, ip_int: "10.60.3.1/32", port: 9000, proto: "tcp", proxy_ip: null, proxy_port: null }, - { address_id: uuid(), service_id: svcShop, ip_int: "fd66:666:3::1/128", port: 9000, proto: "tcp", proxy_ip: null, proxy_port: null }, - { address_id: uuid(), service_id: svcShop, ip_int: "wg0", port: 9000, proto: "tcp", proxy_ip: null, proxy_port: null }, - { address_id: uuid(), service_id: svcScore, ip_int: "10.60.0.0/16", port: 8080, proto: "tcp", proxy_ip: null, proxy_port: null }, - { address_id: uuid(), service_id: svcVault, ip_int: "10.60.3.1", port: 8443, proto: "tcp", proxy_ip: null, proxy_port: null }, - { address_id: uuid(), service_id: svcHijack, ip_int: "10.60.3.1", port: 21, proto: "tcp", proxy_ip: "127.0.0.1", proxy_port: 12021 }, + { address_id: uuid(), service_id: svcShop, ip_int: "10.60.3.1/32", port: 9000, proto: "tcp", edge: "tcp", target_port: null, upstream: "same", proxy_ip: null, proxy_port: null }, + { address_id: uuid(), service_id: svcShop, ip_int: "fd66:666:3::1/128", port: 9000, proto: "tcp", edge: "tcp", target_port: null, upstream: "same", proxy_ip: null, proxy_port: null }, + { address_id: uuid(), service_id: svcShop, ip_int: "wg0", port: 9000, proto: "tcp", edge: "tcp", target_port: null, upstream: "same", proxy_ip: null, proxy_port: null }, + { address_id: uuid(), service_id: svcScore, ip_int: "10.60.0.0/16", port: 8080, proto: "tcp", edge: "tcp", target_port: null, upstream: "same", proxy_ip: null, proxy_port: null }, + { address_id: uuid(), service_id: svcVault, ip_int: "10.60.3.1", port: 8443, proto: "tcp", edge: "tls", target_port: null, upstream: "same", proxy_ip: null, proxy_port: null }, + // `udp`, because that is what the kernel matches for a QUIC service. + { address_id: uuid(), service_id: svcArena, ip_int: "10.60.3.1", port: 4433, proto: "udp", edge: "quic", target_port: null, upstream: "same", proxy_ip: null, proxy_port: null }, + // The three edges of one HTTP service, and what they say: in the clear on :80, + // TLS on :443, HTTP/3 on :443/udp. The last two are *publications* — the service + // itself only ever answered HTTP/1.1 on :80, and has not moved. + { address_id: uuid(), service_id: svcWeb, ip_int: "10.60.3.1", port: 80, proto: "tcp", edge: "tcp", target_port: null, upstream: "same", proxy_ip: null, proxy_port: null }, + { address_id: uuid(), service_id: svcWeb, ip_int: "10.60.3.1", port: 443, proto: "tcp", edge: "tls", target_port: 80, upstream: "tcp", proxy_ip: null, proxy_port: null }, + { address_id: uuid(), service_id: svcWeb, ip_int: "10.60.3.1", port: 443, proto: "udp", edge: "quic", target_port: 80, upstream: "tcp", proxy_ip: null, proxy_port: null }, + { address_id: uuid(), service_id: svcHijack, ip_int: "10.60.3.1", port: 21, proto: "tcp", edge: "tcp", target_port: null, upstream: "same", proxy_ip: "127.0.0.1", proxy_port: 12021 }, ] as Json[], filters: [ // shop-api shows what only the proxy layer can do: two kinds, in an order. @@ -154,6 +175,11 @@ const state = { { filter_id: fltShopPy, service_id: svcShop, position: 1, kind: "pyfilter", proto: "http", name: "python", active: true, blocked: 641 }, { filter_id: fltScoreRegex, service_id: svcScore, position: 0, kind: "regex", proto: "tcp", name: "patterns", active: true, blocked: 1204 }, { filter_id: fltVaultRegex, service_id: svcVault, position: 0, kind: "regex", proto: "tcp", name: "patterns", active: true, blocked: 0 }, + { filter_id: fltArenaRegex, service_id: svcArena, position: 0, kind: "regex", proto: "tcp", name: "patterns", active: true, blocked: 52 }, + // The same file that would run on a plain HTTP/1.1 service, unchanged — which is + // the point of an `http` service: HTTP/1.1, HTTP/2 and HTTP/3 are all shown to it + // as the same HTTP/1.1, so one filter covers the three edges below. + { filter_id: fltWebPy, service_id: svcWeb, position: 0, kind: "pyfilter", proto: "http", name: "python", active: true, blocked: 1190 }, ] as Json[], regexes: [ { regex_id: uuid(), filter_id: fltShopRegex, regex: b64("\\.\\./"), mode: "C", case_sensitive: true, active: true, blocked: 341 }, @@ -163,17 +189,21 @@ const state = { { regex_id: uuid(), filter_id: fltScoreRegex, regex: b64("