diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b46e8a..33d5bba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,8 +35,31 @@ jobs: - name: Run tests run: cd tests && sudo ./run_tests.sh + # The proxy engine's own suite: HTTP/2, HTTP/3 and QUIC termination, the Python worker's + # framing and deadlines, the UDP relay — much of it reachable from no integration test, + # and none of it was run before a release. In the image stage built for it, which is + # the compiler plus the filter library the worker tests start. `backend/` is mounted + # rather than baked in because the worker tests launch `../modules/services/pyworker.py`. + engine_tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build the engine's test image + run: docker build --target enginetest -t firegex-enginetest . + + - name: Run the engine's tests + run: >- + docker run --rm -v "$PWD/backend:/execute" -w /execute/proxysrc + firegex-enginetest cargo test + docker_build: - needs: run_tests + needs: [run_tests, engine_tests] runs-on: ${{ matrix.os }} strategy: matrix: @@ -232,7 +255,7 @@ jobs: # installed is not, which is why the parser ships as source rather than as a `ctypes` # binding against the distribution's `libllhttp`. build_firegex_wheels: - needs: run_tests + needs: [run_tests, engine_tests] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -329,7 +352,7 @@ jobs: password: ${{ secrets.PYPI_API_TOKEN }} pypi_publish_fgex_alias: - needs: run_tests + needs: [run_tests, engine_tests] runs-on: ubuntu-latest permissions: contents: read diff --git a/AGENTS.md b/AGENTS.md index a2fc2e6..ed3170b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,9 +22,9 @@ python3 run.py status python3 run.py config --password [newpass] # change the running instance's password; omit the value to be prompted interactively python3 run.py config --show # print the persisted settings (port, host, socket dir, auth) python3 run.py start --unsafe-disable-auth # no authentication at all — see "Authentication" below -python3 run.py restart --env FGEX_PROXY_QUIC_ALPN=h3,doq # anything run.py has no opinion about +python3 run.py restart --env FGEX_PROXY_FILTER_TIMEOUT_MS=5000 # anything run.py has no opinion about ``` -`--env KEY=VALUE` (repeatable, `KEY=` removes one) is the *only* supported way to set what the engine and the backend read straight out of their environment — `FGEX_PROXY_QUIC_ALPN`, `FGEX_PROXY_FILTER_TIMEOUT_MS`, `FGEX_STREAM_MAX_SIZE`. It exists because `run.py` **rewrites** `.firegex-compose.yml` on every start, so a variable hand-edited into it is gone by the next one, which is what the QUIC documentation's "set it in the container's environment" silently ran into. It is stored in `.firegex-conf.json` like the other flags, reaches the standalone chroot as well as the container, and refuses the names `run.py` sets itself (`MANAGED_ENV`) rather than letting two writers of one variable decide by list order. +`--env KEY=VALUE` (repeatable, `KEY=` removes one) is the *only* supported way to set what the engine and the backend read straight out of their environment — `FGEX_PROXY_FILTER_TIMEOUT_MS`, `FGEX_STREAM_MAX_SIZE`. (`FGEX_PROXY_QUIC_ALPN` was the example here; it is gone — the engine reads each client's list off its first packet, see QUIC below.) It exists because `run.py` **rewrites** `.firegex-compose.yml` on every start, so a variable hand-edited into it is gone by the next one, which is what the QUIC documentation's "set it in the container's environment" silently ran into. It is stored in `.firegex-conf.json` like the other flags, reaches the standalone chroot as well as the container, and refuses the names `run.py` sets itself (`MANAGED_ENV`) rather than letting two writers of one variable decide by list order. `run.py` has zero external dependencies (stdlib only) by design. `restart` runs `docker compose up -d --force-recreate`, so it re-reads the generated compose file and re-applies env vars (`ALLOWED_IPS`, `PROXY_IP_HEADER`, `UNSAFE_DISABLE_AUTH`) — a plain `docker compose restart` would keep the old container environment and silently ignore those. It still does **not** rebuild the image; after a source change use `stop` then the bare start command to force `docker compose up -d --build`. Most `start`/`restart` flags are persisted to `.firegex-conf.json` and become the default for later runs, so a bare `python3 run.py start` inherits whatever was last passed. @@ -73,7 +73,7 @@ a **password** and ran everything against an instance it could not authenticate | Directory | Needs an instance | What it holds | |---|---|---| -| `unit/` | no | `firegex.regex` against libhs, the `firegex.pyfilters` models, the address→nftables translation | +| `unit/` | no | `firegex.regex` against libhs, the `firegex.pyfilters` models, the address→nftables translation, and `fgex pyfilters` run as its own process in front of a local echo (`test_proxysim.py` — nothing else ran the simulator) | | `integration/` | yes | the product, parametrised over the combinations | | `standalone/` | yes, and restarts it | settings read once at process startup | | `helpers/` | — | the API client, stand-in services, certificates, the traffic `Channel` | @@ -108,7 +108,7 @@ Mount `backend/`, not `backend/proxysrc/`: `tests/pyworker.rs` launches `../modu The suite runs against an instance with authentication turned off as well: `login()` reads a 403 there as "there is nothing to log in to" rather than as a failure, since every request is already allowed. `integration/test_auth.py` is the exception and skips itself — it is about passwords, and those endpoints answer 403 in that mode by design. Note `BearerSession.post` copies the header dict rather than writing into it: the form `Content-Type` belongs to the login request alone, and it used to stick, which went unnoticed only because a successful login replaces the whole dict a moment later. -CI (`.github/workflows/release.yml`) runs **only on `release: published`**, not on pushes or pull requests — nothing validates a PR automatically. Its `run_tests` job (`ubuntu-latest` + `ubuntu-24.04-arm`) does `python3 run.py start -P testpassword` then `cd tests && ./run_tests.sh`, and gates every publish job below it. The tests' dependencies are pure python; `iperf3`, matplotlib and numpy belong to `bench/requirements.txt` and CI installs none of them. +CI (`.github/workflows/release.yml`) runs **only on `release: published`**, not on pushes or pull requests — nothing validates a PR automatically. Two jobs gate every publish job below them, both on `ubuntu-latest` and `ubuntu-24.04-arm`: `run_tests` does `python3 run.py start -P testpassword` then `cd tests && sudo ./run_tests.sh`, and `engine_tests` runs the Rust suite (`cargo test` in `proxysrc`) in the Dockerfile's `enginetest` stage — the compiler stage plus the filter library the pyworker tests start. That suite was not run before a release at all, though much of what it covers (HTTP/2, HTTP/3 and QUIC termination, the worker's framing, the UDP relay) no integration test reaches. `enginetest` sits **before** `final` on purpose: a build with no target, which is what compose does, takes the last stage. The tests' dependencies are pure python; `iperf3`, matplotlib and numpy belong to `bench/requirements.txt` and CI installs none of them. ## Architecture @@ -138,7 +138,7 @@ The central idea, and the thing to preserve when changing anything here: **how t Client address transparency is preserved via `transparent::connect_as_udp` using `IP_TRANSPARENT` with mark `0x1339` and policy routing `table 1339`. New UDP addresses can be added to running services dynamically via `ADD_UDP` over the control channel without restarting the proxy. -What UDP keeps: per-flow filter state keyed by client address and released after `IDLE` — UDP has no close to observe, so a timeout is the only thing that ends a flow — and both directions inspected. `Verdict::Reject` drops the datagram rather than closing anything, because there is nothing to close. Replies leave through the **listener** socket so conntrack rewrites them to appear from the address the client dialled; sending from the upstream socket would reach a client not expecting that source. The relay's own dial carries `SELF_MARK`, or the redirect rule would catch it and loop. +What UDP keeps: per-flow filter state keyed by client address and released after `IDLE` — UDP has no close to observe, so a timeout is the only thing that ends a flow — and both directions inspected. `Verdict::Reject` drops the datagram rather than closing anything, because there is nothing to close. Replies leave through the **listener** socket so conntrack rewrites them to appear from the address the client dialled; sending from the upstream socket would reach a client not expecting that source. The relay's own dial carries `SELF_MARK`, or the redirect rule would catch it and loop. A flow outlives its service restarting: a datagram sent while the service was down comes back as ICMP "port unreachable", which the flow's connected socket reports once as `ConnectionRefused`, and `replies` carries on past it. It used to end the task reading the service's answers while the flow stayed in the table, and a client that kept talking kept the flow from ever going idle — so it never heard from the service again, while a new client did (`tests/udp.rs::a_flow_outlives_its_service_restarting`). `ConnectionMeta.l4` carries the transport to the filter, so the library's stream and HTTP models key off what the engine says rather than a hardcoded `True` in the worker. It is an enum and not a boolean because QUIC broke the question in two: a QUIC stream is ordered and reliable, so every model built on a stream applies to it, and it rides on UDP, so every rule that matches the wire matches UDP — one flag answering both got one of them wrong whichever way it was set. The worker is sent both words (`is_tcp` for the wire, `l4` for what can be built on it) and `RawPacket` exposes `l4` and `is_stream` beside `is_tcp`; `is_stream` is what `models/tcp.py` and `models/http.py` gate on. Absent, `l4` derives from `is_tcp`, because the NFQUEUE binaries carry TCP and UDP and have nothing else to say. @@ -151,7 +151,7 @@ Consequences worth keeping: - **`proto` is denormalised onto each address** and rewritten when the service's changes. That is what makes `(ip, port, proto)` a usable uniqueness key, so a TCP service and a UDP one can share an address exactly as the kernel allows. - **The `external` proxy endpoint belongs to the address, not the service**, and a partial unique index forbids two addresses sharing one: the return rule recognises the operator's proxy by address and port to put the original port back, so two addresses behind one endpoint could not be told apart on the way out. - **The proxy listener is dual-stack whenever the host supports IPv6** (`[::]:0`), and `transparent::unmap` turns the v4-mapped peer and local addresses back into IPv4 before anything else in the engine touches them — conntrack, the source-preserving dial and the metadata a filter reads all want the address the packet actually had. -- **Addresses accept network interface names as well as IPs.** An entry in `service_addresses` can be an IP/CIDR or a network interface name (1-15 chars, e.g. `eth0`, `wg0`, `tun0`, `lo`), checked via `utils.is_interface_name` and marked by `Address.is_interface`. The interface is not a second spelling of an address, and the interface is what the operator is choosing when the address is somebody else's to hand out. The frontend says which of the two it understood *while it is being typed* — `addressKind` in `components/InterfaceInput.tsx` puts it on a badge in the field, because `eth-0` is a perfectly good interface name and a perfectly wrong address, and the two are matched by completely different rules. That rule is written once on the frontend too, as `isInterfaceName`/`isIpAddress` in `js/utils.tsx`, reading the same charset the backend does — it used to be pasted into `AddEditService.tsx` and `AddressList.tsx`, two copies of a rule the picker beside them did not consult at all, which is how the hand-off layer came to offer interface names in the list and refuse them on submit. `InterfaceInput` lists interfaces and addresses as two groups, each interface carrying what it holds right now (one question decides between them: "and what is on it"), and stays a combobox rather than a select because an address a service will answer on need not exist on this host yet. **Link-local is not offered** — every interface carries an `fe80::` it was never configured with, which doubled the list with addresses that would put a service where no client looks for it, the same reason `udp_relay_host()` takes them last; one typed by hand is still accepted, and an interface holding nothing else says "link-local only" rather than listing it. A row is **the badge as a column, and everything else in the one beside it** — value, then the note, wrapping inside that column and never under the badge. Neither of those two is given a fixed share, because which one is long changes with the group (the addresses a link carries, or the address itself): fixing it took the whole width from a full-length IPv6 and left the other wrapping a character per line, a row a screen tall beside an address cut off mid-digit; and three items in one wrapping row put that same address at the container's left edge, under the badge, three lines tall. The list's own width is **computed from the longest value in it**, between a floor and a ceiling, and three simpler rules were wrong first: inherited from the content it collapsed to the field's width, because content allowed to wrap asks for less than it needs to stay on one line; pinned to the widest row any host could produce, every list was that wide beside fields half the size; and a floor alone put the one long address back over three lines. A host answering on `10.60.3.1` gets the floor, one carrying a full-length IPv6 gets the room to show it with its note beside it, and monospace is what makes the arithmetic honest. **The row beside them is laid out by the same arithmetic, and got it wrong the same way.** Its note (`now 10.10.0.3`, `on eth0`) is a flex item with a basis wide enough for a full-length IPv6, because a basis is what decides whether an item shares a line or drops to one of its own — at 80px it shared, and the first row with a couple of tags on it squeezed the note into the gap that was left and broke an IPv6 across three lines mid-digit. A min-width beside that basis is not the fix and was tried: on a phone the column is narrower than any floor worth setting, so the floor pushed the address past the card's edge, and it buys nothing anyway — an item only shares a line when its basis fitted, and on a line of its own it is already as wide as it can be. The row's own container grows to what the buttons leave rather than to its contents, or each row is as wide as its own longest line and two rows of one list wrap opposite ways: the one carrying a full IPv6 kept its note inline while the one saying `now 127.0.0.1` pushed it down. `overflowWrap: anywhere` stays as the last resort, which is what a 375px screen needs and nothing wider should reach. **`AddressList` reads in the same words it was chosen in** — the same badge at the same `BADGE_WIDTH`, then the value, then `on eth0` for an address and `now 10.10.0.3` for an interface — because a row an operator comes back to check, described differently from the list it came out of, is a reason to wonder whether it is the same thing. What is *shown* is `bareAddress`: the backend stores what it parsed, so `127.0.0.1` comes back as `127.0.0.1/32`, a spelling nobody types that reads like a range beside real ones; a host-length prefix is dropped for display and a real one is kept, which is also what tells `addressKind` apart. A real range survives an edit for the same reason — the modal seeded itself by cutting at the first slash, so opening "move this address" on a `/16` and saving narrowed a service to one host of it. Where interfaces are not an answer — the `external` proxy endpoint, and the service address on that layer — they are not listed at all and a name typed there is badged red, rather than offered and then refused on submit. Nftables matches use `meta iifname ` for inbound/prerouting and `meta oifname ` for outbound/postrouting/route. On `proxy` (TCP/TLS), redirect rules match `iifname` and `route_chain` marks return packets with `0x1339` on `oifname`. **The output hook cannot match either of those, and matches the interface's addresses instead.** `iifname` means nothing for a packet this host generates, and `oifname` is the wrong question twice over: `oifname eth0 dport 80` says *we* are calling somebody else's port 80, which is the host's own outbound traffic and none of firegex's business — and a connection from this host to one of its **own** addresses is routed through `lo` anyway, so it would never match. So `_add_proxy` resolves what the interface carries (`interface_addresses`, link-local dropped for the reason `udp_relay_host` takes it last) and writes one `daddr` rule per address. For a while it wrote nothing there at all, on the correct observation that `oifname` would hijack the wrong traffic, and the consequence was a service intercepted for the outside world and silently not for anything on the box — reported by an operator curling their own service and watching a blocked payload sail through. The cost is that an address the interface gains later is not covered *for traffic this host generates* until the service restarts; inbound keeps following the interface, and `udp_relay_host` pays the same price for the same reason. Each of those rules carries the interface's name as a **comment**, because nothing else in it does: `delete()` finds rules by what they match, and without it every one would outlive the service that installed it as a redirect to a port nobody is listening on. On UDP `proxy`, the primary IP assigned to the interface is resolved dynamically via `psutil.net_if_addrs()` to bind the relay. On `nfqueue`, interface traffic is queued natively in prerouting/postrouting without needing an IP. On `external`, interfaces are disallowed because return rewrites require a concrete IP. +- **Addresses accept network interface names as well as IPs.** An entry in `service_addresses` can be an IP/CIDR or a network interface name (1-15 chars, e.g. `eth0`, `wg0`, `tun0`, `lo`), checked via `utils.is_interface_name` and marked by `Address.is_interface`. The interface is not a second spelling of an address, and the interface is what the operator is choosing when the address is somebody else's to hand out. The frontend says which of the two it understood *while it is being typed* — `addressKind` in `components/InterfaceInput.tsx` puts it on a badge in the field, because `eth-0` is a perfectly good interface name and a perfectly wrong address, and the two are matched by completely different rules. That rule is written once on the frontend too, as `isInterfaceName`/`isIpAddress` in `js/utils.tsx`, reading the same charset the backend does — it used to be pasted into `AddEditService.tsx` and `AddressList.tsx`, two copies of a rule the picker beside them did not consult at all, which is how the hand-off layer came to offer interface names in the list and refuse them on submit. `InterfaceInput` lists interfaces and addresses as two groups, each interface carrying what it holds right now (one question decides between them: "and what is on it"), and stays a combobox rather than a select because an address a service will answer on need not exist on this host yet. **Link-local is not offered** — every interface carries an `fe80::` it was never configured with, which doubled the list with addresses that would put a service where no client looks for it, the same reason `udp_relay_host()` takes them last; one typed by hand is still accepted, and an interface holding nothing else says "link-local only" rather than listing it. A row is **the badge as a column, and everything else in the one beside it** — value, then the note, wrapping inside that column and never under the badge. Neither of those two is given a fixed share, because which one is long changes with the group (the addresses a link carries, or the address itself): fixing it took the whole width from a full-length IPv6 and left the other wrapping a character per line, a row a screen tall beside an address cut off mid-digit; and three items in one wrapping row put that same address at the container's left edge, under the badge, three lines tall. The list's own width is **computed from the longest value in it**, between a floor and a ceiling, and three simpler rules were wrong first: inherited from the content it collapsed to the field's width, because content allowed to wrap asks for less than it needs to stay on one line; pinned to the widest row any host could produce, every list was that wide beside fields half the size; and a floor alone put the one long address back over three lines. A host answering on `10.60.3.1` gets the floor, one carrying a full-length IPv6 gets the room to show it with its note beside it, and monospace is what makes the arithmetic honest. **The row beside them is laid out by the same arithmetic, and got it wrong the same way.** Its note (`now 10.10.0.3`, `on eth0`) is a flex item with a basis wide enough for a full-length IPv6, because a basis is what decides whether an item shares a line or drops to one of its own — at 80px it shared, and the first row with a couple of tags on it squeezed the note into the gap that was left and broke an IPv6 across three lines mid-digit. A min-width beside that basis is not the fix and was tried: on a phone the column is narrower than any floor worth setting, so the floor pushed the address past the card's edge, and it buys nothing anyway — an item only shares a line when its basis fitted, and on a line of its own it is already as wide as it can be. The row's own container grows to what the buttons leave rather than to its contents, or each row is as wide as its own longest line and two rows of one list wrap opposite ways: the one carrying a full IPv6 kept its note inline while the one saying `now 127.0.0.1` pushed it down. `overflowWrap: anywhere` stays as the last resort, which is what a 375px screen needs and nothing wider should reach. **`AddressList` reads in the same words it was chosen in** — the same badge at the same `BADGE_WIDTH`, then the value, then `on eth0` for an address and `now 10.10.0.3` for an interface — because a row an operator comes back to check, described differently from the list it came out of, is a reason to wonder whether it is the same thing. What is *shown* is `bareAddress`: the backend stores what it parsed, so `127.0.0.1` comes back as `127.0.0.1/32`, a spelling nobody types that reads like a range beside real ones; a host-length prefix is dropped for display and a real one is kept, which is also what tells `addressKind` apart. A real range survives an edit for the same reason — the modal seeded itself by cutting at the first slash, so opening "move this address" on a `/16` and saving narrowed a service to one host of it. Where interfaces are not an answer — the `external` proxy endpoint, and the service address on that layer — they are not listed at all and a name typed there is badged red, rather than offered and then refused on submit. Nftables matches use `meta iifname ` for inbound/prerouting and `meta oifname ` for outbound/postrouting/route. On `proxy` (TCP/TLS), redirect rules match `iifname`; the return path needs no per-interface rule, because it follows the conntrack mark (below). **The output hook cannot match either of those, and matches the interface's addresses instead.** `iifname` means nothing for a packet this host generates, and `oifname` is the wrong question twice over: `oifname eth0 dport 80` says *we* are calling somebody else's port 80, which is the host's own outbound traffic and none of firegex's business — and a connection from this host to one of its **own** addresses is routed through `lo` anyway, so it would never match. So `_add_proxy` resolves what the interface carries (`interface_addresses`, link-local dropped for the reason `udp_relay_host` takes it last) and writes one `daddr` rule per address. For a while it wrote nothing there at all, on the correct observation that `oifname` would hijack the wrong traffic, and the consequence was a service intercepted for the outside world and silently not for anything on the box — reported by an operator curling their own service and watching a blocked payload sail through. The cost is that an address the interface gains later is not covered *for traffic this host generates* until the service restarts; inbound keeps following the interface, and `udp_relay_host` pays the same price for the same reason. Each of those rules carries the interface's name as a **comment**, because nothing else in it does: `delete()` finds rules by what they match, and without it every one would outlive the service that installed it as a redirect to a port nobody is listening on. On UDP `proxy`, the primary IP assigned to the interface is resolved dynamically via `psutil.net_if_addrs()` to bind the relay. On `nfqueue`, interface traffic is queued natively in prerouting/postrouting without needing an IP. On `external`, interfaces are disallowed because return rewrites require a concrete IP. **An address says what is spoken at it and where the service is, and those are two different questions from the one the rules ask.** `service_addresses` carries `edge` — `tcp`, `tls`, `udp` or `quic`, the same words a service speaks minus `http`, which is the one that means "ask the addresses" — and `target_port`, which is where the traffic goes when that is not the port it arrived on. The two are scoped by different things, and confusing the two scopes is what the first version of this did. `edge` is only ever *chosen* on an `http` service, because that is the only protocol whose addresses are not all the same; elsewhere it is the service's own and a value disagreeing with it is refused, while `proto` (what the kernel matches) stays `L4.l4_of(edge)`. `target_port` is a question for the **layer**: publishing is being reachable one more way than the service is, and what that takes is something that *dials*. The proxy terminates the connection and opens the one to the service, so it can open it on another port, whatever the protocol — a plain `tcp` service published on a second port is exactly as real as an HTTPS one, and `_address_target()` was keyed on `http` only because `http` was the protocol the case was first noticed on. NFQUEUE hands the kernel a verdict on packets already on their way and opens nothing; `external` gives the traffic to the operator's own proxy, which decides for itself where the service is. On those two it is **refused** rather than stored, because a port nothing reads would tell an operator their service is published while it is merely intercepted. **And every one of them is editable after the fact.** `edit_address` rewrites `ip_int`, `port`, `proto`, `edge`, `target_port` and `upstream` — it took the first two and the proxy endpoint alone for a while, which made the rest add-only and left an operator who had set one while creating the service with no way back to it but deleting the address. The edit already takes that address's rules back and reinstalls them, which is the whole cost of changing any of it, and only that address stops being steered meanwhile. Two properties hold it together: everything the row will say is worked out **before** the old rules come back, so a refusal changes nothing; and a field the caller did not mention **keeps what the row had**, because this is also the endpoint that merely moves an address, and a move sending only an address and a port would otherwise quietly unpublish it. Taking one off is a value (`0`, `same`), not an absence. The interface asks for all of it in one place, `AddressOptions.tsx`, used by the ⚙ on each row of the creation form and by the modal on the service's page: the capability rules and the wording were written twice before, which is how the form came to offer what the backend then refused, and how one setting came to be described in two sets of words on two screens. The wording rule there took two corrections to settle. It began as field names — "The service is on port", "The service behind speaks" — which were true and left the operator to work out the effect. Rewritten as pure effect, "Send it to another port" was **read backwards**: with a port at each end, an operator with a cleartext service on `:80` who wanted it reached over TLS on `:443` typed `80` as the address and `443` here, and got a service intercepted where it already listened and published nowhere. So the label names the **fact the operator supplies** and the description says **what firegex does with it**, naming both ends: "The service is on another port" over "Clients arrive on :443 — firegex dials the service on :80". And because the misreading is really about the model rather than the sentence, the empty state says where a second way in comes from: another address, not another port in this box. @@ -175,6 +175,10 @@ Two implementations of the same concept is the mistake this module was restructu - **A pattern blocks. There is no second action, and `rewrite` is not coming back.** It existed, and was withdrawn because it could not keep the promise it made: rewriting scanned one chunk at a time, since bytes already forwarded cannot be taken back, so a match straddling two chunks was never rewritten. Measured — a pattern sent whole arrived redacted, the same pattern split across two TCP segments arrived intact, and blocking (which scans in stream mode) caught both. The miss was not the problem; its silence was — no block, no log, no counter, just a rule the operator trusted that had quietly not applied. The removal has to be complete to mean anything, and it was half-done for a while: an `action` column that was gone from the datapath still had a `rewritten` field travelling from `debug.rs` through the backend model to a panel in `RegexDebugger.tsx` that could never render, a single-member `Action` enum in `firegex.regex`, and a `replace_with` on the frontend `Regex` type. If any of those spellings reappears, it is residue, not a feature. - **A pattern is validated against the mode it will run in**, which is streaming, because that is how a pattern follows a connection across chunk boundaries. hyperscan does not accept quite the same patterns in every mode (`a{1,1000}b` streams but does not block-scan), and judging by the wrong one refuses rules that work — the same class of lie as a tester that disagrees with the engine. Block-mode scanning survives for one job only: reporting *where* a match starts, so the tester can highlight it. `debug.rs` reports `unscannable` for patterns that are valid, will run, and cannot be highlighted. - **The C++ binaries swap their configuration through `std::atomic>`** (`regex_config`, `config`). The updater thread replaces it while every queue thread copies it per packet or per stream, and a plain `shared_ptr` read and reset from two threads at once is a data race: the copy can take a reference on a control block the reset has just freed, so an operator editing a rule under load could crash the binary with it. Neither binary defines `DEBUG` any more — `cpproxy` did, which printed the operator's whole filter source to its stderr, and so into the service log, on every reload. +- **A new configuration reaches the connections already open, on every engine.** The proxy layer hands a connection's next chunk to the new chain, `cppregex` resets its matchers when the version changes, and `cpproxy` now does the same with its filter contexts (`PyCodeConfig::version`, `follow_config`, `stream_ctx::clean_filters`, which leaves the TCP sequence bookkeeping alone). It kept `latest_config_ver` and never read it, so a stream ran the code it started with for as long as it stayed open: a check added against the attack in progress did not reach the connection carrying it. `cpproxy` also no longer runs a full `PyGC_Collect()` per packet, and runs one per `COLLECT_EVERY` (64) dropped contexts rather than per context. A full pass cost ~0.9 ms with the library loaded, and the subinterpreter's own collector is on (`before_loop`), so the per-packet one bought nothing. The per-context one did buy something, which is why it is batched rather than dropped: a context is a cycle (the module's functions hold its globals), and left to the automatic schedule a promoted one waits for a full pass triggered by object counts rather than bytes. Measured with a filter building a table at module level, 1500 short connections: ~8 s with a pass per context and memory flat at 95 MB; ~0.8 s with none, but memory settling around 300 MB; ~0.8 s and 135–160 MB batched. +- **A connection the NFQUEUE binary did not see open is still inspected.** Both binaries call `follower.follow_partial_streams(true)` in `before_loop`. libtins' default drops any stream whose SYN it missed, and on this layer that is the ordinary case, not an edge one: a new filter is a new process, a chain that changes shape is rebuilt, and a service can be started on a port that already has connections. Each time, every open connection went uninspected until it closed, and nothing said so. It was found by holding a connection open across adding a filter and watching the pattern go through (`test_a_filter_added_to_a_running_service_reaches_a_connection_already_open`, and `test_a_connection_open_before_the_service_started_is_inspected`). A stream met halfway has no handshake to say which end is the client, so `cpproxy` takes the direction from the packet (`pkt->is_input`, from the queue mark) instead of from which libtins callback fired; `cppregex` already did. A stream met halfway can be met mid-message, and so can one whose filter state was replaced (below): see the next bullet for what the HTTP model does about it. The proxy layer has nothing equivalent to fix: a connection made before its redirect existed was never NATed to it. +- **A filter that takes over a connection does not judge a message by its second half.** Three things give a Python filter a context that begins after its connection did: a queued stage meeting a stream halfway (above), `cpproxy` replacing its contexts for new code (`clean_filters`, which records the TCP streams in `stream_ctx::taken_over`), and a proxy-layer worker that never got the connection's open frame (new code, or respawned after a crash). Each sets `__firegex_joined_late` in that context's globals, and the HTTP model then treats a parse failure *before the direction has parsed anything cleanly* as having come in mid-message: that direction is carried without its HTTP filters until the connection closes, with one `[info]` line per process (`_met_mid_message`). It used to go to `invalid_encoding_action`: with REJECT a valid upload in flight was refused whenever a filter changed, and without it a working client was logged as "not HTTP". Measured on both layers before the fix, pinned by `test_a_request_in_flight_when_the_filter_changes_is_not_judged_by_its_second_half`. A connection met between two messages parses, is in step (`_in_step`), and answers to the usual rules from then on; a connection seen from its first byte never gets the benefit. The trade is deliberate — never break a working client over firegex's own change of state: an attacker holding a message open across a filter change keeps that direction out of the HTTP filters for the connection's life, where before the partial-stream fix every open connection was out of every filter. +- **An edit reaches only the filter it was made to.** Every edit reloads the whole chain, and a reload used to reset every Python filter's per-connection state — module globals, the HTTP parser's place, on the proxy the addresses told at open — over a pattern changed in some other filter. `_QueueStage.reload` no longer sends a stage the payload it last acknowledged (`_in_force`), which also keeps `cppregex`'s matchers mid-stream; the proxy carries a `PyWorkerRule` over into the new chain when `Filter::reuse_key` is unchanged (`parse_ruleset_reusing`: id, command, enabled functions, deadline and a hash of the code itself, since saving rewrites the same path). Pinned by `an_unchanged_worker_is_carried_into_the_next_ruleset_with_its_state` and `test_an_edit_to_another_filter_leaves_a_python_filters_connections_alone`. - **The Python API is `firegex.pyfilters`, everywhere.** `cpproxy` embeds it; `modules/services/pyworker.py` runs it out of process. An earlier version of that worker defined a look-alike `@pyfilter` taking `(data, direction)`, which meant a filter written from the docs loaded on one transport and failed on the other. - **A filter file's protocol is read off the file, never declared.** The library already decides when to call a filter from what its parameters are annotated with, so `infer_proto` in `firegex/pyfilters/internals` reads the same table back the other way: asking for an `HttpRequest` makes a file an HTTP filter, asking only for a `RawPacket` makes one run on anything, and asking for two *different* application protocols is refused naming both functions. `type_annotations_associations` is the single place a protocol is defined; a second list of names would be one more thing to keep in step, and the day it fell behind the symptom would be a filter that silently never runs. The backend stores the detected value on the row so the interface can show it, and `PUT .../code` is what recomputes it. Passing one in from outside is exactly what used to make an HTTP filter on a TCP service a 500: the *service's* protocol was sent to the datapath, so the filter was refused for asking for an `HttpRequest`, which is the only thing an HTTP filter does. - **A filter file's functions run in the order the file defines them**, and the first to answer anything but ACCEPT ends the packet. `@pyfilter` sets a mark on the function and hands it back; `collect_pyfilters` reads a module's globals, which are in definition order. What that replaced was a *set of names* on a module-global attribute of the decorator, so the order was whatever hashing produced — which decided which function got credited with a block and whether an earlier `UNSTABLE_MANGLE` survived, with nothing in the file able to tell you which. The `position` column on `pyfilters` carries the same order to the interface, which used to list them `ORDER BY name`. Three consequences of that registry are also gone with it: `get_filters_info` had to clear it before and after (and reached it by `exec`ing import strings), two compilations could race, and a function registered from somewhere the module globals could not reach was registered and then not found. @@ -187,6 +191,7 @@ Two implementations of the same concept is the mistake this module was restructu - `Filters` compiles once under the operator's file name instead of re-parsing per connection, so runtime tracebacks in the log name their file and their line too. - A filter is called directly. `context_call` used to write the arguments into the module globals, `eval` a call expression built from them, and delete the three keys again — per filter, per packet — which is exactly `func(*args)`, because the operator's code was `exec`ed into those globals and so the function's `__globals__` *is* that dict. - **`FGEX_STREAM_MAX_SIZE` bounds what a stream costs in memory, and three things had got past it.** A compressed body was decoded whole (`on_message_complete`): a megabyte of gzip under the cap is a gigabyte after it, so every step of the decoding is bounded by the cap now, and a body that would go past it meets `FGEX_FULL_STREAM_ACTION` like any other stream past the cap (with `FLUSH`, handed over still encoded). A WebSocket message is bounded the same way (`max_size` to `Frame.parse`). The flush itself cleared the *view* of a body and subtracted it from the count while the buffer it is read from kept growing — so on a gRPC stream, whose body is handed over as it grows, memory grew past every flush and the count went negative (`_flush_body`); framed gRPC bytes are also dropped as they are handed over (`_forget_framed`), and a frame past the cap is skipped to its end so the next one is still found where it starts. And `HttpHistory` counted messages but not bytes, a hundred bodies of a megabyte per direction per connection; each direction's history is held under the cap too (`_trim_history`). Separately, the flush raised on a message still in its headers (`len(None)`), so headers past the cap were an exception and the packet failed open. + - **Traffic the HTTP parser cannot read is carried by default, and said so** (`invalid_encoding_action` defaults to `ACCEPT`). It was refused, so a request the service would have taken and llhttp would not — one client's quirk, a protocol that only resembles HTTP — closed the connection with nobody having chosen that, credited to the filter function as if it had decided it. Refusing is one line in the filter file for an operator who wants it against request smuggling; when it refuses, the block is credited to `@INVALID_ENCODING`, and the warning is one line per connection and direction rather than a traceback per packet. - The default for `FGEX_STREAM_MAX_SIZE` is 1 MiB, as documented. It read `1*8e20` under a comment saying "1MB", so `FGEX_FULL_STREAM_ACTION` never fired unless a size was set by hand, and the float it produced would have been refused by the setter beside it. - **The editor's hints are introspected, never written down.** `modules/services/pyapi.py` reads the models out of `type_annotations_associations` and their members out of the classes — including `writable`, from `property.fset`, which is how the interface can say whether a member can be assigned to — none can, since the payload stopped being writable with `UNSTABLE_MANGLE`. `GET /api/services/pyfilter-api` serves it and `pyfilterHints.ts` feeds Monaco's completion and hover from it. A second, hand-written description would be wrong the first time a model gained a property, and autocompletion that lies is worse than none. The route is declared **before** `/{service_id}`, or that one matches it and looks for a service by that name. - **A filter that raises says so the same way on both layers.** `_on_exception` writes one sentence — the traffic was forwarded unfiltered — and it has to be reachable from either. NFQUEUE gets there directly: the interpreter is embedded in the binary, which sends `EXCEPTION`. The proxy cannot, because `pyworker.py` catches the exception in a separate process and answers with an ordinary ACCEPT, so nothing crosses the frame boundary that the engine could tell apart from a filter agreeing. It marks stderr with `EXCEPTION_MARK` instead and `Transport._pump_stderr` turns that back into the same call. The traceback is printed *beside* it, not instead of it: the traceback says which filter and which line, and the sentence says what happened to the connection — for a while the proxy printed only the first, which tells an operator a filter broke and not that traffic went through unchecked. The marker is spelled in both files because the worker is launched as a script and imports nothing from the backend, which is what keeps the user's code out of this process's imports; `tests/integration/test_resilience.py` asks for the sentence on both layers and so keeps them in step. @@ -204,6 +209,8 @@ Each module (`services`, `firewall`) follows the same shape: **Every nftables object firegex creates is named `fgex_`, and lives in a table firegex owns.** The services module holds `table inet fgex` with `fgex_queue_in_`, `fgex_queue_out_`, `fgex_hijack_*`, `fgex_divert`, `fgex_nat`, `fgex_nat_out`, `fgex_route`, `fgex_guard` and the set `fgex_datapath` — and its `reset()` deletes **the whole table**, because flushing it chain by chain was one batch that nft refuses outright if any chain named in it is missing, so a table left by a version with other chains (an upgrade after a crash) was never cleared and went on steering traffic at dead queues and ports; the firewall module holds `fgex_filter` and `fgex_mangle` per family, with base chains `fgex_input`/`fgex_forward`/`fgex_output`/`fgex_prerouting`/`fgex_postrouting` jumping to `fgex_rules_in`/`_out`/`_fwd`. One `grep fgex_` over the ruleset is the whole footprint. +**Something else removing those tables is noticed, said and undone, within `TABLE_WATCH` (3 s).** `nft -f /etc/nftables.conf` begins with `flush ruleset` on Debian, and so does restarting its unit; before this, every service went on reading ACTIVE with nothing reaching a filter, and a default-deny firewall became no firewall, with nothing said. Each module's `FirewallManager` runs a `_watch_table` task from `init()` to `close()`/`stop_watching()`, asking `nft.intact()` — one small chain that `init()` always leaves rules in, so a deleted table and an emptied one both fail it. The services module rebuilds the table and calls `ServiceManager.put_rules_back()`, which is the second half of `enable()` alone: the datapath never stopped, so nothing restarts and no connection is dropped. It runs under the service's lock, like `disable()`, or rules put back while a stop is waiting on its datapath would outlive the stop (`test_rules_are_not_put_back_behind_a_stop`). Each service logs it as an error, at most once per `TABLE_TOLD_QUIET`, because traffic went through unfiltered in between. Partial surgery — one rule of ours deleted by hand — is not detected; that is somebody editing firegex's table, not somebody else's tool wiping the ruleset. + **An interface rule says where traffic *arrived*, not where it was going, and that is deliberate.** `iifname docker0 tcp dport 443` matches every container's **outbound** HTTPS as well, because that traffic arrives on `docker0` too, and a `redirect` then takes it into the engine, which terminates it with this service's certificate — measured: `pip install` inside any container failed with `CERTIFICATE_VERIFY_FAILED: hostname mismatch` while such a service was running, and worked the moment it stopped. Pinning the destination to the addresses the interface carries was written, tested and **reverted**, because it removes the one case interfaces exist for: an operator names a link when **the address is not theirs to know**, which includes firegex in front of a service on another machine reached through it, and a rule pinned to this host's own addresses silently stops protecting that. Protecting a link protects what crosses it. Naming one that carries other people's traffic is a choice, and the answer to not wanting it is a narrower address, not a narrower rule — `docs/services.md` says so where the choice is made, because it is surprising the first time. Do not re-narrow this without a way to tell the two deployments apart. **An interface stands for the addresses it carries, and every place that needs an address has to resolve it — three of them gave up instead.** `udp_relay_host()` got this right from the start; the others did not, and each failure was silent in its own way. `_add_proxy` wrote no output-hook rule at all, so a service protected on `lo` filtered the outside world and not a client on the box. `_published()` and `ServiceManager.address_added` skipped the announcement, because the engine keys that map on what `SO_ORIGINAL_DST` hands back and an interface name is not an address — so an HTTPS edge declared on `lo:443` was never announced, the engine met the connection knowing nothing, terminated it as an ordinary one and dialled `:443` where nothing listens. HTTPS on that address simply did not answer while the same address written `127.0.0.1:443` worked, which is how it was found and is as confusing a symptom as this module has produced. All three now go through `interface_addresses()`, which returns the concrete addresses (link-local dropped) and hands back a single-element list for an address, so the two spellings take one path. The cost is the one `udp_relay_host` already pays: the answer is resolved when the rules are installed, so an address the interface gains later is not covered until the service restarts. When adding anything that needs *an address* rather than *a match*, the question to ask is what it does when handed `eth0` — and the answer is never "skip it". @@ -215,19 +222,21 @@ The firewall module is the one that had to move to get there, and the reason is - **`iptables` stopped working on the host.** `iptables-nft` refuses an entire table as soon as one rule in it cannot be expressed in iptables' own format, and it stores conntrack matches as an `xt` blob where nft writes a native `ct state`. Bisecting the rules firegex writes, `ct state` was the only one that tripped it — which is to say `allow_established` and `drop_invalid`, on in any sane configuration. From then on `iptables -L` and `iptables-save` answer ``table `filter' is incompatible, use 'nft' tool``, on a box where Docker, ufw or the organisers' scripts are the ones asking. - **Stopping firegex opened the host.** `reset()` re-added `filter INPUT` with `policy accept`, and in nftables `add chain` on an existing chain *updates* its policy. That chain was the administrator's: a host carrying default-deny had it silently removed when the firegex firewall was switched off. -A base chain carries its own policy wherever it lives, so default-deny is expressed exactly as before and a `drop` is terminal whichever table's chain reached it. **One thing about what gets blocked did change, and `allow_dnat` puts it back**: in iptables' own `FORWARD` chain, Docker's or podman's accept for a published port came before or after firegex's jump in the same chain, and either way traffic to a published container port never met firegex's policy; a base chain of our own is evaluated on its own, so a drop policy dropped every connection to a published container port — how every service on a CTF box is reached. `dnat_rules()` appends `ct status dnat accept` to `fgex_forward` after the jump (on by default, a switch in the firewall settings): the operator's forward rules still apply first, and the runtime's own chain still decides for itself. What changed is that firegex now adds a layer instead of taking one over: an `accept` in `fgex_rules_in` means "firegex does not block this", never "nothing else will", which is all it ever honestly meant. `reset()` is now deleting two tables per family (`add` before `delete`, because a batch is atomic and deleting a table that is not there fails all of it). +A base chain carries its own policy wherever it lives, so default-deny is expressed exactly as before and a `drop` is terminal whichever table's chain reached it. **One thing about what gets blocked did change, and `allow_dnat` puts it back**: in iptables' own `FORWARD` chain, Docker's or podman's accept for a published port came before or after firegex's jump in the same chain, and either way traffic to a published container port never met firegex's policy; a base chain of our own is evaluated on its own, so a drop policy dropped every connection to a published container port — how every service on a CTF box is reached. `dnat_rules()` appends `ct status dnat accept` to `fgex_forward` after the jump (on by default, a switch in the firewall settings): the operator's forward rules still apply first, and the runtime's own chain still decides for itself. **And what the containers send**, which the first version of it left out: with the policy at drop, a web container could not reach its database on the same bridge and no container reached the network — measured with two containers on a Docker network — because Docker accepts that in `FORWARD` too, beside where firegex's rules used to be. So traffic arriving from a container runtime's bridge (`CONTAINER_BRIDGES`: `docker0`, `br-*`, `docker_gwbridge`, `podman*`, `cni-podman*`; br_netfilter shows bridged packets with the bridge as their interface) is left to the runtime the same way. `get_rules` also had the family of every rule narrowed by the first one naming an address (`families` set once, outside the loop): an "accept port 80" listed below an IPv4 rule did not exist for IPv6. What changed is that firegex now adds a layer instead of taking one over: an `accept` in `fgex_rules_in` means "firegex does not block this", never "nothing else will", which is all it ever honestly meant. `reset()` is now deleting two tables per family (`add` before `delete`, because a batch is atomic and deleting a table that is not there fails all of it). `Rule.table` is still `filter` or `mangle` in the database, the API and the interface — it names the hook a rule is evaluated at. `NFT_TABLES` maps it to the table it lands in; the two were one string only because that string used to be somebody else's table.<> A refused edit must change nothing: `_apply_chain(service_id, undo)` in `routers/services.py` rolls the write back when the transport rejects the resulting chain. Without it an operator is told their filter was rejected and then finds it in the list, and the next start fails for the reason they thought they had avoided. -`edit_service` holds itself to the same rule, and did not. It now checks the service it would become **before writing** — `Transport.check()` against the chain, whether or not the service is running (a stopped one used to take any combination and fail on its next start), a published port refused on a layer that does not dial, and the addresses' `edge`/`proto`/`upstream` recomputed for the new protocol (`_address_under`) — then writes the service and its addresses in **one transaction** (`db.queries`), because a protocol change that collided with another service's address used to leave the service rewritten and its addresses not. A running service whose new definition will not start has the old one written back and is started again on it, rather than being left stopped on the definition that failed. What an address can still be given afterwards — the port of the operator's own proxy on a service moved to the hand-off layer — is not checked there, exactly as at creation: `ExternalTransport.check` names it at start. +`edit_service` holds itself to the same rule, and did not. It now checks the service it would become **before writing** — `Transport.check()` against the chain, whether or not the service is running (a stopped one used to take any combination and fail on its next start), a published port refused on a layer that does not dial, and the addresses' `edge`/`proto`/`upstream` recomputed for the new protocol (`_address_under`) — then writes the service and its addresses in **one transaction** (`db.queries`), because a protocol change that collided with another service's address used to leave the service rewritten and its addresses not. A running service whose new definition will not start has the old one written back and is started again on it, rather than being left stopped on the definition that failed. What an address can still be given afterwards — the port of the operator's own proxy on a service moved to the hand-off layer — is not checked there, exactly as at creation: `ExternalTransport.check` names it at start. **Only what differs from the row counts as an edit.** The form sends every setting on every save, and each save used to stop and start a running service — resetting every connection it carried for opening the dialog and pressing save. A change of name alone is written and nothing is restarted, because nothing running reads it. Order is the load-bearing detail in `services/firewall.py`, because getting it wrong is not an error anybody sees — it is a window during which traffic goes unfiltered. Starting: the capture interface if the service decrypts (the engine opens its socket to it at startup, so one that appears afterwards is one this process will not find), then the datapath, then the nft rules that point at it. Stopping is the exact reverse. ### Block statistics (`modules/services/stats.py`) A cumulative counter per filter, per pattern and per `@pyfilter` function lives in the database; the bucketed history answers the other question — *when* did this start. Blocks are counted per filter into one-minute buckets, kept for `HISTORY_BUCKETS`, and flushed at most every `FLUSH_INTERVAL` seconds. Both properties are load-bearing for the same reason the live log's are: a service refusing thousands of connections a minute must not be able to take the backend down through its own bookkeeping, which is what one row per block would do. There is deliberately **no background task** — flushing happens on the next write once the interval has passed, and always on a read — so nothing has to be started, cancelled or waited for at shutdown. The frontend draws it as hand-written SVG (`StatsPanel.tsx`) rather than pulling in a charting library several times the size of the page it sits on. +**Per-block counters are written once a second, in one transaction** (`ServiceManager.flush_blocks`, `BLOCK_FLUSH`), and the names a block is logged under are looked up once per rule until the chain changes. They were three commits per refused connection, about 17 ms a block, so the backend fell behind any flood of refusals — and that was not only its own problem: the engine wrote `BLOCKED` to a pipe the backend was not reading fast enough, the write blocked the runtime thread that made it, and the flood slowed every other client of the service (measured: 5000 refusals took 74 s and a legitimate client waited a median 4.5 s, failing half the time; now 4 s and 1 ms). Every services API request writes what is waiting first (the router's `_counters_written` dependency), so a read is exact and an edit resetting a pattern's counter cannot have the old pattern's last blocks written onto it afterwards. On the engine's side **nothing on the datapath writes to stdout** (`report.rs`): lines go through a bounded queue and a thread of their own; `BLOCKED` and `STATS` are dropped and counted when the backend is that far behind, and replies the backend waits for (`PORT`, `UDP`, `ACK`) wait for room, in order. + `share` is computed in the backend, not the browser, so a chart and the table beside it cannot disagree about what a rule accounts for. **History is stored per *rule*, not per filter, and that is what makes a time range mean one thing.** `block_history` keys on the token the datapath reported — a pattern id, `/`, or a filter id — so `GET .../stats?range_from=&range_to=` can answer the chart, the per-filter totals, the per-pattern totals and the shares over the *same* window. Storing only per filter would have left the chart honouring the range and the tables beside it reporting all of time. A filter's total is the **sum of its rules**, not a lookup of its own id: that token appears only when nothing named itself. @@ -250,20 +259,23 @@ The cost is that some blocks in a range then belong to no rule still present — - NFQUEUE has no connection to count, so it reports `null` for the share rather than dividing refused connections by a packet count. **The asymmetry is the difference between the layers showing through; do not "fix" it by inventing a denominator.** The engine's counters arrive on a timer, so a test that reads them has to wait for them. ### The live log (`modules/services/logs.py`) -A bounded ring per service plus a coalescing flush over a `log` Socket.IO event (distinct from `update`, which only invalidates queries). Two properties are load-bearing and easy to lose: the ring is capped, and the flush is on a timer with a per-batch cap that reports a dropped count rather than carrying it — a service refusing thousands of connections a minute must not be able to take the backend or the browser down through its own logging. +A bounded ring per service plus a coalescing flush over a `log` Socket.IO event (distinct from `update`, which only invalidates queries). **A warning or an error does not wait for the service's page to be opened**: `ProblemNotifier` (mounted in `MainLayout`) turns `warn`/`error` lines into a notification wherever the operator is — one per service per 20 s, the rest counted into it, except that an error is never folded into a warning ("restarting it" followed by "could not restart it" hid the second until a third came, which it need not) — and the service list carries the newest one (`ServiceLog.last_problem`, `problem` on `ServiceModel`) as a badge for an hour; clearing the log dismisses it. The log panel was the only place these went, so an engine that died and came back, a filter raising on every packet, or a fallback that cost the service its clients' addresses waited for somebody to look. `cpproxy`'s merged stdout/stderr goes through `_on_engine` like every other datapath's (`_QueueStage._forward_output`, whole lines only): it went in as the user's output, so its `[error]` lines and the traceback of a filter that raised were grey text among the prints. Two properties are load-bearing and easy to lose: the ring is capped, and the flush is on a timer with a per-batch cap that reports a dropped count rather than carrying it — a service refusing thousands of connections a minute must not be able to take the backend or the browser down through its own logging. Sources: block attribution (resolved to the filter name and pattern, one query per block, on a path that only runs when something was already refused), the user's `print()`, and the datapath's own stderr. That last one is why `pyworker.py` reassigns `sys.stdout = sys.stderr` before running user code: stdout is the length-prefixed protocol channel, and a `print()` landing in the middle of a frame gets the worker killed on every packet with nothing to explain it. `tests/pyworker.rs` has a regression test for exactly that. ### The proxy datapath (`backend/proxysrc/`) A Rust binary, built in the Dockerfile's `compiler` stage rather than a `rust` image on purpose: it links the same `libhs` the C++ binaries do, and a binary built against one distro's vectorscan and run against another's is a bad afternoon. -- `filter.rs` — the chain, and everything that keeps a filter from taking the datapath down: `catch_unwind`, a deadline on `spawn_blocking`, and a filter that misbehaves losing its say rather than the traffic being held. `panic = "unwind"` in `Cargo.toml` is load-bearing, not tuning. +- `filter.rs` — the chain, and everything that keeps a filter from taking the datapath down: `catch_unwind`, a deadline on `spawn_blocking`, and a filter that misbehaves losing its say rather than the traffic being held. `panic = "unwind"` in `Cargo.toml` is load-bearing, not tuning. **Losing its say for missing the deadline is a pause, not a sentence** (`REARM_AFTER`, 30 s): it used to last for the life of the chain, and a Python filter misses its deadline when exchanges queue behind one another — which is what a flood does, the one moment it is wanted — so the protection stayed off until somebody next edited a rule. A filter that is still slow loses its say again after the same few misses, which bounds what a genuinely hung one strands to a few threads per pause. A panic is broken code and stays out until the chain is replaced; the warning says how. - Filters that keep state do so **per connection, per direction** (`ChainSessions`), because one client's bytes deciding another client's verdict is both a false positive and a way to smuggle a pattern past a filter by splitting it across two connections. - `pyworker.rs` frames carry a connection id and a close frame, so the Python side can honour the documented promise that each stream gets its own module globals. - Source preservation is unconditional: the engine always dials the service from the client's address. `ct status dnat` is what separates the engine answering a client from the service answering the engine, which is why redirect (not tproxy) works even when the service is on this host. A transparent dial the service **refuses** is not retried as ourselves: nothing is listening, and the retry used to blame source preservation for a service that was simply down. +- **A service behind a container runtime's rewrite works from anywhere, on both layers** — it is the ordinary CTF deployment. On the proxy layer the engine dials the published address as the client, the runtime's DNAT sends it to the container, and the container answers **from its own address**: `socket transparent` in `fgex_divert` found no socket talking to that address, so the answer went on to the real client, which reset it, and every connection from another host waited out the dial and was retried as this host — slow, and the client's address lost. `fgex_divert` now also marks home replies carrying the engine's conntrack mark (the one `fgex_route` copies onto every connection the engine opens). On NFQUEUE, a client **on this host** was never inspected: the runtime rewrites that connection in `nat OUTPUT`, so it leaves towards the container without passing prerouting, and the container's answer arrives for a local socket without passing postrouting. `fgex_queue_lreq_` (output) and `fgex_queue_lrep_` (input), at `LOCAL_QUEUE_PRIORITY` (-90: after the rewrite on the way out, before it is undone on the way in, so both halves show the container's address and a stream follower sees one stream), match such connections by conntrack's **original** destination with `ct status dnat`, and are found again by `get()` through their `address ` comment. Measured with a real Docker container; `tests/integration/test_remote_clients.py` stages the same rewrite into a network namespace so no Docker is needed. +- **The service's answers are brought home by who opened the connection, not by the port they come from.** `fgex_route` copies the engine's `SELF_MARK` onto the conntrack entry of every connection it opens and marks whatever answers one of those for local delivery — and that is all it holds. The per-address rule that used to sit there matched the service by the address it is protected on, which is where the engine dials it only while the address *is* the service: one published on another port, or an HTTP/3 edge in front of an HTTP/1.1 service (dialled over TCP), answered from a port nothing matched, and the answer was routed out to the client instead of home. A client on this host never noticed — the reply is local either way — so every test passed while a client on another host timed out. `tests/integration/test_remote_clients.py` puts the client in a network namespace to ask. Matching every reply *from* the service's port instead was not the fix: that port can be reachable directly, and its own clients' answers would have been diverted. The per-address rule was exactly that, for the protected port itself, and it was removed for exactly that reason: a connection open before the service started is never redirected (conntrack translates a connection at its first packet), so its answers belong to the client, and a client on another host stopped hearing back the moment protection began. Measured with a connection held open from a network namespace across the start, TCP and UDP (`test_a_connection_already_open_survives_the_service_starting`); NFQUEUE never had the problem. - **The engine's own ports answer only what was redirected to them** (`fgex_guard`). The TCP listener and every UDP/QUIC relay are bound to the wildcard, because a redirect delivers to whichever address the packet arrived on — so a relay, which forwards to its service whoever sent the datagram, put a service protected on `127.0.0.1:53` on the network. `fgex_datapath` holds `l4proto . port` of every one of them and `fgex_guard` (input hook) drops what reaches one without conntrack's DNAT status; `fgex_divert`'s mark exempts replies to the engine's transparent sockets, whose destination is a client's port and can be any number. A port is added when its redirect is installed and released only when the **whole service** comes down (`FiregexTables._guarded`), because a relay stays bound after its address is removed and would otherwise be reachable directly again — and an element left behind after a stop would drop traffic for whatever process is given that port next. - **A connection nothing redirected is refused** (`dialled_at_ourselves`), behind the guard as well as in front of it. The listener is on an ephemeral port of every address (`[::]:0`), so anybody can dial it straight; `SO_ORIGINAL_DST` then answers with the address the connection was accepted on, and forwarding there is the engine dialling itself — measured, one such client became as many connections as the limit allowed in two seconds, and with no limit it grows until the process runs out of descriptors. The old guard compared with the configured listen address, which is the wildcard and never matches. -- **The backend watches the engine** (`ProxyTransport._watch`, the same `on_died` → `_recover_datapath` the NFQUEUE stages always had). The nft rules outlive the process, so a dead engine used to leave every connection redirected at a closed port — a total outage while the service read `ACTIVE`, and nothing ever restarted it. `reload()` on a dead engine raises rather than returning quietly, so an edit is not reported applied to nothing. +- **Stopping or restarting a proxied service does not cut what the engine carries.** The engine terminates every connection, so `stop()` — killing it — cut them all, measured: a connection through the engine read EOF on the next write after a stop and after a restart, while the same test on NFQUEUE carried on. `ServiceManager.disable` takes the rules off and calls `Transport.retire` instead: conntrack keeps delivering what it already translated to the old engine's ports, new connections go to the service (a stop) or to the new engine (a restart — `handing_over`, from `restart()` and `refresh()`), and the old engine exits once a `STATS` line counted after it was taken out (`DRAIN_FRESH_AFTER`) says `live=0` — which counts TCP connections, UDP flows and QUIC connections alike. A stop pushes it an empty chain first, because stopping is how an operator takes a hurting filter out of the way; a restart keeps the old chain. Bounded by `DRAIN_LIMIT` (10 min) and `MAX_DRAINING` engines per service, both logged when they close something. Its ports stay guarded until it is gone (`nft.delete(keep_guards=True)` → `release_guards`, which leaves alone a port another service has since been given), and its TLS files are removed at once — it read them at startup, and removed later they would be the new engine's copies. `FirewallManager.close()` still kills: the table that brings the answers home goes with it (`stop_draining`). Pinned by `test_stopping_a_proxied_service_does_not_cut_its_connections`, `test_a_restart_hands_over_without_cutting_connections` and `unit/test_proxy_drain.py`. +- **The backend watches the engine** (`ProxyTransport._watch`, the same `on_died` → `_recover_datapath` the NFQUEUE stages always had). The nft rules outlive the process, so a dead engine used to leave every connection redirected at a closed port — a total outage while the service read `ACTIVE`, and nothing ever restarted it. `reload()` on a dead engine raises rather than returning quietly, so an edit is not reported applied to nothing. **A datapath that does not acknowledge a command is killed, and one that refuses it is left running** (`Transport._abandon`). A refusal leaves the previous configuration in force on all three binaries; the NFQUEUE stages used to be stopped on one anyway — and stopped *on purpose*, so the watchdog said nothing and the service read active with no process behind its queue. A missing answer is different: what was applied cannot be known, and the answer arriving late was read as the answer to the next command, putting every command after it one answer out of step. So the process is killed without being marked stopped, and the watchdog rebuilds the service from the saved configuration. The wait for a configuration is `LOAD_TIMEOUT` per Python filter rather than `ACK_TIMEOUT`, because loading one runs its module body and the code check allows that eight seconds. - **The client's SNI goes on to a TLS upstream**, beside its ALPN list (and on the QUIC → HTTPS path once the client's handshake has said it). A service choosing its certificate or virtual host by SNI was handed the bare address — which TLS does not even send — and answered as its default host. A QUIC upstream is dialled before the client's handshake and cannot be told. ### The `external` hand-off @@ -327,7 +339,7 @@ On the TCP edge that is nothing but a config: `build_tls` hands back no upstream - **The request target has to be reduced to origin form.** HTTP/2 and HTTP/3 carry it split across `:scheme`, `:authority` and `:path`, which `http::Uri` holds as one absolute URI; written to a socket unchanged that is `GET https://host/path`, the *proxy* form, which an origin server may refuse and which every service reading the path back gets wrong. `h1up` reduces it and moves the authority to `host` — the same decision `render_request` already makes for the chain, so what the filters saw and what the service receives stay the same message. The tests caught this immediately; it is not theoretical. - **One connection per exchange**, because a QUIC stream is one request and HTTP/1.1 has no multiplexing. Pooling would put two clients' requests on one socket and with it the thing `ChainSessions` exists to prevent, at a layer nobody would think to look at. Its body is a channel into hyper, and `finish()` **drops the sender** — waiting on the channel instead (`Sender::closed()`) waits for the service to answer a request this end has not finished sending, which is a deadlock the chunked test found at once. -One combination is refused, and it is impossible rather than unimplemented: a QUIC edge carrying something that is **not HTTP/3**, whose opaque stream bytes have no HTTP/1.1 form. That is a fact about the **ALPN**, not about the protocol — `upstream_refusal()` in `modules/services/models.py` keys on `quic_alpn()` and so refuses exactly the instances that set `FGEX_PROXY_QUIC_ALPN` to something else, on `quic` and `http` services alike. Keyed on the protocol instead, as it was first written, it refused every QUIC service and allowed every HTTPS one, which is a rule with no reason behind it. The same function is asked from all three places that would otherwise drift: at creation (against the addresses in the form), when an address is added, and in `ProxyTransport.check`. +One combination cannot work, and it is impossible rather than unimplemented: a QUIC edge carrying something that is **not HTTP/3** to a service that speaks HTTP/1.1, because opaque stream bytes have no HTTP/1.1 form. Nothing refuses it at configuration time any more — there is no list to check it against, since the protocol is each client's (see below) — so on such an address the engine answers `h3` itself and a client offering only something else is refused by that handshake. `upstream_refusal()` used to refuse it up front against a configured list, and went with the list. **The decrypted traffic is written to an interface of its own, and those are reconstructed packets.** Terminating in-process means the plaintext is never a packet, so `proxysrc/src/capture.rs` frames it as the TCP stream it was and sends it to `firegex0` over `AF_PACKET`: one interface carrying every decrypted service's plaintext — TLS and QUIC both, see the QUIC section for what a stream costs there — and nothing else, which is what a capture tool wants pointed at it. The bytes are real — what the filters saw and what was forwarded — but the framing is invented, because the framing on the wire was encrypted. Sequence numbers start at zero, there are no retransmissions, and the segmentation is the engine's read sizes; a SYN and a FIN are emitted so Wireshark has a stream to follow rather than orphan segments. Say so wherever it is offered: a capture from here is not evidence of what was on the wire. @@ -341,7 +353,7 @@ Note for tests: a client now dials the service's own address, because there is n It is UDP to the kernel, and that is the whole of what the backend had to learn: `L4.l4_of(quic) == udp`, so the addresses, the uniqueness key, the nft rules, `udp_relay_key`/`udp_relay_host`, `ProxyTransport.udp_ports` and `ADD_UDP` are the code that was already there. What changes is what binds the port — `FGEX_PROXY_QUIC=1` beside the certificate makes the engine build `quic::QuicManager` instead of `udp::UdpManager`, and `relays::Relays` is the two-armed enum the control channel and the startup list both go through. `Service.decrypts` (once `terminates_tls`) is what gates the certificate, on both protocols. -**The ALPN is asked of the service, which is the one thing that could not be kept from the TLS path.** There the ClientHello is held at `LazyConfigAcceptor`; in QUIC it arrives inside an encrypted Initial whose processing *is* the handshake, and quinn offers nothing to hold it at. So the order is reversed: `dial_upstream` goes first with the candidate list, and `QuicSetup::answering_with` advertises to the client exactly the one protocol the service chose. The invariant survives — the client is never told something the service did not say — and what is lost is knowing in advance whether the client would take it, which shows up as a failed handshake with a log line naming the protocol. The list is `FGEX_PROXY_QUIC_ALPN` (default `h3`), read from the instance's environment like the filter deadline beside it; a per-service list would be a column and nothing has wanted one. +**The ALPN is mirrored, as on the TLS path, and nothing is configured.** The ClientHello travels inside an Initial packet, and that was taken for a reason it could not be read before answering — so the engine offered the service a candidate list instead, `FGEX_PROXY_QUIC_ALPN` for the whole instance and then a per-service field, and a QUIC service speaking anything but HTTP/3 was unreachable until somebody found the setting. But Initial keys are derived from the Destination Connection ID the client sends in the clear (RFC 9001 §5.2), exactly so a middlebox can read this. `quic_hello.rs` does: `SniffingSocket` wraps the endpoint's socket, every long-header Initial is decrypted with `rustls::quic::Suite::keys`, CRYPTO frames are reassembled by offset across datagrams (a ClientHello with post-quantum key shares does not fit one packet), and the ALPN list is parked per client address for `QuicRelay::offered`, which waits `HELLO_WAIT` for the rest of a split hello. `dial_upstream` then offers the service exactly that list and `QuicSetup::answering_with` tells the client the one protocol the service chose — both built per connection, as the TLS path does. The map is bounded (`MAX_PENDING`, `KEPT_FOR`) because forged Initials cost the sender nothing. A hello that cannot be read falls back to `h3`, said once per relay. The parser is tested against packets protected with rustls's own client-side keys, and `tests/quic.rs` drives real quinn clients through it, including one whose hello spans packets. **One stream is one connection to the chain** (`Carrier::open`), with its own `ChainSessions` and its own module globals, released when the stream ends. A filter's state follows a stream of bytes from start to end, which is what a QUIC stream is and what a QUIC connection is not; on HTTP/3 it also lands where an operator expects, one request to a stream. The raw path reuses `proxy.rs`'s pumps unchanged — `spawn_pumps`, `pump` and `join_pumps` are `pub(crate)` for exactly this, and `spawn_pumps` takes the `spoken` flag rather than making one, because the first-byte deadline is the *connection's* question and a flag per stream would never answer it for a connection that opened a hundred and spoke on none. diff --git a/Dockerfile b/Dockerfile index 160f1b4..9f7d96f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,15 @@ RUN g++ binsrc/pyfilter.cpp -o cpproxy -std=c++23 -O3 -Wall -lnetfilter_queue -l COPY ./backend/proxysrc /execute/proxysrc RUN cd /execute/proxysrc && cargo build --release +# Where the engine's own suite runs (`cargo test` in proxysrc). It starts real Python +# workers through the filter library, so it needs the compiler stage *and* that library +# with what it imports — without them the pyworker tests fail with a ModuleNotFoundError +# buried in captured output. Nothing below copies from it, and it sits before `final` so a +# build with no target (what compose does) is not handed this as the image. +FROM --platform=$TARGETARCH compiler AS enginetest +COPY ./fgex-lib /fgex-lib +RUN python3.14 -m pip install --no-cache-dir --break-system-packages /fgex-lib + #Building main conteiner FROM --platform=$TARGETARCH base AS final diff --git a/backend/binsrc/pyproxy/pyproxy.cpp b/backend/binsrc/pyproxy/pyproxy.cpp index a5e260b..9150aaa 100644 --- a/backend/binsrc/pyproxy/pyproxy.cpp +++ b/backend/binsrc/pyproxy/pyproxy.cpp @@ -30,7 +30,22 @@ namespace PyProxy { class PyProxyQueue: public NfQueue::ThreadNfQueue { private: - u_int16_t latest_config_ver = 0; + // The configuration this thread's contexts were built from. + uint32_t latest_config_ver = 0; + + // A new configuration reaches the connections already open, the way it does on the + // other two engines: `cppregex` resets its matchers on a new version and the proxy + // layer hands the next chunk to the new chain. This was declared and never read, so + // here a stream kept the code it had started with for as long as it stayed open — a + // function switched off, or a check added against the attack in progress, went on not + // reaching the one connection it was meant for. Its state starts again, as it does + // there. + void follow_config(){ + const uint32_t current = config.load()->version; + if (current == latest_config_ver) return; + sctx.clean_filters(); + latest_config_ver = current; + } public: stream_ctx sctx; StreamFollower follower; @@ -71,6 +86,14 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue { } handle_packet_code = unmarshal_code(py_handle_packet_code); + // Streams whose start this process did not see are followed from the packet it + // does see. They are otherwise not followed at all, and a packet of one was simply + // accepted: every connection open before the service started, or before its chain + // was rebuilt — which adding a filter does — and every one quiet for longer than + // the follower keeps a stream, carried whatever it said afterwards. A persistent + // connection was a way past any filter added after it opened. Recovery mode is + // switched on for them in `on_new_stream`. + follower.follow_partial_streams(true); // Setting callbacks for the stream follower follower.new_stream_callback(bind(on_new_stream, placeholders::_1, this)); follower.stream_termination_callback(bind(on_stream_close, placeholders::_1, this)); @@ -168,6 +191,13 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue { stream.ignore_server_data(); return pkt->accept(); } + // This context starts after its connection did: the process met the + // stream halfway, or new code replaced the context it had. What arrives + // first can be the middle of a message, and the library must not judge + // a message by its second half — see `_met_mid_message` there. + if (stream.is_partial_stream() || sctx.taken_over.erase(pkt->sid) > 0){ + stream_match->set_item_to_glob("__firegex_joined_late", PyBool_FromLong(1)); + } sctx.streams_ctx.insert_or_assign(pkt->sid, stream_match); } }else{ @@ -204,7 +234,12 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue { } - static void on_data_recv(Stream& stream, PyProxyQueue* pyq, const string& data, bool is_client) { + // The direction is the packet's own, from the mark the rules put on it, rather than + // which side of the stream the follower calls the client: on a stream followed from + // the middle that is whoever it happened to see first, and the service answering + // before the client speaks again made the filter read the answer as the request. + static void on_data_recv(Stream& stream, PyProxyQueue* pyq, const string& data, bool) { + const bool is_client = pyq->pkt->is_input; pyq->pkt->fix_data_payload(); pyq->filter_action(pyq->pkt, stream, data, is_client); //Only here the rebuilt_tcp_data is set } @@ -255,6 +290,7 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue { void handle_next_packet(NfQueue::PktRequest* _pkt) override{ pkt = _pkt; // Setting packet context + follow_config(); if (pkt->l4_proto == NfQueue::L4Proto::UDP){ // Straight to the filter: the stream follower is TCP's, and so is every diff --git a/backend/binsrc/pyproxy/settings.cpp b/backend/binsrc/pyproxy/settings.cpp index 2aedc0d..bfca866 100644 --- a/backend/binsrc/pyproxy/settings.cpp +++ b/backend/binsrc/pyproxy/settings.cpp @@ -27,8 +27,14 @@ PyObject* unmarshal_code(string encoded_code){ } class PyCodeConfig{ + private: + // Only the updater thread builds configurations, so this needs no lock. + static inline uint32_t glob_seq = 0; public: string encoded_code; + // Which configuration this is, so a queue thread can tell that the contexts it + // holds were built from code no longer in force. 0 is the empty one it starts on. + uint32_t version = 0; PyCodeConfig(const string& pycode){ PyObject* compiled_code = Py_CompileStringExFlags(pycode.c_str(), "", Py_file_input, NULL, 2); @@ -61,6 +67,7 @@ class PyCodeConfig{ } encoded_code = string(PyBytes_AsString(code_dump), PyBytes_Size(code_dump)); Py_DECREF(code_dump); + version = ++glob_seq; } PyObject* compiled_code(){ diff --git a/backend/binsrc/pyproxy/stream_ctx.cpp b/backend/binsrc/pyproxy/stream_ctx.cpp index 52ff60d..542d9e7 100644 --- a/backend/binsrc/pyproxy/stream_ctx.cpp +++ b/backend/binsrc/pyproxy/stream_ctx.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,11 @@ inline bool traceback_is_due() { previous, now, std::memory_order_relaxed); } +// How many dropped filter contexts may wait for a full collection, and how many have on +// this queue thread — each thread runs an interpreter of its own, so each counts its own. +constexpr unsigned COLLECT_EVERY = 64; +static thread_local unsigned dropped_since_collect = 0; + struct pyfilter_ctx { PyObject * glob = nullptr; @@ -108,7 +114,18 @@ struct pyfilter_ctx { ~pyfilter_ctx(){ Py_DECREF(glob); Py_DECREF(py_handle_packet); - PyGC_Collect(); + // The module's functions hold these globals and the globals hold the functions, + // so a context is a cycle only the collector frees. Left to the interpreter's + // own schedule, one that lived long enough to be promoted waits for a full pass, + // which is triggered by object counts rather than by bytes — measured, a filter + // building a table at module level held ~200 MB more across a stream of short + // connections. A full pass per context held memory flat and cost ~1 ms each, a + // ceiling of about two hundred connections a second per queue thread. One pass + // every `COLLECT_EVERY` contexts keeps both. + if (++dropped_since_collect >= COLLECT_EVERY){ + dropped_since_collect = 0; + PyGC_Collect(); + } } inline void set_item_to_glob(const char* key, PyObject* value){ @@ -160,8 +177,11 @@ struct pyfilter_ctx { // Set packet info to the global context set_item_to_glob("__firegex_packet_info", packet_info); + // No collection here. The interpreter's own collector is on (`before_loop` + // makes sure), and a full pass per packet cost about a millisecond with the + // library loaded — a ceiling of a thousand packets a second per queue thread, + // spent finding nothing. PyObject * result = PyEval_EvalCode(py_handle_packet, glob, glob); - PyGC_Collect(); del_item_from_glob("__firegex_packet_info"); if (PyErr_Occurred()){ @@ -315,7 +335,13 @@ struct stream_ctx { list> udp_recent; map>::iterator> udp_where; + // The TCP streams whose filter context was thrown away while they were open — new + // code arrived — so the context built for their next packet knows it is taking over + // halfway, possibly in the middle of a message. + set taken_over; + void clean_stream_by_id(stream_id sid){ + taken_over.erase(sid); auto stream_search = streams_ctx.find(sid); if (stream_search != streams_ctx.end()){ auto stream_match = stream_search->second; @@ -359,6 +385,22 @@ struct stream_ctx { } } + // Every filter context, TCP and UDP alike, for when the code they were built from is + // no longer the code in force. The sequence bookkeeping stays: it belongs to the + // connection, not to the filter, and a stream whose payload was already cut would + // have its acknowledgements go wrong without it. + void clean_filters(){ + for (auto ele: streams_ctx){ + if (udp_where.find(ele.first) == udp_where.end()){ + taken_over.insert(ele.first); + } + delete ele.second; + } + streams_ctx.clear(); + udp_recent.clear(); + udp_where.clear(); + } + void clean_tcp_ack_by_id(stream_id sid){ auto tcp_ack_search = tcp_ack_ctx.find(sid); if (tcp_ack_search != tcp_ack_ctx.end()){ @@ -369,16 +411,12 @@ struct stream_ctx { } void clean(){ - for (auto ele: streams_ctx){ - delete ele.second; - } + clean_filters(); + taken_over.clear(); for (auto ele: tcp_ack_ctx){ delete ele.second; } tcp_ack_ctx.clear(); - streams_ctx.clear(); - udp_recent.clear(); - udp_where.clear(); } }; diff --git a/backend/binsrc/regex/regexfilter.cpp b/backend/binsrc/regex/regexfilter.cpp index d5f5ada..e6e15c7 100644 --- a/backend/binsrc/regex/regexfilter.cpp +++ b/backend/binsrc/regex/regexfilter.cpp @@ -181,6 +181,14 @@ public: } void before_loop() override{ + // Streams whose start this process did not see are followed from the packet it + // does see. They are otherwise not followed at all, and a packet of one was simply + // accepted: every connection open before the service started, or before its chain + // was rebuilt — which adding a filter does — and every one quiet for longer than + // the follower keeps a stream, carried whatever it said afterwards. A persistent + // connection was a way past any filter added after it opened. Recovery mode is + // switched on for them in `on_new_stream`. + follower.follow_partial_streams(true); follower.new_stream_callback(bind(on_new_stream, placeholders::_1, this)); follower.stream_termination_callback(bind(on_stream_close, placeholders::_1, this)); } diff --git a/backend/modules/firewall/firewall.py b/backend/modules/firewall/firewall.py index 8864c7e..077a9ff 100644 --- a/backend/modules/firewall/firewall.py +++ b/backend/modules/firewall/firewall.py @@ -1,4 +1,5 @@ import asyncio +import traceback from modules.firewall.nftables import FiregexTables from modules.firewall.models import Rule, FirewallSettings from utils.sqlite import SQLite @@ -6,18 +7,44 @@ from modules.firewall.models import Action nft = FiregexTables() +#: How often the kernel is asked whether the firewall is still there. The services +#: module asks the same of its own table, for the same reason: `nft flush ruleset` — +#: the first line of a Debian `/etc/nftables.conf` — takes this with it, and a +#: default-deny firewall silently became no firewall at all. +TABLE_WATCH = 3.0 + class FirewallManager: def __init__(self, db:SQLite): self.db = db self.lock = asyncio.Lock() + self._watch: asyncio.Task | None = None async def close(self): + self.stop_watching() async with self.lock: nft.reset() async def init(self): nft.init() await self.reload() + if self._watch is None or self._watch.done(): + self._watch = asyncio.create_task(self._watch_table()) + + async def _watch_table(self): + while True: + await asyncio.sleep(TABLE_WATCH) + try: + if self.enabled and not nft.intact(): + print("[error] [firewall] the firewall's nftables tables were removed " + "by something else; putting them back", flush=True) + await self.reload() + except Exception: + traceback.print_exc() + + def stop_watching(self) -> None: + if self._watch is not None: + self._watch.cancel() + self._watch = None async def reload(self): async with self.lock: diff --git a/backend/modules/firewall/nftables.py b/backend/modules/firewall/nftables.py index 17b9a0d..5cc8d63 100644 --- a/backend/modules/firewall/nftables.py +++ b/backend/modules/firewall/nftables.py @@ -27,16 +27,23 @@ practice. Everything installed is visible under one prefix and removed by deleti tables per family. One thing did change, and `dnat_rules` is what puts it back: sharing iptables' own -`FORWARD` chain, traffic Docker or podman accepted for a published port never met -firegex's policy, while a chain of our own is evaluated on its own — so a drop policy -dropped every connection to a published container port. `allow_dnat` (on by default) -leaves destination-NATed traffic no rule of ours matched to the rules that published it. +`FORWARD` chain, traffic Docker or podman accepted never met firegex's policy, while a +chain of our own is evaluated on its own — so a drop policy dropped every connection to a +published container port, every connection between two containers and every one a +container made to the outside. `allow_dnat` (on by default) leaves that traffic, when no +rule of ours matched it, to the container runtime's rules as it always was. """ from modules.firewall.models import FirewallSettings, Action, Rule, Protocol, Mode, Table from utils import nftables_int_to_json, ip_family, NFTableManager, is_ip_parse import copy +#: The bridges a container runtime forwards through: Docker's default one and its user +#: networks (`br-`), and podman's. Traffic arriving from one of them is a container +#: talking — to another container, or out — and iptables' `FORWARD`, where Docker and +#: podman accept it, is where firegex's rules used to sit too. +CONTAINER_BRIDGES = ("docker0", "br-*", "docker_gwbridge", "podman*", "cni-podman*") + #: Where a rule's `table` — which is the operator's word, stored in the database and #: shown in the interface — actually lands. The two are deliberately not the same string #: any more: "filter" and "mangle" describe what a rule *does*, and used to double as the @@ -257,6 +264,17 @@ class FiregexTables(NFTableManager): ) ]) + def intact(self) -> bool: + """Whether the firewall firegex installed is still in the kernel. + + The input chain of the IPv4 filter table always holds its jump, so it is the one + place a deleted table and an emptied one both show. + """ + code, listed, _ = self.raw_cmd({"list": {"chain": { + "family": "ip", "table": self.filter_table, "name": "fgex_input", + }}}) + return code == 0 and any("rule" in item for item in listed["nftables"]) + def set(self, srvs:list[Rule], policy:str=Action.ACCEPT, opt:FirewallSettings = None): srvs = list(srvs) self.reset() @@ -298,15 +316,23 @@ class FiregexTables(NFTableManager): """ if opt is None or not opt.allow_dnat: return [] + matches = [ + {"match": {"op": "in", "left": {"ct": {"key": "status"}}, "right": "dnat"}}, + # What the containers send, to each other and out. Leaving only the published + # ports to the runtime was the first version, and with the policy at drop it + # cut a web container off from its database on the same bridge, and every + # container off from the network — neither of which met firegex's policy + # while its rules lived in iptables' `FORWARD` beside Docker's own accepts. + *({"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": bridge}} + for bridge in CONTAINER_BRIDGES), + ] return [ {"add": {"rule": { "family": family, "table": self.filter_table, "chain": "fgex_forward", - "expr": [ - {"match": {"op": "in", "left": {"ct": {"key": "status"}}, "right": "dnat"}}, - {"accept": None}, - ], + "expr": [match, {"accept": None}], }}} for family in ("ip", "ip6") + for match in matches ] def get_rules(self,*srvs:Rule): @@ -314,15 +340,23 @@ class FiregexTables(NFTableManager): final_srvs:list[Rule] = [] for ele in srvs: if ele.proto == Protocol.BOTH: + # Copies both ways: the rule handed in is the caller's, and rewriting its + # protocol in place turned a stored "both" into "tcp" for whoever read it + # next. udp_rule = copy.deepcopy(ele) udp_rule.proto = Protocol.UDP.value - ele.proto = Protocol.TCP.value - final_srvs.append(udp_rule) + tcp_rule = copy.deepcopy(ele) + tcp_rule.proto = Protocol.TCP.value + final_srvs.extend((udp_rule, tcp_rule)) + continue final_srvs.append(ele) - - families = ["ip", "ip6"] - + for srv in final_srvs: + # Per rule. It was set once, before the loop, and narrowed by the first rule + # naming an address — so every rule after that one was installed for that + # family only: an "accept port 80" written below an IPv4 rule did not exist + # for IPv6, and with the policy at drop every IPv6 client was refused. + families = ["ip", "ip6"] ip_filters = [] if srv.src != "": diff --git a/backend/modules/services/firewall.py b/backend/modules/services/firewall.py index b435d5d..57bac58 100644 --- a/backend/modules/services/firewall.py +++ b/backend/modules/services/firewall.py @@ -59,6 +59,24 @@ OVER_LIMIT_QUIET = 30 #: was in the log pushed out behind it. RAISED_QUIET = 30 +#: How long a block's counters may wait in memory before being written. They used to be +#: three writes per refused connection, each its own commit — about 17 ms a block, so the +#: backend fell behind any flood of them, the engine's pipe filled, and the flood slowed +#: every other client of the service. Once a second, in one transaction, whatever the +#: rate. Anything reading the counters through the API writes what is waiting first. +BLOCK_FLUSH = 1.0 + +#: How often the kernel is asked whether firegex's rules are still there. +#: +#: Something else can remove them: `nft -f /etc/nftables.conf` begins with `flush +#: ruleset` on Debian, and so does restarting the `nftables` unit. Every service then +#: went on reading `ACTIVE` while nothing reached a filter, and nothing said so. Every few +#: seconds is a short window and a cheap question — one small chain. +TABLE_WATCH = 3.0 +#: And how long between two log lines about it, per service. Something removing them on a +#: timer would otherwise write one every `TABLE_WATCH`. +TABLE_TOLD_QUIET = 60 + def code_path(filter_id: str) -> str: return os.path.join(CODE_DIR, f"{filter_id}.py") @@ -113,6 +131,16 @@ class ServiceManager: #: Carried between lines so a multi-line traceback keeps its severity. self._in_traceback = False self._engine_level = LEVEL.INFO + #: Blocks counted and not yet written, by the token the datapath reported. + self._unwritten_blocks: dict[str, int] = {} + #: Rules put back since the log last said so, and when it did. + self._rules_lost = 0 + self._rules_lost_told = 0.0 + self._block_flush: asyncio.TimerHandle | None = None + #: What a reported token is called and which filter owns it, looked up once per + #: token rather than once per block, and forgotten whenever the chain changes. + self._named: dict[tuple[str, str], str] = {} + self._owners: dict[str, str] = {} # --- reading the configuration ------------------------------------------- @@ -165,23 +193,13 @@ class ServiceManager: # token both network layers report. Split once, here, so there is a single place # that knows the format. rule_id, _, function = reported.partition("/") - if function: - self.db.query( - "UPDATE pyfilters SET blocked = blocked + 1 WHERE filter_id = ? AND name = ?;", - rule_id, - function, - ) - # What is left is a regex rule or a whole filter, depending on which refused. - # Only one of these matches, and a blind update is cheaper than asking first. - self.db.query( - "UPDATE regexes SET blocked = blocked + 1 WHERE regex_id = ?;", rule_id - ) - self.db.query( - "UPDATE filters SET blocked = blocked + 1 WHERE filter_id = ? OR filter_id = " - "(SELECT filter_id FROM regexes WHERE regex_id = ?);", - rule_id, - rule_id, - ) + self._unwritten_blocks[reported] = self._unwritten_blocks.get(reported, 0) + 1 + if self._block_flush is None: + try: + self._block_flush = asyncio.get_running_loop().call_later( + BLOCK_FLUSH, self.flush_blocks) + except RuntimeError: + self.flush_blocks() # The chart is attributed to the filter, not to the pattern or the function: it # is about which link in the chain is carrying the traffic, and a chain of twenty # patterns would be a chart nobody can read. The finer totals are listed @@ -192,20 +210,62 @@ class ServiceManager: stats.flush(self.db) self.log.add(LEVEL.BLOCK, f"connection refused by {self._describe(rule_id, function)}") + def flush_blocks(self) -> None: + """Write the counters of every block reported since the last time.""" + if self._block_flush is not None: + self._block_flush.cancel() + self._block_flush = None + pending, self._unwritten_blocks = self._unwritten_blocks, {} + if not pending: + return + queries = [] + for reported, count in pending.items(): + rule_id, _, function = reported.partition("/") + if function: + queries.append(( + "UPDATE pyfilters SET blocked = blocked + ? " + "WHERE filter_id = ? AND name = ?;", + count, rule_id, function, + )) + # What is left is a regex rule or a whole filter, depending on which refused. + # Only one of these matches, and a blind update is cheaper than asking first. + queries.append(( + "UPDATE regexes SET blocked = blocked + ? WHERE regex_id = ?;", + count, rule_id, + )) + queries.append(( + "UPDATE filters SET blocked = blocked + ? WHERE filter_id = ? OR " + "filter_id = (SELECT filter_id FROM regexes WHERE regex_id = ?);", + count, rule_id, rule_id, + )) + try: + self.db.queries(queries) + except Exception: + traceback.print_exc() + def _owning_filter(self, rule_id: str) -> str: """Which filter a block belongs to. The id is a filter's or a pattern's.""" - found = self.db.query( - "SELECT filter_id FROM regexes WHERE regex_id = ?;", rule_id - ) - return found[0]["filter_id"] if found else rule_id + owner = self._owners.get(rule_id) + if owner is None: + found = self.db.query( + "SELECT filter_id FROM regexes WHERE regex_id = ?;", rule_id + ) + owner = self._owners[rule_id] = found[0]["filter_id"] if found else rule_id + return owner def _describe(self, rule_id: str, function: str = "") -> str: """Name what refused a connection, rather than echoing an opaque id. - One query per block, on a path that only runs when something was already - refused — a block is rare compared to the traffic that is not blocked, and an - operator reading `refused by 4f2a91c8` learns nothing. + Looked up once per rule and kept until the chain changes: a flood of refusals is + one rule refusing thousands of times, and an operator reading `refused by + 4f2a91c8` learns nothing. """ + named = self._named.get((rule_id, function)) + if named is None: + named = self._named[(rule_id, function)] = self._look_up_name(rule_id, function) + return named + + def _look_up_name(self, rule_id: str, function: str) -> str: if function: # A file holds several functions, so naming the file alone would leave the # operator to guess which of them refused the connection. @@ -433,25 +493,72 @@ class ServiceManager: f"{len([link for link in self.chain() if link.filter.active])} filter(s) active", ) - async def disable(self, persist: bool = True): + async def disable(self, persist: bool = True, handing_over: bool = False, + drain: bool = True): + """Stop filtering this service. + + The connections the datapath is carrying are not cut (`drain`): the rules come + off first, so nothing new reaches it, and the transport carries what it already + has until it closes — see `ProxyTransport.retire`. `handing_over` is a restart, + whose connections carry on under the filters they started with; a stop carries + them unfiltered. Only shutting firegex down cuts them, since the rules that bring + their answers home go with it. + """ # Under the lock for the same reason: a stop arriving while a start was still in # progress saw a service that was not active yet, returned at once, and the start # then finished — so the stop the operator asked for was simply lost. async with self.lock: + self.flush_blocks() if not self.active: return - nft.delete(self.srv) + guarded = nft.delete(self.srv, keep_guards=drain) self._steer = {} if self.transport: - await self.transport.stop() + if drain: + await self.transport.retire( + keep_filtering=handing_over, + then=lambda: nft.release_guards(guarded), + ) + else: + await self.transport.stop() self.transport = None + else: + nft.release_guards(guarded) self._set_status(False, persist=persist) self.log.add(LEVEL.INFO, "stopped") async def restart(self): - await self.disable() + await self.disable(handing_over=True) await self.enable() + async def put_rules_back(self) -> None: + """Steer this service at its datapath again, after something else removed the rules. + + The datapath never went anywhere — the engine or the queue binaries are still + running and still listening where `_steer` says — so this is the second half of + `enable()` alone. Under the lock, like `disable()`, so a stop in progress cannot + have its rules put back behind it. + """ + async with self.lock: + if not self.active or self.transport is None: + return + nft.delete(self.srv) + nft.add(self.srv, **self._steer) + self._rules_lost += 1 + now = time.time() + if now - self._rules_lost_told < TABLE_TOLD_QUIET: + return + times = f" ({self._rules_lost} times since the last report)" \ + if self._rules_lost > 1 else "" + self._rules_lost, self._rules_lost_told = 0, now + self.log.add( + LEVEL.ERROR, + "firegex's rules were removed from the kernel by something else — an " + "`nft flush ruleset`, or a firewall being reloaded — and traffic reached " + f"this service unfiltered until they were put back{times}. Make whatever " + "manages this host's nftables leave the `fgex` tables alone.", + ) + async def update_chain(self): """Push the current chain to a running datapath. @@ -466,6 +573,11 @@ class ServiceManager: silently enforcing the old order. """ restart = False + # What was refused before this edit is written under the names it had, and what + # is refused after it is looked up again. + self.flush_blocks() + self._named.clear() + self._owners.clear() async with self.lock: if not self.active or not self.transport: return @@ -633,7 +745,7 @@ class ServiceManager: """The service's own definition changed, which the datapath cannot absorb.""" was_active = self.active if was_active: - await self.disable() + await self.disable(handing_over=True) self.srv = srv if was_active: await self.enable() @@ -660,6 +772,7 @@ class FirewallManager: self.db = db self.services: dict[str, ServiceManager] = {} self.lock = asyncio.Lock() + self._watch: asyncio.Task | None = None async def init(self): nft.init() @@ -670,6 +783,38 @@ class FirewallManager: # takes the tool with it. mirror.ensure() await self.reload() + if self._watch is None or self._watch.done(): + self._watch = asyncio.create_task(self._watch_table()) + + async def _watch_table(self): + """Put the rules back whenever something outside firegex takes them away.""" + while True: + await asyncio.sleep(TABLE_WATCH) + try: + if not nft.intact(): + await self._put_rules_back() + except Exception: + traceback.print_exc() + + async def _put_rules_back(self): + async with self.lock: + # Asked again under the lock, which may have been held by something that + # rebuilt the table while this waited for it. + if nft.intact(): + return + print("[error] [backend] firegex's nftables table was removed by something " + "else; putting it back", flush=True) + nft.init() + for manager in list(self.services.values()): + try: + await manager.put_rules_back() + except Exception as e: + manager.log.add(LEVEL.ERROR, f"could not put its rules back: {e}") + + def stop_watching(self) -> None: + if self._watch is not None: + self._watch.cancel() + self._watch = None async def reload(self): async with self.lock: @@ -690,10 +835,10 @@ class FirewallManager: # of it working. traceback.print_exc() - async def remove(self, srv_id: str, persist: bool = True): + async def remove(self, srv_id: str, persist: bool = True, drain: bool = True): async with self.lock: if srv_id in self.services: - await self.services[srv_id].disable(persist=persist) + await self.services[srv_id].disable(persist=persist, drain=drain) del self.services[srv_id] if persist: # A deleted service's log has nothing left to be about. @@ -706,16 +851,32 @@ class FirewallManager: releasing here would make the device disappear and come back underneath whatever was capturing from it — the exact failure it was made persistent to avoid. """ + # First, or a table deleted on the way down is one it would put back. + self.stop_watching() + # Cut, not drained: what brings a drained connection's answers home is this + # table's, and it is about to go — as is the process that would end the drain. for key in list(self.services.keys()): try: - await self.remove(key, persist=False) + await self.remove(key, persist=False, drain=False) except Exception: self.services.pop(key, None) + await transports.stop_draining() + stats.flush(self.db, force=True) def release_capture(self) -> None: """Take the capture interface away, at process shutdown and nowhere else.""" mirror.release() + def flush_blocks(self) -> None: + """Write every service's waiting block counters, before anything reads them. + + The history behind the chart too: it waits up to `stats.FLUSH_INTERVAL` for the + next block, and at shutdown there is no next block. + """ + for manager in self.services.values(): + manager.flush_blocks() + stats.flush(self.db, force=True) + def get(self, srv_id: str) -> ServiceManager: if srv_id not in self.services: raise ServiceNotFoundException() diff --git a/backend/modules/services/logs.py b/backend/modules/services/logs.py index 745dbc8..e2e454d 100644 --- a/backend/modules/services/logs.py +++ b/backend/modules/services/logs.py @@ -141,7 +141,10 @@ class ServiceLog: final count — so the chart sat at whatever it happened to reach two seconds in. The last block of a burst is precisely the one worth hearing about. """ - if not any(entry["level"] == LEVEL.BLOCK for entry in entries): + # A warning or an error changes the service list too: it carries the latest one + # (`last_problem`). + if not any(entry["level"] in (LEVEL.BLOCK, LEVEL.WARN, LEVEL.ERROR) + for entry in entries): return waited = time.monotonic() - self._counters_told if waited >= COUNTER_INTERVAL: @@ -170,6 +173,19 @@ class ServiceLog: def entries(self) -> list[dict]: return list(self.history) + def last_problem(self) -> dict | None: + """The newest warning or error still in the tail, for the service list to show. + + The live log is on the service's own page, and a warning nobody has that page open + for is a warning nobody reads: a datapath that died and came back, a filter that + raised, a fallback that lost the client's address. The list carries the latest one + beside the service, and clearing the log is how it is acknowledged. + """ + for entry in reversed(self.history): + if entry["level"] in (LEVEL.WARN, LEVEL.ERROR): + return entry + return None + def clear(self) -> None: self.history.clear() self._pending.clear() diff --git a/backend/modules/services/models.py b/backend/modules/services/models.py index c7d0746..4abbb4d 100644 --- a/backend/modules/services/models.py +++ b/backend/modules/services/models.py @@ -12,7 +12,6 @@ that can host it, and the service does not have to be recreated to change either """ import base64 -import os import uuid from utils import is_ip_parse @@ -319,58 +318,6 @@ class Address: return f"
" -def quic_alpn() -> tuple[str, ...]: - """What this instance offers a QUIC service's clients, in order. - - Instance-wide (`FGEX_PROXY_QUIC_ALPN`) rather than per service, because nothing has - yet wanted a column for it: `h3` is what a QUIC service speaks nine times in ten, and - an instance in front of something else sets it in its own environment the way the - filter deadline beside it is set. Read here as well as where the engine is launched - so that the one question it decides — whether an exchange can be rendered at all — - has a single answer. - """ - raw = os.getenv("FGEX_PROXY_QUIC_ALPN", "h3") - listed = tuple(p.strip() for p in raw.split(",") if p.strip()) - return listed or ("h3",) - - -def upstream_refusal(proto: str, upstream: str, edge_is_udp: bool) -> str | None: - """Why one address cannot be forwarded the way it was told to, or `None`. - - Asked of an **address** rather than of the service, because that is where it is - chosen: what is behind `:443` is the same daemon as what is behind `:80`, but the two - reach it differently — one may be re-encrypted and the other handed the plaintext, and - on an `http` service the two are separate ports with separate answers. Asked from the - three places that would otherwise drift: when an address is created, when one is - added, and in `ProxyTransport.check`. - - One thing is impossible, and it is impossible rather than unimplemented: a QUIC edge - carrying something that is **not HTTP/3**. What lets an HTTP/3 client reach a service - that speaks HTTP/1.1 is that the exchange is already rendered as HTTP/1.1 for the - chain, so sending that rendering on invents nothing. Opaque bytes on a QUIC stream - have no such form — there is no HTTP/1.1 spelling of "some bytes" — so an address - whose clients speak anything else has nothing that could be forwarded. - - Which is a fact about the **ALPN**, not about the protocol: a `quic` service offering - `h3`, which is the default and the usual case, is exactly as renderable as an `http` - one, and an instance that has set `FGEX_PROXY_QUIC_ALPN` to something else is the one - that cannot do this — on both. - """ - if str(upstream) == UPSTREAM.SAME: - return None - if str(proto) not in (L4.TLS, L4.QUIC, L4.HTTP): - return None - alpn = quic_alpn() - if edge_is_udp and "h3" not in alpn: - return ( - f"This instance offers QUIC clients {', '.join(alpn)}, and only HTTP/3 can be " - f"rendered to a service that speaks HTTP/1.1 — anything else on a QUIC stream " - f"is opaque bytes with no HTTP/1.1 form to forward. Let this address be " - f"forwarded as it arrives, or take its HTTP/3 edge away." - ) - return None - - class Service: """One protected endpoint and the transport in front of it.""" diff --git a/backend/modules/services/nftables.py b/backend/modules/services/nftables.py index d508228..24cffd5 100644 --- a/backend/modules/services/nftables.py +++ b/backend/modules/services/nftables.py @@ -18,8 +18,8 @@ engine dials the service as the client, two flows look almost identical: No address distinguishes them. Conntrack does: the first belongs to the intercepted, redirected connection, the second is a separate entry the engine opened itself. So -`ct status dnat` lets the first out, and everything else from a protected service is -marked home. That is what makes source preservation work even with the service on +`ct status dnat` lets the first out, and everything else from a protected service — and +anything answering a connection the engine opened, from whichever port — is marked home. That is what makes source preservation work even with the service on this very host — which is also the case that ruled tproxy out, since tproxy leaves no NAT entry to key off and so could never reach a local service at all. """ @@ -50,6 +50,16 @@ QUEUE_MARK_OUTPUT = 0x1338 #: growing except that every position costs two chains whether or not anyone uses it. MAX_CHAIN_POSITIONS = 8 +#: Where the NFQUEUE layer meets a connection this host opened to one of its own addresses +#: that somebody rewrote on the way — a port a container runtime publishes. Docker's DNAT +#: for this host's own traffic sits in `nat OUTPUT` at `dstnat` (-100) and sends it out +#: towards the container, so it never comes back in through prerouting where the queue +#: chains are, and neither does the container's answer: it arrives for a local socket +#: and goes to `input`. These run after that rewrite on the way out and before it is +#: undone on the way in (`srcnat`, 100), so both halves show the container's address and +#: a stream follower sees one stream. +LOCAL_QUEUE_PRIORITY = -90 + #: Where the proxy layer's redirect sits in the prerouting and output hooks. #: #: **Before `dstnat` (-100), and that is the whole point.** Docker and podman publish a @@ -193,6 +203,11 @@ def udp_relay_host(ip: str) -> str: #: `interface_addresses` — so the rule says which interface it belongs to itself. IFACE_COMMENT = "iface " +#: Written on a rule that matches its address through conntrack rather than in the packet +#: — `
` — for the same reason: `get()` reads what a rule matches to +#: know whose it is, and there is nothing in these for it to read. +ADDRESS_COMMENT = "address " + def interface_addresses(ip: str) -> list[str]: """Every address an interface carries right now, as a rule can match them. @@ -324,6 +339,15 @@ class FiregexTables(NFTableManager): def queue_output_chain(position: int) -> str: return f"fgex_queue_out_{position}" + # And the connections this host opens to a rewritten address: see LOCAL_QUEUE_PRIORITY. + @staticmethod + def queue_local_request_chain(position: int) -> str: + return f"fgex_queue_lreq_{position}" + + @staticmethod + def queue_local_reply_chain(position: int) -> str: + return f"fgex_queue_lrep_{position}" + # External transport: hand the traffic to a proxy the operator runs hijack_in_chain = "fgex_hijack_in" hijack_local_chain = "fgex_hijack_local" @@ -350,6 +374,10 @@ class FiregexTables(NFTableManager): -307 + position), _chain(self.queue_output_chain(position), "filter", "postrouting", 107 + position), + _chain(self.queue_local_request_chain(position), "filter", "output", + LOCAL_QUEUE_PRIORITY + position), + _chain(self.queue_local_reply_chain(position), "filter", "input", + LOCAL_QUEUE_PRIORITY + position), ) ], # Earlier than the queue chains, so traffic destined for somebody else's @@ -388,6 +416,43 @@ class FiregexTables(NFTableManager): } } }, + # The same, for a reply the socket lookup cannot place: one to a connection + # the engine opened towards an address somebody rewrote — a port a container + # runtime publishes, above all. The engine dials the published address as + # the client, Docker's DNAT sends it on to the container, and the container + # answers from its own address: `socket transparent` looks for a socket + # talking to *that* and finds none, so the answer was routed on to the real + # client, which reset it. A client on another host reached a Docker service + # through the proxy layer only after the dial timed out and was retried as + # this host — slowly, and with the client's address lost. Conntrack still + # knows whose connection it is, by the mark `fgex_route` copied onto it. + { + "add": { + "rule": { + "family": "inet", + "table": self.table_name, + "chain": self.divert_chain, + "expr": [ + { + "match": { + "op": "==", + "left": {"ct": {"key": "direction"}}, + "right": "reply", + } + }, + { + "match": { + "op": "==", + "left": {"ct": {"key": "mark"}}, + "right": PROXY_SELF_MARK, + } + }, + {"mangle": {"key": {"meta": {"key": "mark"}}, "value": PROXY_MARK}}, + {"accept": None}, + ], + } + } + }, _chain(self.nat_chain, "nat", "prerouting", NAT_PRIORITY), # Locally generated traffic never reaches the nat prerouting hook, and # a client on this host protecting a service on this host is exactly that. @@ -455,9 +520,8 @@ class FiregexTables(NFTableManager): } }, # Installed once. A packet on the intercepted (redirected) connection - # is the engine answering a client and has to leave normally; anything - # else from a protected service is a reply to the engine's own dial, - # and the per-service rule below marks it home. + # is the engine answering a client and has to leave normally; a reply to + # the engine's own dial is marked home by the two rules after this one. { "add": { "rule": { @@ -477,6 +541,64 @@ class FiregexTables(NFTableManager): } } }, + # **A reply to a connection the engine opened, from whatever port it + # comes — and nothing else.** A per-address rule used to do this, knowing + # the service by the address it is protected on. That is where the engine + # dials it only while the address *is* the service: one published on + # another port is dialled there, and an HTTP/3 edge in front of an + # HTTP/1.1 service is dialled over TCP, and their answers were routed out + # towards the client instead of home. It also matched the answers on + # connections the engine never saw, the ones open before the service + # started, and brought those home to nobody. Conntrack knows who opened a + # connection: the engine's mark is copied onto it on the way out and read + # back off everything that answers it. + { + "add": { + "rule": { + "family": "inet", + "table": self.table_name, + "chain": self.route_chain, + "expr": [ + { + "match": { + "op": "==", + "left": {"meta": {"key": "mark"}}, + "right": PROXY_SELF_MARK, + } + }, + {"mangle": {"key": {"ct": {"key": "mark"}}, + "value": PROXY_SELF_MARK}}, + ], + } + } + }, + { + "add": { + "rule": { + "family": "inet", + "table": self.table_name, + "chain": self.route_chain, + "expr": [ + { + "match": { + "op": "==", + "left": {"ct": {"key": "direction"}}, + "right": "reply", + } + }, + { + "match": { + "op": "==", + "left": {"ct": {"key": "mark"}}, + "right": PROXY_SELF_MARK, + } + }, + {"mangle": {"key": {"meta": {"key": "mark"}}, + "value": PROXY_MARK}}, + ], + } + } + }, ], # The whole table, which is firegex's alone. It used to be flushed chain by # chain in one batch, and nft refuses a batch outright if any object in it is @@ -517,6 +639,19 @@ class FiregexTables(NFTableManager): check=False, stderr=subprocess.DEVNULL, ) + def intact(self) -> bool: + """Whether what `init()` put in the kernel is still there. + + One small chain rather than the whole table, because this is asked every few + seconds: `init()` always leaves rules in the route chain, so a table that was + deleted (`nft flush ruleset`, which is how a Debian `/etc/nftables.conf` begins) + and one that was only emptied (`nft flush table`) both fail it. + """ + code, listed, _ = self.raw_cmd({"list": {"chain": { + "family": "inet", "table": self.table_name, "name": self.route_chain, + }}}) + return code == 0 and any("rule" in item for item in listed["nftables"]) + def reset(self): super().reset() # The set went with the table. @@ -709,8 +844,58 @@ class FiregexTables(NFTableManager): ], insert=True, )) + if not is_ip_parse(ip): + # A connection this host opens is not matched by interface, for the reason + # the proxy layer's output-hook rules give — and what these catch is one + # to an address, rewritten. + continue + # This host dialling one of these addresses when something rewrote it on the + # way out: a port Docker publishes, which is how a CTF service is usually + # reached. Recognised by where the connection was *going*, which conntrack + # keeps, because by now the packet itself says the container's address. + # Without these the request and the answer both went round every queue here, + # and a payload the filter refuses from anywhere else reached the service + # from this host untouched. + original = self._rewritten_to(ip, port, family, l4) + note = f"{ADDRESS_COMMENT}{l4} {int(port)} {ip}" + cmds.append(self._rule( + self.queue_local_request_chain(position), + original + self._ct_direction("original") + [ + self.COUNTER, + {"mangle": {"key": {"meta": {"key": "mark"}}, "value": QUEUE_MARK_INPUT}}, + queue, + ], + insert=True, comment=note, + )) + cmds.append(self._rule( + self.queue_local_reply_chain(position), + original + self._ct_direction("reply") + [ + {"mangle": {"key": {"meta": {"key": "mark"}}, "value": QUEUE_MARK_OUTPUT}}, + queue, + ], + insert=True, comment=note, + )) self.cmd(*cmds) + @staticmethod + def _rewritten_to(ip: str, port: int, family: str, l4: str) -> list: + """A connection that was headed for `ip:port` and had its destination rewritten.""" + return [ + {"match": {"op": "in", "left": {"ct": {"key": "status"}}, "right": "dnat"}}, + {"match": {"op": "==", "left": {"meta": {"key": "l4proto"}}, "right": l4}}, + {"match": {"op": "==", + "left": {"ct": {"key": f"{family} daddr", "dir": "original"}}, + "right": nftables_int_to_json(ip)}}, + {"match": {"op": "==", + "left": {"ct": {"key": "proto-dst", "dir": "original"}}, + "right": int(port)}}, + ] + + @staticmethod + def _ct_direction(direction: str) -> list: + return [{"match": {"op": "==", "left": {"ct": {"key": "direction"}}, + "right": direction}}] + def _add_external(self, srv: Service, addr: Address, ip: str, port: int, family: str, l4: str): """Put somebody else's proxy in the path, invisibly. @@ -783,14 +968,14 @@ class FiregexTables(NFTableManager): + self._match(ip, port, family, l4, "daddr", "dport") + [self.COUNTER, {"redirect": {"port": int(proxy_port)}}] ) + # Nothing per service on the way back. The engine's answers are brought home by + # the conntrack mark on the connections it opened (`init()`), whatever address + # and port they come from. A rule here used to mark everything leaving the + # service's address and port as well, which added nothing for the engine and + # caught the connections that were open before the service started: nothing + # redirected those, so their answers belonged to the client, and a client on + # another host stopped hearing back the moment protection began. cmds = [self._rule(self.nat_chain, redirect)] - cmds.append( - self._rule( - self.route_chain, - self._match(ip, port, family, l4, "saddr", "sport") - + [{"mangle": {"key": {"meta": {"key": "mark"}}, "value": PROXY_MARK}}], - ) - ) if not is_iface: cmds.append(self._rule(self.nat_output_chain, redirect)) else: @@ -836,6 +1021,8 @@ class FiregexTables(NFTableManager): chains = [ *[self.queue_input_chain(p) for p in range(MAX_CHAIN_POSITIONS)], *[self.queue_output_chain(p) for p in range(MAX_CHAIN_POSITIONS)], + *[self.queue_local_request_chain(p) for p in range(MAX_CHAIN_POSITIONS)], + *[self.queue_local_reply_chain(p) for p in range(MAX_CHAIN_POSITIONS)], self.hijack_in_chain, self.hijack_local_chain, self.hijack_out_chain, @@ -844,9 +1031,22 @@ class FiregexTables(NFTableManager): self.nat_output_chain, self.route_chain, ] - for rule in self.list_rules(tables=[self.table_name], chains=chains): + for rule in self.list_rules(tables=[self.table_name], chains=chains, family="inet"): try: expr = rule["expr"] + note = str(rule.get("comment") or "") + counter = next( + (e["counter"] for e in expr if isinstance(e, dict) and "counter" in e), + {}, + ) + if note.startswith(ADDRESS_COMMENT): + proto, port, ip_int = note[len(ADDRESS_COMMENT):].split(" ", 2) + res.append(InstalledRule( + chain=rule["chain"], handle=int(rule["handle"]), proto=proto, + port=int(port), ip_int=ip_int, + packets=counter.get("packets", 0), bytes_=counter.get("bytes", 0), + )) + continue # An intercept rule starts with the self-mark exclusion; every other # per-service rule starts with the address. The two rules init() # installs match on `socket` or on `ct`, and fall out here. @@ -869,13 +1069,8 @@ class FiregexTables(NFTableManager): # The comment does, and reading it back here is what lets the rest of # this file go on treating the rule as the interface's — including # `delete()`, which would otherwise never find it. - note = str(rule.get("comment") or "") if note.startswith(IFACE_COMMENT): ip_int = note[len(IFACE_COMMENT):] - counter = next( - (e["counter"] for e in expr if isinstance(e, dict) and "counter" in e), - {}, - ) res.append( InstalledRule( chain=rule["chain"], @@ -887,7 +1082,7 @@ class FiregexTables(NFTableManager): bytes_=counter.get("bytes", 0), ) ) - except (KeyError, TypeError, IndexError): + except (KeyError, TypeError, IndexError, ValueError): continue # a rule of a shape we did not write; not ours to touch return res @@ -912,7 +1107,9 @@ class FiregexTables(NFTableManager): # says. The engine already reports connections properly, so this reports nothing # there rather than a packet count that is off by the length of every flow. inbound = { - TRANSPORT.NFQUEUE: {self.queue_input_chain(0)}, + # Two chains, but never both for one packet: from elsewhere a request passes + # prerouting, and from this host to a rewritten address it passes only output. + TRANSPORT.NFQUEUE: {self.queue_input_chain(0), self.queue_local_request_chain(0)}, TRANSPORT.EXTERNAL: {self.hijack_in_chain, self.hijack_local_chain}, }.get(srv.transport, set()) targets = addresses if addresses is not None else srv.addresses @@ -923,11 +1120,16 @@ class FiregexTables(NFTableManager): total_bytes += rule.bytes return {"packets": packets, "bytes": total_bytes} - def delete(self, srv: Service, addresses: list[Address] | None = None): + def delete(self, srv: Service, addresses: list[Address] | None = None, + keep_guards: bool = False) -> set[tuple[str, int]]: """Take back the rules for these addresses, or for all of the service's. Passing a subset is how one address is removed from a running service: the others keep their rules, and the datapath keeps their connections. + + `keep_guards` leaves the engine's ports guarded and hands them back instead, for + `release_guards` once the engine has gone: one draining the connections it still + carries is listening on them until then. """ targets = addresses if addresses is not None else srv.addresses cmds = [ @@ -951,7 +1153,21 @@ class FiregexTables(NFTableManager): # so its port stays guarded until the engine goes; released earlier, the relay # would be reachable directly again with nothing steering traffic at it. if addresses is None: - for l4, port in self._guarded.pop(str(srv.id), set()): + guarded = self._guarded.pop(str(srv.id), set()) + if keep_guards: + return guarded + self.release_guards(guarded) + return set() + + def release_guards(self, ports: set[tuple[str, int]]) -> None: + """Stop guarding ports an engine no longer listens on. + + Except one some running service is guarding now: an engine that drained for + minutes gave its port back, and the kernel is free to have handed it to another. + """ + owned = set().union(*self._guarded.values()) if self._guarded else set() + for l4, port in ports: + if (l4, port) not in owned: self._unguard(l4, port) def _element(self, verb: str, l4: str, port: int) -> dict: diff --git a/backend/modules/services/pyworker.py b/backend/modules/services/pyworker.py index aa122f9..74c4932 100644 --- a/backend/modules/services/pyworker.py +++ b/backend/modules/services/pyworker.py @@ -271,6 +271,13 @@ class Filters: if ctx is None: ctx = {} exec(self.code, ctx, ctx) # noqa: S102 + # No open frame came for it, so this worker is taking over a connection + # already under way — new code, or one restarted after the last died — and + # its first chunk can be the middle of a message. The library then carries + # what it cannot read instead of judging half a message; see + # `_met_mid_message` in the HTTP model. + if connection not in self.endpoints: + ctx["__firegex_joined_late"] = True self.contexts[connection] = ctx return ctx diff --git a/backend/modules/services/transports.py b/backend/modules/services/transports.py index 755d31b..6840f1a 100644 --- a/backend/modules/services/transports.py +++ b/backend/modules/services/transports.py @@ -22,11 +22,12 @@ import asyncio import json import os import sys +import time import traceback from typing import NamedTuple from modules.services.models import (KIND, L4, PROTO, TRANSPORT, UPSTREAM, Filter, - Regex, quic_alpn, upstream_refusal) + Regex) from modules.services.nftables import (MAX_CHAIN_POSITIONS, NoRelayAddress, interface_addresses, service_at, udp_relay_host, udp_relay_key, udp_relay_slot) @@ -52,9 +53,40 @@ PYWORKER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pyworker.py #: both network layers. PYWORKER_EXCEPTION_MARK = "[exception] [filter]" -#: How long to wait for a binary to acknowledge a configuration push. +#: How long to wait for a binary to acknowledge a control command, or to say it started. ACK_TIMEOUT = 3 +#: How long a binary may take over one filter's worth of a configuration push. Loading a +#: Python filter runs its module body, which the code check lets take up to eight seconds +#: (`CHECK_TIMEOUT` in the router) — so a push held to `ACK_TIMEOUT` gave up on a file the +#: check had just accepted, and the answer that came afterwards was read as the answer to +#: whichever command was sent next. +LOAD_TIMEOUT = 10 + +#: How long a proxy engine taken out of service may go on carrying the connections it +#: already had. It is taken out of the rules at once — new connections go straight to the +#: service, or to the engine that replaced it — and it exits as soon as it reports none +#: left. Killed instead, which is what a stop used to do, every connection it carried +#: was cut: a download, a websocket, a request half answered. The ceiling is for the ones +#: that never close. +DRAIN_LIMIT = 600 +#: The engine reports its connections every two seconds (`STATS_INTERVAL` in `main.rs`); +#: a report sent before it was taken out of the rules cannot say it has none left. +DRAIN_FRESH_AFTER = 2.5 +#: How many engines of one service may be draining at once. A service restarted over and +#: over while clients hold connections open would otherwise keep one process per restart. +MAX_DRAINING = 3 + +#: Every engine taken out of service and still carrying connections, by service id. +_draining: dict[str, list["ProxyTransport"]] = {} + + +async def stop_draining() -> None: + """End every drain now: the rules that carry their connections home are going too.""" + for engines in list(_draining.values()): + for engine in list(engines): + await engine._end_drain(None) + class UnsupportedChain(Exception): """The chosen transport cannot host the chain that was asked of it.""" @@ -254,6 +286,17 @@ class Transport: async def stop(self) -> None: raise NotImplementedError + async def retire(self, keep_filtering: bool, then=None) -> None: + """Stop, sparing whatever connections this layer can spare. See `ProxyTransport`. + + Only the proxy layer has any to spare: it is the one that terminates them. A + queued service's connections never belonged to its processes, and a handed-off + one's belong to the operator's proxy — both go on when the rules come off. + """ + await self.stop() + if then: + then() + async def _kill(self): if self.process and self.process.returncode is None: self.process.kill() @@ -308,6 +351,21 @@ class Transport: return asyncio.create_task(run()) + def _abandon(self, process, what: str, command: str, waited: float) -> None: + """Give up on a datapath process that stopped answering its control channel. + + Killed rather than waited on further, because its answer may still come — and it + would then be read as the answer to the next command, and every command after that + would be told what the one before it got. Killed **without** being marked stopped, + so its watchdog reports it and the service is rebuilt from the saved configuration: + what was applied, if anything, cannot be known from here. + """ + if self.on_engine: + self.on_engine(f"[error] [backend] {what} did not acknowledge {command} within " + f"{waited:.0f}s; restarting it") + if process is not None and process.returncode is None: + process.kill() + async def _died_because(self, fallback: str) -> str: """What to tell the operator when the datapath would not come up. @@ -352,6 +410,9 @@ class _QueueStage: #: did not. Without the distinction every ordinary stop looks like a crash. self._stopped = False self._watchdog: asyncio.Task | None = None + #: What this process last acknowledged, so an edit elsewhere in the chain does + #: not send it the same filters again. See `reload`. + self._in_force: bytes | None = None async def start(self) -> int: if self.link.kind == KIND.PYFILTER: @@ -453,25 +514,48 @@ class _QueueStage: self._sock_ready.set() async def _pump_output(self): + import codecs + + # Whole lines only, decoded across reads: a line cut at a read boundary was two + # log entries, and a character cut there was two replacement marks. + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + pending = "" try: while True: data = await self.process.stdout.read(10 * 1024) if not data: - return - text = data.decode(errors="replace") - # The binary's own diagnostics share this pipe with the user's output, so - # the reason it gives for dying is read here or not at all — and without - # it a stage that would not start was reported as merely "would not - # start", with the sentence that said why sitting in the log. - for line in text.splitlines(): - if "[fatal]" in line: - self.owner._last_fatal = line.split("[fatal]", 1)[1].strip() - if self.owner.on_output: - self.owner.on_output(self.srv.id, text) + break + *lines, pending = (pending + decoder.decode(data)).split("\n") + self._forward_output(lines) except (asyncio.CancelledError, asyncio.IncompleteReadError): pass except Exception: traceback.print_exc() + if pending: + self._forward_output([pending]) + + def _forward_output(self, lines: list[str]) -> None: + """The binary's own diagnostics share this pipe with whatever the user's code prints. + + So both go through the classifier the other datapaths' stderr goes through, which + reads a line's tag for its level, a traceback as an error, and an untagged line as + the user's own output. They used to go in as the user's output, all of them: + `[error]` lines and the traceback of a filter that raised were grey text among the + prints, where a warning is looked for and not found. + """ + for line in lines: + # The reason it gives for dying is read here or not at all — and without it a + # stage that would not start was reported as merely "would not start", with + # the sentence that said why sitting in the log. + if "[fatal]" in line: + self.owner._last_fatal = line.split("[fatal]", 1)[1].strip() + text = "\n".join(lines) + if not text.strip(): + return + if self.owner.on_engine: + self.owner.on_engine(text) + elif self.owner.on_output: + self.owner.on_output(self.srv.id, text) # --- shared --------------------------------------------------------------- @@ -518,7 +602,16 @@ class _QueueStage: payload = self._regex_payload(link.regexes) else: payload = self._python_payload(link) + # Every stage is reloaded on every edit to the chain, and a new configuration is + # a clean slate for the connections under way: `cppregex` resets its matchers + # and `cpproxy` its filters' state. For the filter that was edited that is the + # point. For the others it cost every open connection its place — a request in + # the middle of an upload was read again from its second half — over a pattern + # changed in a filter they have nothing to do with. + if payload == self._in_force: + return await self._push(payload) + self._in_force = payload def _regex_payload(self, regexes: list[Regex]) -> bytes: """cppregex takes its patterns as hex codes, one per direction. @@ -586,14 +679,25 @@ class _QueueStage: else: raise Exception("the nfqueue binary is not running") try: - ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT) + ok, detail = await asyncio.wait_for(self._ack, timeout=LOAD_TIMEOUT) except asyncio.TimeoutError: - await self.stop() + # Whether it took the new filters or is still at it cannot be known now, and + # an answer arriving later would be read as the answer to the next push. So + # it is killed *without* being marked stopped: the watchdog reports it and + # the service is rebuilt from what the database says — which, after an edit + # that was refused, is what it said before. At start the caller stops every + # stage anyway. + self.owner._abandon(self.process, f"the {self.link.filter.name} interceptor", + "the filters", LOAD_TIMEOUT) raise Exception("the nfqueue binary did not acknowledge the filters") finally: self._ack = None if not ok: - await self.stop() + # Not fatal, and it must not be made so: both binaries answer `ACK FAIL` with + # the filters they had still in force. This used to stop the stage, which + # during an edit left a service reported active with nothing reading its + # queue — unfiltered where it fails open, silent where it does not — and + # stopped on purpose, so the watchdog had nothing to say about it. raise Exception(f"the nfqueue binary rejected the filters: {detail}") async def stop(self) -> None: @@ -794,14 +898,6 @@ class ProxyTransport(Transport): @classmethod def check(cls, srv, chain: list[ChainLink]) -> None: super().check(srv, chain) - # Asked here as well as where each was set, because an address added afterwards - # can be one this instance cannot honour. - for addr in srv.addresses: - refusal = upstream_refusal( - srv.proto, addr.upstream, L4.l4_of(addr.edge) == L4.UDP - ) - if refusal: - raise UnsupportedChain(refusal) if srv.carries(L4.UDP): from utils import get_interface_ips, is_ip_parse # Only the addresses that need a relay of their own. On an `http` service @@ -831,6 +927,11 @@ class ProxyTransport(Transport): #: did not — the same distinction the NFQUEUE stages make. self._stopped = False self._watchdog: asyncio.Task | None = None + #: When the engine last reported its counters, for a drain to know that `live` + #: was counted after the engine was taken out of the rules. + self._stats_at = 0.0 + self._drain: asyncio.Task | None = None + self._after_drain = None def _published(self) -> list[str]: """What this engine is told about its TCP addresses, one entry each. @@ -944,21 +1045,13 @@ class ProxyTransport(Transport): "FGEX_PROXY_TLS": "1", "FGEX_PROXY_TLS_OPTIONAL": "1", "FGEX_PROXY_QUIC": "1", - "FGEX_PROXY_QUIC_ALPN": ",".join(quic_alpn()), } if str(self.srv.proto) == L4.QUIC: return { **material, "FGEX_PROXY_QUIC": "1", - # What the engine offers the service, in this order. It cannot be asked - # of the client the way the TLS path asks: a QUIC ClientHello arrives - # inside an encrypted Initial whose processing *is* the handshake, so - # there is nothing to hold it at and nothing to read before answering. - # `h3` is what a QUIC service speaks nine times in ten; an instance in - # front of something else sets this in its own environment, the same way - # the filter deadline beside it is set. Per service it would be a column, - # and nothing yet has wanted one. - "FGEX_PROXY_QUIC_ALPN": ",".join(quic_alpn()), + # No protocol list: the engine offers the service what each client + # offered, read off the client's first packet — see `quic_hello.rs`. } return material @@ -1135,13 +1228,19 @@ class ProxyTransport(Transport): # to an engine that was not there to apply it. The watchdog restarts a dead # engine from what the database holds. raise Exception("the proxy engine is not running") + # The engine answers once every Python filter of the chain has loaded, one after + # the other, so the wait grows with them. + python = sum(1 for link in chain + if link.filter.active and link.kind == KIND.PYFILTER and link.code_path) + waited = LOAD_TIMEOUT * (1 + python) async with self._cmd_lock: self._ack = asyncio.get_running_loop().create_future() self.process.stdin.write((self._payload(chain) + "\n").encode()) await self.process.stdin.drain() try: - ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT) + ok, detail = await asyncio.wait_for(self._ack, timeout=waited) except asyncio.TimeoutError: + self._abandon(self.process, "the proxy engine", "the chain", waited) raise Exception("the proxy engine did not acknowledge the chain") finally: self._ack = None @@ -1181,6 +1280,7 @@ class ProxyTransport(Transport): try: ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT) except asyncio.TimeoutError: + self._abandon(self.process, "the proxy engine", "ADD_UDP", ACK_TIMEOUT) raise Exception(f"the proxy engine did not acknowledge ADD_UDP for {target}") finally: self._ack = None @@ -1221,6 +1321,7 @@ class ProxyTransport(Transport): try: ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT) except asyncio.TimeoutError: + self._abandon(self.process, "the proxy engine", line.split()[0], ACK_TIMEOUT) raise Exception(f"the proxy engine did not acknowledge {line.split()[0]}") finally: self._ack = None @@ -1253,6 +1354,7 @@ class ProxyTransport(Transport): key, _, value = token.partition("=") if value.isdigit(): self.counters[key] = int(value) + self._stats_at = time.monotonic() # Only the increase, and only when there is one: the engine counts # from zero every time it starts, so handing the absolute number up # would reset the durable trace on every restart of the service. @@ -1281,6 +1383,103 @@ class ProxyTransport(Transport): # reads it is still running would be a key removed from under a restart. self._clear_crypto_material() + async def retire(self, keep_filtering: bool, then=None) -> None: + """Take the engine out of service without cutting the connections it carries. + + The caller has already taken it out of the rules, so nothing new reaches it; what + conntrack already sent here keeps coming here, and is carried until it closes. + `keep_filtering` is a restart's answer — the connections go on under the filters + they started with — and a stop's is to carry them unfiltered: stopping is how an + operator takes a filter that is hurting the service out of the way, and a filter + that went on judging the connections already open would not be out of the way. + + `then` runs once the engine has gone, which is when its ports can stop being + guarded: until then a relay would be reachable directly with nothing steering + traffic at it. + """ + self._stopped = True + if self._watchdog and self._watchdog is not asyncio.current_task(): + self._watchdog.cancel() + self._watchdog = None + self.port = None + self._after_drain = then + # The engine read these when it started and never again, and a restart writes + # its own copies for the engine replacing this one: removed later, they would be + # the new engine's. + self._clear_crypto_material() + if not self.process or self.process.returncode is not None: + await self._end_drain(None) + return + if not keep_filtering: + try: + await self.reload([]) + except Exception: + # Not answering is `_abandon`'s to deal with, and it already has. + pass + engines = _draining.setdefault(self.srv.id, []) + engines.append(self) + while len(engines) > MAX_DRAINING: + oldest = engines[0] + await oldest._end_drain( + f"[warn] [drain] {oldest.counters.get('live', 0)} connection(s) carried " + f"from before an earlier restart were closed: {MAX_DRAINING} engines were " + f"already draining") + self._drain = asyncio.create_task(self._wait_out(keep_filtering, time.monotonic())) + + async def _wait_out(self, kept_filtering: bool, began: float) -> None: + told = False + while self.process and self.process.returncode is None: + await asyncio.sleep(0.5) + fresh = self._stats_at >= began + DRAIN_FRESH_AFTER + live = self.counters.get("live", 0) + if fresh and live == 0: + if told: + await self._end_drain("[info] [drain] the connections from before " + "have all closed") + else: + await self._end_drain(None) + return + if fresh and not told: + told = True + self._say( + f"[info] [drain] {live} connection(s) were open through the engine; " + f"they carry on " + f"{'under the filters they started with' if kept_filtering else 'unfiltered'}" + f" until they close, for up to {DRAIN_LIMIT // 60} minutes") + if time.monotonic() - began >= DRAIN_LIMIT: + await self._end_drain( + f"[warn] [drain] {live} connection(s) from before were still open after " + f"{DRAIN_LIMIT // 60} minutes and were closed") + return + await self._end_drain(None) + + async def _end_drain(self, said: str | None) -> None: + engines = _draining.get(self.srv.id, []) + if self in engines: + engines.remove(self) + if not engines: + _draining.pop(self.srv.id, None) + if self._drain and self._drain is not asyncio.current_task(): + self._drain.cancel() + self._drain = None + if said: + self._say(said) + for task in (self._reader_task, self._stderr_pump): + if task: + task.cancel() + self._reader_task = self._stderr_pump = None + await self._kill() + then, self._after_drain = self._after_drain, None + if then: + try: + then() + except Exception: + traceback.print_exc() + + def _say(self, line: str) -> None: + if self.on_engine: + self.on_engine(line) + class ExternalTransport(Transport): """Hand the traffic to a proxy the operator runs themselves. diff --git a/backend/proxysrc/README.md b/backend/proxysrc/README.md index 0c49b4c..e39910e 100644 --- a/backend/proxysrc/README.md +++ b/backend/proxysrc/README.md @@ -227,13 +227,12 @@ terminates (`quic::QuicManager`) instead of something that forwards (`udp::UdpMa `relays::Relays` is the two-armed enum both the startup list and the control channel go through. -- **The ALPN is asked of the service**, which is the one thing that could not be kept from - the TLS path: a QUIC ClientHello arrives inside an encrypted Initial whose processing - *is* the handshake, and quinn offers nothing to hold it at. So the order is reversed — - the service is dialled first with the candidate list (`FGEX_PROXY_QUIC_ALPN`, default - `h3`) and the client is advertised exactly the one it chose. The invariant survives; what - is lost is knowing in advance whether the client would take it, which shows up as a - failed handshake with a log line naming the protocol. +- **The ALPN is mirrored, as on the TLS path.** quinn offers nothing to hold a ClientHello + at, but an Initial packet's keys come from the connection ID it carries in the clear, so + `quic_hello.rs` reads the client's list off the socket before the handshake is answered. + The service is dialled first offering exactly that list, and the client is advertised + the one protocol the service chose — nothing to configure, whatever the service speaks. + A hello that cannot be read falls back to offering `h3`, and says so. - **One stream is one connection to the chain**, with its own sessions and its own module globals, released when the stream ends. A filter's state follows a stream of bytes from start to end, which is what a QUIC stream is and what a QUIC connection is not. @@ -486,7 +485,6 @@ outliving its control channel matters. | `FGEX_PROXY_TLS` / `_TLS_OPTIONAL` | terminate TLS on the TCP listener; `_OPTIONAL` means only for the connections that start a handshake | | `FGEX_PROXY_TLS_CERT` / `_TLS_KEY` | **paths**, never the material | | `FGEX_PROXY_QUIC` | bind the relays with a QUIC endpoint instead of a datagram socket | -| `FGEX_PROXY_QUIC_ALPN` | what to offer the service, in order. Default `h3` | | `FGEX_PROXY_MAX_CONNECTIONS` | 0 is no limit | | `FGEX_PROXY_OVER_LIMIT_FORWARD` | forward what does not fit, unfiltered, instead of refusing it | | `FGEX_PROXY_FIRST_BYTE_TIMEOUT` | seconds; 0 is off | diff --git a/backend/proxysrc/src/control.rs b/backend/proxysrc/src/control.rs index 4b81860..71f4802 100644 --- a/backend/proxysrc/src/control.rs +++ b/backend/proxysrc/src/control.rs @@ -11,16 +11,10 @@ use tokio::io::{AsyncBufReadExt, BufReader}; use crate::filter::{ChainHandle, FilterChain}; use crate::proxy::{Edge, Onward, Published, Targets}; use crate::relays::Relays; -use crate::rules::parse_ruleset; +use crate::rules::parse_ruleset_reusing; fn ack(ok: bool, detail: &str) { - if ok { - println!("ACK OK"); - } else { - println!("ACK FAIL {detail}"); - } - use std::io::Write; - let _ = std::io::stdout().flush(); + crate::report::reply(if ok { "ACK OK".to_string() } else { format!("ACK FAIL {detail}") }); } /// Apply rulesets and handle control commands until stdin closes. @@ -80,7 +74,7 @@ pub async fn serve_stdin( match target_str.parse::() { Ok(upstream) => match relays.add_relay(upstream, onward).await { Ok(port) => { - println!("UDP {upstream}|{} {port}", onward.word()); + crate::report::reply(format!("UDP {upstream}|{} {port}", onward.word())); ack(true, ""); } Err(e) => { @@ -165,8 +159,9 @@ pub async fn serve_stdin( // runtime with every relay: with one worker thread, which is what a one-core // host is given, the service carried nothing while a reload was loading a file. let line = trimmed.to_string(); + let previous = chain.current().filters(); let prepared = tokio::task::spawn_blocking(move || { - parse_ruleset(&line).and_then(|filters| { + parse_ruleset_reusing(&line, &previous).and_then(|filters| { for filter in &filters { filter.prepare()?; } diff --git a/backend/proxysrc/src/filter.rs b/backend/proxysrc/src/filter.rs index e49c0d7..1d2290d 100644 --- a/backend/proxysrc/src/filter.rs +++ b/backend/proxysrc/src/filter.rs @@ -8,7 +8,8 @@ use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; -use std::time::Duration; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; /// Which half of the connection a chunk came from. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -189,6 +190,18 @@ pub trait Filter: Send + Sync + 'static { /// process of its own — has no other way to learn that it can let go, and would /// otherwise grow for as long as the service runs. fn connection_closed(&self, _connection: ConnectionId) {} + + /// What makes this filter the same filter in the next ruleset, if anything does. + /// + /// A ruleset is rebuilt whole on every edit, and a filter that keeps per-connection + /// state outside its sessions loses all of it when it is rebuilt: the Python worker + /// is a process, and a new one meets every open connection halfway. A filter that + /// answers here, with a key that changes whenever its configuration does, is carried + /// over into the new chain instead when the key matches. `None` — the default — is + /// always rebuilt. + fn reuse_key(&self) -> Option<&str> { + None + } } /// One filter's state for one direction of one connection. @@ -250,11 +263,30 @@ pub struct ChainStats { /// because zero means "no sessions opened yet". static CHAIN_GENERATION: AtomicU64 = AtomicU64::new(1); +/// How long a filter that kept missing its deadline sits out before it is asked again. +/// +/// Slow is not broken. A Python filter misses its deadline when exchanges queue behind one +/// another, which is what a flood does — the one moment its protection is wanted — and +/// disabled for the life of the chain it stayed off until somebody next edited a rule, +/// with one log line to say so. So it is tried again after a pause, and if it is still +/// slow it loses its say again after the same few misses: a filter that genuinely never +/// returns strands a few threads per pause rather than one per chunk. A panic is broken +/// code, and stays out until the chain is replaced. +pub const REARM_AFTER: Duration = Duration::from_secs(30); + +/// Milliseconds on a monotonic clock, never zero, so zero can mean "not scheduled". +fn monotonic_ms() -> u64 { + static START: OnceLock = OnceLock::new(); + START.get_or_init(Instant::now).elapsed().as_millis() as u64 + 1 +} + struct Slot { filter: Arc, /// Set once the filter has misbehaved enough to lose its say. disabled: AtomicBool, consecutive_timeouts: AtomicU64, + /// When a filter that timed out is asked again (`monotonic_ms`), or 0 for never. + rearm_at_ms: AtomicU64, } /// An immutable set of filters. Swapping the whole chain is how configuration @@ -266,6 +298,10 @@ pub struct FilterChain { generation: u64, deadline: Duration, max_consecutive_timeouts: u64, + /// How long a filter that timed out sits out. `REARM_AFTER` but for tests. + rearm_after: Duration, + /// The earliest `rearm_at_ms` of any slot, or 0: what lets the hot path skip the clock. + next_rearm_ms: AtomicU64, /// Trips when no filter is left to consult, making the chain a pure relay. degraded: AtomicBool, pub stats: Arc, @@ -276,6 +312,12 @@ impl FilterChain { Self::with_policy(filters, deadline, 3) } + /// The filters this chain consults, in order — for the next ruleset to carry over + /// the ones it has not changed (`Filter::reuse_key`). + pub fn filters(&self) -> Vec> { + self.slots.iter().map(|slot| slot.filter.clone()).collect() + } + pub fn with_policy( filters: Vec>, deadline: Duration, @@ -299,16 +341,26 @@ impl FilterChain { filter, disabled: AtomicBool::new(false), consecutive_timeouts: AtomicU64::new(0), + rearm_at_ms: AtomicU64::new(0), }) .collect(), generation: CHAIN_GENERATION.fetch_add(1, Ordering::Relaxed), deadline, max_consecutive_timeouts, + rearm_after: REARM_AFTER, + next_rearm_ms: AtomicU64::new(0), degraded, stats: Arc::new(ChainStats::default()), } } + /// The same chain, with filters that time out asked again after `after` rather than + /// `REARM_AFTER`. + pub fn rearming_after(mut self, after: Duration) -> Self { + self.rearm_after = after; + self + } + pub fn empty() -> Self { Self::new(Vec::new(), Duration::from_millis(50)) } @@ -333,11 +385,26 @@ impl FilterChain { return; // already out of the loop, do not double count } self.stats.filters_disabled.fetch_add(1, Ordering::Relaxed); - eprintln!( - "[warn] [filter] '{}' disabled after {:?}: traffic keeps flowing without it", - slot.filter.name(), - reason - ); + match reason { + DisableReason::TimedOut => { + let at = monotonic_ms() + self.rearm_after.as_millis() as u64; + slot.rearm_at_ms.store(at, Ordering::Relaxed); + let _ = self.next_rearm_ms.fetch_update(Ordering::Relaxed, Ordering::Relaxed, + |next| (next == 0 || at < next).then_some(at)); + eprintln!( + "[warn] [filter] '{}' kept missing its {:?} deadline: traffic keeps \ + flowing without it, and it is asked again in {}s", + slot.filter.name(), + self.deadline, + self.rearm_after.as_secs(), + ); + } + DisableReason::Panicked => eprintln!( + "[warn] [filter] '{}' panicked: traffic keeps flowing without it until the \ + filters are next applied — save a change to them, or restart the service", + slot.filter.name(), + ), + } if self .slots .iter() @@ -367,6 +434,38 @@ impl FilterChain { /// Run the chain over one chunk. /// + /// Put back the filters whose pause is over. See `REARM_AFTER`. + fn rearm(&self) { + let now = monotonic_ms(); + let mut next = 0; + for slot in &self.slots { + let at = slot.rearm_at_ms.load(Ordering::Relaxed); + if at == 0 { + continue; + } + if now < at { + next = if next == 0 { at } else { next.min(at) }; + continue; + } + // Whoever wins this puts the filter back; everybody else sees it done. + if slot + .rearm_at_ms + .compare_exchange(at, 0, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + slot.consecutive_timeouts.store(0, Ordering::Relaxed); + slot.disabled.store(false, Ordering::Relaxed); + self.degraded.store(false, Ordering::Relaxed); + eprintln!( + "[info] [filter] '{}' is back in the chain after sitting out {}s", + slot.filter.name(), + self.rearm_after.as_secs(), + ); + } + } + self.next_rearm_ms.store(next, Ordering::Relaxed); + } + /// Any way a filter can fail — panic, or simply never returning — resolves to /// `Accept` for the chunk in flight. The chunk is never held hostage by the /// filter's failure. @@ -376,6 +475,10 @@ impl FilterChain { data: &[u8], sessions: &mut ChainSessions, ) -> Verdict { + let due = self.next_rearm_ms.load(Ordering::Relaxed); + if due != 0 && monotonic_ms() >= due { + self.rearm(); + } if self.is_bypassed() { self.stats.bypassed_chunks.fetch_add(1, Ordering::Relaxed); return Verdict::Accept; @@ -418,9 +521,7 @@ impl FilterChain { Ok(Verdict::Reject(by)) => { self.stats.rejected.fetch_add(1, Ordering::Relaxed); let id = by.clone().unwrap_or_else(|| slot.filter.name().to_string()); - println!("BLOCKED {id}"); - use std::io::Write; - let _ = std::io::stdout().flush(); + crate::report::blocked(&id); return Verdict::Reject(by); } Err(_) => { @@ -456,9 +557,7 @@ impl FilterChain { // The backend attributes the block to a rule by this id, // the same way cppregex reports `BLOCKED `. let id = by.clone().unwrap_or_else(|| slot.filter.name().to_string()); - println!("BLOCKED {id}"); - use std::io::Write; - let _ = std::io::stdout().flush(); + crate::report::blocked(&id); return Verdict::Reject(by); } } diff --git a/backend/proxysrc/src/lib.rs b/backend/proxysrc/src/lib.rs index 7bae667..0a6bdb5 100644 --- a/backend/proxysrc/src/lib.rs +++ b/backend/proxysrc/src/lib.rs @@ -15,6 +15,7 @@ pub mod http1; pub mod hyperscan; pub mod proxy; pub mod quic; +pub mod quic_hello; pub mod pyworker; pub mod rules; pub mod spec; @@ -22,6 +23,7 @@ pub mod capture; pub mod tls; pub mod transparent; pub mod relays; +pub mod report; pub mod udp; pub use filter::{ diff --git a/backend/proxysrc/src/main.rs b/backend/proxysrc/src/main.rs index 4d0084e..4a3453e 100644 --- a/backend/proxysrc/src/main.rs +++ b/backend/proxysrc/src/main.rs @@ -70,28 +70,6 @@ fn read_material() -> Result, String> { } } -/// The protocols offered to the service, in the operator's order. -/// -/// Offered to the *service*, which is what makes the list needed at all: the client's own -/// list arrives inside an encrypted Initial and cannot be read before answering, so this -/// is what firegex has to go on. `h3` alone by default, because that is what a QUIC -/// service is nine times in ten; anything else — a game protocol, a CTF's own — is named -/// here or its handshake fails saying so. -fn quic_alpn() -> Vec> { - let raw = std::env::var("FGEX_PROXY_QUIC_ALPN").unwrap_or_default(); - let listed: Vec> = raw - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(|s| s.as_bytes().to_vec()) - .collect(); - if listed.is_empty() { - vec![b"h3".to_vec()] - } else { - listed - } -} - fn build_tls(material: Option<&(String, String)>, optional: bool) -> Result { let server = match material { Some((cert, key)) => Some(fgex_proxy::tls::server_config(cert, key)?), @@ -383,9 +361,7 @@ async fn run() { // pair of numbers whichever half of it the traffic arrived on. let counters = proxy.stats(); - println!("PORT {}", bound.port()); - use std::io::Write; - let _ = std::io::stdout().flush(); + fgex_proxy::report::reply(format!("PORT {}", bound.port())); eprintln!( "[info] [main] listening on {bound} -> {upstream_raw} \ (spoof_source={spoof_source})" @@ -405,8 +381,7 @@ async fn run() { exit(2); } }; - let alpn = quic_alpn(); - let setup = match QuicSetup::build(cert, key, alpn.clone()) { + let setup = match QuicSetup::build(cert, key) { Ok(setup) => Arc::new(setup), Err(e) => { eprintln!("[fatal] [main] {e}"); @@ -414,11 +389,8 @@ async fn run() { } }; eprintln!( - "[info] [main] QUIC terminated here; offering the service {}", - alpn.iter() - .map(|p| String::from_utf8_lossy(p).into_owned()) - .collect::>() - .join(", ") + "[info] [main] QUIC terminated here; each client's own protocols are offered \ + to the service" ); Relays::Quic(QuicManager::new( chain_handle.clone(), @@ -474,9 +446,7 @@ async fn run() { // With what it speaks onward, because that is half of what names a // relay: two addresses sending to one service port can want different // answers, and each gets a relay of its own. - println!("UDP {upstream}|{} {port}", onward.word()); - use std::io::Write; - let _ = std::io::stdout().flush(); + fgex_proxy::report::reply(format!("UDP {upstream}|{} {port}", onward.word())); } Err(e) => { eprintln!("[fatal] [main] cannot bind a UDP relay for {upstream}: {e}"); @@ -507,15 +477,16 @@ async fn run() { // the backend parses this line as key/value pairs, so a new number costs no // protocol. `over_limit` is cumulative on purpose — a service that hit the // wall once an hour ago still says so, which is the whole point of a trace. - println!( + // Dropped rather than waited for when the backend is behind: the next one + // carries the same cumulative numbers two seconds later. + fgex_proxy::report::event(format!( "STATS seen={} refused={} live={} over_limit={} no_first_byte={}", counters.accepted.load(std::sync::atomic::Ordering::Relaxed), counters.closed_by_filter.load(std::sync::atomic::Ordering::Relaxed), counters.live.load(std::sync::atomic::Ordering::Relaxed), counters.over_limit.load(std::sync::atomic::Ordering::Relaxed), counters.no_first_byte.load(std::sync::atomic::Ordering::Relaxed), - ); - let _ = std::io::stdout().flush(); + )); } }); diff --git a/backend/proxysrc/src/pyworker.rs b/backend/proxysrc/src/pyworker.rs index 318cd4b..8cfc1d1 100644 --- a/backend/proxysrc/src/pyworker.rs +++ b/backend/proxysrc/src/pyworker.rs @@ -114,6 +114,9 @@ pub struct PyWorkerRule { /// chunk, which runs where blocking is allowed and keeps an open ahead of the data. pending: Mutex>, pub stats: std::sync::Arc, + /// Everything that would make a new worker behave differently from this one; see + /// `Filter::reuse_key`. `None` when the file could not be read to say so. + reuse_key: Option, } impl PyWorkerRule { @@ -133,9 +136,39 @@ impl PyWorkerRule { child: Mutex::new(None), pending: Mutex::new(Vec::new()), stats: std::sync::Arc::new(WorkerStats::default()), + reuse_key: None, } } + /// What this rule would be built from, as one string: the id, the command, which + /// functions are on, the deadline and the code itself. The same string means a new + /// worker would run exactly what this one runs, so it is kept instead — with the + /// state it holds for every open connection. The code is read rather than trusted to + /// its path, because saving a filter rewrites the same file. + pub fn key_for( + id: &str, + command: &[String], + code_path: &str, + enabled: &Option>, + deadline: Duration, + ) -> Option { + use std::hash::{Hash, Hasher}; + let code = std::fs::read(code_path).ok()?; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + code.hash(&mut hasher); + Some(format!( + "{id}\0{command:?}\0{code_path}\0{enabled:?}\0{}\0{:016x}", + deadline.as_millis(), + hasher.finish() + )) + } + + /// Answer to `key` from `Filter::reuse_key`. + pub fn reusable_as(mut self, key: Option) -> Self { + self.reuse_key = key; + self + } + fn spawn(&self) -> Result { let (program, args) = self .command @@ -343,6 +376,9 @@ impl Filter for PyWorkerRule { fn prepare(&self) -> Result<(), String> { self.warm_up() } + fn reuse_key(&self) -> Option<&str> { + self.reuse_key.as_deref() + } fn inspect(&self, ctx: &FilterCtx<'_>) -> Verdict { match self.exchange(ctx.connection, ctx.direction, ctx.chunk) { Ok(verdict) => verdict, diff --git a/backend/proxysrc/src/quic.rs b/backend/proxysrc/src/quic.rs index cf44d06..556365a 100644 --- a/backend/proxysrc/src/quic.rs +++ b/backend/proxysrc/src/quic.rs @@ -12,16 +12,15 @@ //! per address and for the same reason — `SO_ORIGINAL_DST` is TCP and SCTP only, so a //! single listener could not recover where a datagram was headed. //! -//! **The ALPN is asked of the service, not of the client**, which is the one place this -//! differs from the TLS path beside it. There the ClientHello is held open -//! (`LazyConfigAcceptor`), the service is asked, and the client is told what the service -//! picked. In QUIC the ClientHello arrives inside an encrypted Initial whose processing -//! *is* the handshake, and there is nothing to hold it at. So the order is reversed: the -//! service is offered the candidates, and the client is told the one thing the service -//! agreed to. The invariant that matters is unchanged — the client is never told a -//! protocol the service did not choose — and what is lost is knowing in advance whether -//! the client would have accepted it. When it does not, the handshake fails and says so, -//! rather than a protocol being quietly broken. +//! **The ALPN is mirrored, exactly as over TLS.** The client's ClientHello travels inside +//! an Initial packet, but Initial keys come from a connection ID the client sends in the +//! clear, so [`crate::quic_hello`] reads the list the client offered off the socket before +//! the handshake is answered. The service is dialled offering exactly that list, and the +//! client is told the one protocol the service chose. It used to be a candidate list the +//! operator wrote down per service, `h3` unless told otherwise, because the Initial was +//! taken to be unreadable: a QUIC service speaking anything but HTTP/3 was unreachable +//! until somebody found the setting. When a hello cannot be read — a QUIC version this +//! does not know, say — `h3` is offered and the log says so. use std::collections::HashMap; use std::io; @@ -34,6 +33,7 @@ use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig}; use quinn::{Connection, Endpoint, Incoming, VarInt}; use crate::capture::{Capture, Tap}; +use crate::quic_hello::{Hellos, SniffingSocket}; use crate::filter::{ next_connection_id, ChainHandle, ChainSessions, ConnectionId, ConnectionMeta, Direction, FilterChain, Verdict, L4, @@ -63,28 +63,37 @@ const IDLE: Duration = Duration::from_secs(60); /// that sends faster than the chain judges would be buying memory in this process. const DATAGRAM_BUFFER: usize = 256 * 1024; +/// How long a client's hello may take to arrive whole, when it spans more than one +/// datagram. Past it, `h3` is offered and the log says the hello could not be read. +const HELLO_WAIT: Duration = Duration::from_millis(1500); + +/// What is offered to a service when the client's own list could not be read. +const FALLBACK_ALPN: &[u8] = b"h3"; + /// Everything a QUIC service needs decided once, before any client arrives. +/// +/// No protocols among it: those are each client's own, and each connection's accepting +/// and dialling configurations are built from them — a clone of a rustls config and a +/// cipher-suite lookup, which is nothing beside the two handshakes they are for, and the +/// same trade the TLS path makes per connection for the same reason. pub struct QuicSetup { - /// One accept-side configuration per protocol the service might pick, built up front - /// rather than per connection: the candidate list is short and known, and a rustls - /// config clone plus a cipher-suite lookup on the accept path is work done while a - /// client is waiting. - answering: HashMap, Arc>, - /// For the case the service agreed to nothing. QUIC requires ALPN, so this is a - /// configuration that will refuse — kept so the refusal is the handshake's, with a - /// reason, rather than a panic here. + /// The service's certificate, answering. Cloned per connection to advertise the one + /// protocol the service chose. + server: Arc, + /// Towards the service. Cloned per connection to offer what the client offered. + client: rustls::ClientConfig, + /// Without datagrams, for HTTP/3; with them, for everything else. See `build`. + streams_only: Arc, + with_datagrams: Arc, + /// What the endpoint answers with before a service has been asked: only ever used + /// for connections this relay turns away, which need a configuration to be turned + /// away *by*. It advertises nothing, so nothing is agreed on it. silent: Arc, - /// Offered to the service, in the operator's order. - client: quinn::ClientConfig, - candidates: Vec>, } impl QuicSetup { - /// Build both edges from the service's certificate and the protocols it may speak. - pub fn build(cert_pem: &str, key_pem: &str, candidates: Vec>) -> Result { - if candidates.is_empty() { - return Err("a QUIC service needs at least one ALPN protocol".to_string()); - } + /// Build both edges from the service's certificate. + pub fn build(cert_pem: &str, key_pem: &str) -> Result { // Two of them, differing in one advertisement. **HTTP/3 connections do not carry // datagrams here**, and the honest way to say that is not to advertise the // extension on them: a peer then knows from the handshake and falls back, instead @@ -95,58 +104,52 @@ impl QuicSetup { // far side. Everything that is not h3 has no such coupling and carries them. let streams_only = transport_config(false); let with_datagrams = transport_config(true); - let base = crate::tls::server_config(cert_pem, key_pem)?; - - let mut answering = HashMap::new(); - for protocol in &candidates { - let transport = if protocol == b"h3" { &streams_only } else { &with_datagrams }; - answering.insert( - protocol.clone(), - Arc::new(server_config(&base, &[protocol.clone()], transport)?), - ); - } - let silent = Arc::new(server_config(&base, &[], &streams_only)?); - - let mut rustls_client = crate::tls::quic_client_config()?; - rustls_client.alpn_protocols = candidates.clone(); - let crypto = QuicClientConfig::try_from(rustls_client) - .map_err(|e| format!("cannot configure QUIC towards the service: {e}"))?; - let mut client = quinn::ClientConfig::new(Arc::new(crypto)); - // The service is dialled before it has said which protocol it speaks, so this - // edge is decided by the candidate list rather than by the answer: with h3 the - // only candidate there is nothing a datagram could be for, and advertising it - // upstream would invite what this end would then have to drop. - client.transport_config(if candidates.iter().all(|p| p == b"h3") { - Arc::clone(&streams_only) - } else { - Arc::clone(&with_datagrams) - }); - + let server = crate::tls::server_config(cert_pem, key_pem)?; + let silent = Arc::new(server_config(&server, &[], &streams_only)?); Ok(Self { - answering, + server, + client: crate::tls::quic_client_config()?, + streams_only, + with_datagrams, silent, - client, - candidates, }) } - /// The configuration that advertises the one protocol the service agreed to. - fn answering_with(&self, agreed: Option<&[u8]>) -> Arc { - agreed - .and_then(|p| self.answering.get(p)) - .map(Arc::clone) - .unwrap_or_else(|| Arc::clone(&self.silent)) + fn transport_for(&self, protocols: &[Vec]) -> &Arc { + if protocols.iter().all(|p| p == b"h3") { + &self.streams_only + } else { + &self.with_datagrams + } + } + + /// The configuration that advertises the one protocol the service agreed to — or + /// nothing, when it agreed to nothing, so the refusal is the handshake's with a reason. + fn answering_with(&self, agreed: Option<&[u8]>) -> Result, String> { + let alpn: Vec> = agreed.map(|p| vec![p.to_vec()]).unwrap_or_default(); + if alpn.is_empty() { + return Ok(Arc::clone(&self.silent)); + } + Ok(Arc::new(server_config(&self.server, &alpn, self.transport_for(&alpn))?)) + } + + /// Towards the service, offering exactly what the client offered. The service is + /// dialled before it has said which protocol it speaks, so the datagram extension is + /// decided by the list rather than by the answer: with nothing but h3 on it there is + /// nothing a datagram could be for, and advertising it upstream would invite what this + /// end would then have to drop. + fn client_for(&self, offered: &[Vec]) -> Result { + let mut rustls_client = self.client.clone(); + rustls_client.alpn_protocols = offered.to_vec(); + let crypto = QuicClientConfig::try_from(rustls_client) + .map_err(|e| format!("cannot configure QUIC towards the service: {e}"))?; + let mut client = quinn::ClientConfig::new(Arc::new(crypto)); + client.transport_config(Arc::clone(self.transport_for(offered))); + Ok(client) } - /// What the endpoint answers with before a service has been asked. Only ever used - /// for the connections this relay turns away, which need a configuration to be - /// turned away *by*. fn default_server(&self) -> Arc { - self.candidates - .first() - .and_then(|p| self.answering.get(p)) - .map(Arc::clone) - .unwrap_or_else(|| Arc::clone(&self.silent)) + Arc::clone(&self.silent) } } @@ -212,6 +215,10 @@ pub struct QuicRelay { chain: ChainHandle, cfg: QuicConfig, stats: Arc, + /// The hellos read off this endpoint's socket, for each client's own ALPN list. + hellos: Arc, + /// Whether a hello that could not be read has been said once already. + warned_hello: AtomicBool, } impl QuicRelay { @@ -228,10 +235,13 @@ impl QuicRelay { let socket = std::net::UdpSocket::bind(listen)?; let runtime = quinn::default_runtime() .ok_or_else(|| io::Error::other("no async runtime for the QUIC endpoint"))?; - let endpoint = Endpoint::new( + // Everything quinn receives passes by the hello reader first. + let hellos = Arc::new(Hellos::new()); + let sniffing = SniffingSocket::new(runtime.wrap_udp_socket(socket)?, Arc::clone(&hellos)); + let endpoint = Endpoint::new_with_abstract_socket( quinn::EndpointConfig::default(), Some((*cfg.setup.default_server()).clone()), - socket, + Arc::new(sniffing), runtime, )?; Ok(Self { @@ -240,6 +250,8 @@ impl QuicRelay { chain, cfg, stats, + hellos, + warned_hello: AtomicBool::new(false), }) } @@ -338,10 +350,11 @@ impl QuicRelay { // request to send. let (endpoint, behind, agreed) = match self.cfg.upstream { Onward::Same => { + let offered = self.offered(&incoming, client).await; // The endpoint is held for as long as the connection is: it owns the // socket the connection speaks through, and dropping it would take the // connection with it. - let (endpoint, service) = self.dial_upstream(client).await?; + let (endpoint, service) = self.dial_upstream(client, &offered).await?; let agreed = negotiated_protocol(&service); (Some(endpoint), Behind::Quic(service), agreed) } @@ -366,9 +379,12 @@ impl QuicRelay { ), }; - let accepting = incoming - .accept_with(self.cfg.setup.answering_with(agreed.as_deref())) + let answering = self + .cfg + .setup + .answering_with(agreed.as_deref()) .map_err(io::Error::other)?; + let accepting = incoming.accept_with(answering).map_err(io::Error::other)?; let peer = match tokio::time::timeout(self.cfg.connect_timeout, accepting).await { Ok(Ok(connection)) => connection, Ok(Err(e)) => { @@ -420,8 +436,32 @@ impl QuicRelay { Ok(()) } - /// Open the connection to the service, from the client's own address where we can. - async fn dial_upstream(&self, client: SocketAddr) -> io::Result<(Endpoint, Connection)> { + /// The protocols this client offered, read off its hello; `h3` if it could not be. + async fn offered(&self, incoming: &Incoming, client: SocketAddr) -> Vec> { + match self.hellos.offered(incoming.remote_address(), HELLO_WAIT).await { + Some(offered) if !offered.is_empty() => offered, + _ => { + if !self.warned_hello.swap(true, Ordering::Relaxed) { + eprintln!( + "[warn] [quic] could not read which protocols {client} offered; \ + offering {} to {} instead. A client speaking something else is \ + refused by the handshake.", + String::from_utf8_lossy(FALLBACK_ALPN), + self.upstream, + ); + } + vec![FALLBACK_ALPN.to_vec()] + } + } + } + + /// Open the connection to the service, from the client's own address where we can, + /// offering what the client offered. + async fn dial_upstream( + &self, + client: SocketAddr, + offered: &[Vec], + ) -> io::Result<(Endpoint, Connection)> { let endpoint = match self.endpoint_as(client) { Ok(endpoint) => endpoint, Err(e) => { @@ -447,7 +487,11 @@ impl QuicRelay { // authenticated — but rustls needs a name to put in the handshake. let name = self.upstream.ip().to_string(); let connecting = endpoint - .connect_with(self.cfg.setup.client.clone(), self.upstream, &name) + .connect_with( + self.cfg.setup.client_for(offered).map_err(io::Error::other)?, + self.upstream, + &name, + ) .map_err(io::Error::other)?; let connection = tokio::time::timeout(self.cfg.connect_timeout, connecting) .await diff --git a/backend/proxysrc/src/quic_hello.rs b/backend/proxysrc/src/quic_hello.rs new file mode 100644 index 0000000..8d5ce0f --- /dev/null +++ b/backend/proxysrc/src/quic_hello.rs @@ -0,0 +1,552 @@ +//! What a QUIC client offers, read off its first packets before anybody answers them. +//! +//! The TLS path mirrors ALPN by holding the ClientHello open, asking the service with the +//! client's own list and telling the client what came back. QUIC was thought to leave +//! nothing to hold: the ClientHello travels inside an Initial packet, encrypted. But the +//! Initial keys are derived from the Destination Connection ID the client put in the +//! clear (RFC 9001 §5.2) — deliberately, so a middlebox can read exactly this — and so +//! the list can be read before the handshake is answered. That is what this does. +//! +//! What it replaced was a list of candidates the operator had to write down per service, +//! `h3` unless told otherwise: a QUIC service speaking anything else was unreachable +//! through firegex until somebody found the setting. With the client's own list the +//! service chooses from what the client actually offered, exactly as over TLS, and there +//! is nothing to configure. +//! +//! It reads; it never answers. The socket the endpoint owns is wrapped +//! ([`SniffingSocket`]) so every datagram quinn is about to process passes by here first, +//! and only long-header Initial packets are looked at — one comparison for everything +//! else. A ClientHello too large for one packet (post-quantum key shares make that +//! ordinary) is reassembled from its CRYPTO frames across datagrams, in whatever order +//! they arrive. + +use std::collections::HashMap; +use std::fmt; +use std::io::{self, IoSliceMut}; +use std::net::SocketAddr; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +use quinn::udp::{RecvMeta, Transmit}; +use quinn::{AsyncUdpSocket, UdpPoller}; +use rustls::quic::{Suite, Version}; +use rustls::Side; + +/// How many clients' hellos may be in progress at once. Past it new ones are not read — +/// their connections fall back rather than this growing without end under a flood of +/// forged Initials, which cost the sender nothing: the keys are public, so a valid one can +/// be made from any address. With `MAX_HELLO` this bounds what a flood can make this +/// process hold to 16 MiB. +const MAX_PENDING: usize = 1024; + +/// How long an unfinished or unclaimed hello is kept. +const KEPT_FOR: Duration = Duration::from_secs(10); + +/// Most CRYPTO bytes kept for one hello. A ClientHello is one or two kilobytes, a few with +/// post-quantum key shares and a resumption ticket; this is a cap on what a peer can make +/// this process hold, not a size anybody meets. +const MAX_HELLO: usize = 16 * 1024; + +/// One client's hello as it arrives. +struct Pending { + /// The connection ID its Initials were protected with. A new one means a new attempt — + /// a Retry answered, or another connection from the same port. + dcid: Vec, + fragments: Vec<(u64, Vec)>, + held: usize, + alpn: Option>>, + since: Instant, +} + +/// The hellos being read, by the address they came from. +pub struct Hellos { + pending: Mutex>, + arrived: tokio::sync::Notify, + suite: Suite, +} + +impl fmt::Debug for Hellos { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Hellos").finish_non_exhaustive() + } +} + +impl Default for Hellos { + fn default() -> Self { + Self::new() + } +} + +impl Hellos { + pub fn new() -> Self { + // Every version's Initial packets are protected with this suite, whatever the + // handshake goes on to choose: RFC 9001 §5.2. + let suite = rustls::crypto::ring::cipher_suite::TLS13_AES_128_GCM_SHA256 + .tls13() + .and_then(|s| s.quic_suite()) + .expect("TLS13_AES_128_GCM_SHA256 carries QUIC"); + Self { + pending: Mutex::new(HashMap::new()), + arrived: tokio::sync::Notify::new(), + suite, + } + } + + /// Look at one datagram received from `from`. Cheap for anything but an Initial. + pub fn observe(&self, from: SocketAddr, datagram: &[u8]) { + let mut rest = datagram; + // An Initial can have other packets coalesced behind it in one datagram; only + // the Initials matter, and they come first. + while let Some(packet) = decrypt_initial(&self.suite, rest) { + rest = &rest[packet.length..]; + self.absorb(from, packet); + } + } + + fn absorb(&self, from: SocketAddr, packet: Initial) { + let Ok(mut pending) = self.pending.lock() else { return }; + let now = Instant::now(); + if pending.len() >= MAX_PENDING { + pending.retain(|_, p| now.duration_since(p.since) < KEPT_FOR); + if pending.len() >= MAX_PENDING && !pending.contains_key(&from) { + return; + } + } + let entry = pending.entry(from).or_insert_with(|| Pending { + dcid: packet.dcid.clone(), + fragments: Vec::new(), + held: 0, + alpn: None, + since: now, + }); + if entry.dcid != packet.dcid { + *entry = Pending { + dcid: packet.dcid.clone(), + fragments: Vec::new(), + held: 0, + alpn: None, + since: now, + }; + } + if entry.alpn.is_some() { + return; + } + for (offset, data) in packet.crypto { + if entry.held + data.len() > MAX_HELLO { + return; + } + entry.held += data.len(); + entry.fragments.push((offset, data)); + } + if let Some(alpn) = client_hello_alpn(&entry.fragments) { + entry.alpn = Some(alpn); + entry.fragments.clear(); + self.arrived.notify_waiters(); + } + } + + fn take(&self, from: SocketAddr) -> Option>> { + let mut pending = self.pending.lock().ok()?; + if pending.get(&from)?.alpn.is_some() { + return pending.remove(&from).and_then(|p| p.alpn); + } + None + } + + /// The protocols the client at `from` offered, waiting up to `wait` for the rest of a + /// hello split across datagrams. `None` when it could not be read. + pub async fn offered(&self, from: SocketAddr, wait: Duration) -> Option>> { + let deadline = tokio::time::Instant::now() + wait; + loop { + // Registered before looking, so a hello completed in between is not missed. + let notified = self.arrived.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if let Some(alpn) = self.take(from) { + return Some(alpn); + } + if tokio::time::timeout_at(deadline, notified).await.is_err() { + return self.take(from); + } + } + } +} + +/// The socket a relay's endpoint receives through, reading hellos on the way past. +#[derive(Debug)] +pub struct SniffingSocket { + inner: Arc, + hellos: Arc, +} + +impl SniffingSocket { + pub fn new(inner: Arc, hellos: Arc) -> Self { + Self { inner, hellos } + } +} + +impl AsyncUdpSocket for SniffingSocket { + fn create_io_poller(self: Arc) -> Pin> { + Arc::clone(&self.inner).create_io_poller() + } + + fn try_send(&self, transmit: &Transmit) -> io::Result<()> { + self.inner.try_send(transmit) + } + + fn poll_recv( + &self, + cx: &mut Context, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll> { + let polled = self.inner.poll_recv(cx, bufs, meta); + if let Poll::Ready(Ok(count)) = &polled { + for (buf, meta) in bufs.iter().zip(meta.iter()).take(*count) { + let received = &buf[..meta.len.min(buf.len())]; + // Several datagrams of `stride` bytes each where the kernel coalesced them. + let stride = if meta.stride == 0 { received.len() } else { meta.stride }; + for datagram in received.chunks(stride.max(1)) { + if is_long_header(datagram) { + self.hellos.observe(meta.addr, datagram); + } + } + } + } + polled + } + + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + + fn max_transmit_segments(&self) -> usize { + self.inner.max_transmit_segments() + } + + fn max_receive_segments(&self) -> usize { + self.inner.max_receive_segments() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } +} + +fn is_long_header(datagram: &[u8]) -> bool { + datagram.first().is_some_and(|b| b & 0x80 != 0) +} + +/// One Initial packet, decrypted: what it was protected with, and its CRYPTO frames. +struct Initial { + /// How many bytes of the datagram it took, so a packet coalesced behind it is found. + length: usize, + dcid: Vec, + crypto: Vec<(u64, Vec)>, +} + +fn varint(buf: &[u8], pos: &mut usize) -> Option { + let first = *buf.get(*pos)?; + let len = 1usize << (first >> 6); + let bytes = buf.get(*pos..*pos + len)?; + let mut value = u64::from(first & 0x3f); + for byte in &bytes[1..] { + value = (value << 8) | u64::from(*byte); + } + *pos += len; + Some(value) +} + +/// Decrypt the Initial packet at the start of `packet`, if that is what it is. +fn decrypt_initial(suite: &Suite, packet: &[u8]) -> Option { + let first = *packet.first()?; + if first & 0x80 == 0 { + return None; + } + let version = u32::from_be_bytes(packet.get(1..5)?.try_into().ok()?); + let (version, initial_type) = match version { + 0x0000_0001 => (Version::V1, 0), + 0x6b33_43cf => (Version::V2, 1), + 0xff00_001d..=0xff00_0020 => (Version::V1Draft, 0), + _ => return None, + }; + if (first >> 4) & 0x03 != initial_type { + return None; + } + let mut pos = 5; + let dcid_len = usize::from(*packet.get(pos)?); + let dcid = packet.get(pos + 1..pos + 1 + dcid_len)?.to_vec(); + pos += 1 + dcid_len; + let scid_len = usize::from(*packet.get(pos)?); + pos += 1 + scid_len; + let token_len = usize::try_from(varint(packet, &mut pos)?).ok()?; + pos = pos.checked_add(token_len)?; + let length = usize::try_from(varint(packet, &mut pos)?).ok()?; + let pn_offset = pos; + let end = pn_offset.checked_add(length)?; + if end > packet.len() { + return None; + } + + let keys = suite.keys(&dcid, Side::Server, version); + let sample_len = keys.remote.header.sample_len(); + let sample = packet.get(pn_offset + 4..pn_offset + 4 + sample_len)?.to_vec(); + let mut buf = packet[..end].to_vec(); + let (head, tail) = buf.split_at_mut(1); + let pn_field = tail.get_mut(pn_offset - 1..pn_offset + 3)?; + keys.remote + .header + .decrypt_in_place(&sample, &mut head[0], pn_field) + .ok()?; + let pn_len = usize::from(buf[0] & 0x03) + 1; + let mut number = 0u64; + for byte in &buf[pn_offset..pn_offset + pn_len] { + number = (number << 8) | u64::from(*byte); + } + let (header, payload) = buf.split_at_mut(pn_offset + pn_len); + let plain = keys + .remote + .packet + .decrypt_in_place(number, header, payload) + .ok()?; + + Some(Initial { + length: end, + dcid, + crypto: crypto_frames(plain)?, + }) +} + +/// The CRYPTO frames of a decrypted Initial payload. `None` for a frame an Initial cannot +/// carry, which means this was not what it looked like. +fn crypto_frames(payload: &[u8]) -> Option)>> { + let mut frames = Vec::new(); + let mut pos = 0; + while pos < payload.len() { + match varint(payload, &mut pos)? { + 0x00 | 0x01 => {} // PADDING, PING + kind @ (0x02 | 0x03) => { + // ACK: largest, delay, range count, first range, the ranges, and ECN. + varint(payload, &mut pos)?; + varint(payload, &mut pos)?; + let ranges = varint(payload, &mut pos)?; + varint(payload, &mut pos)?; + for _ in 0..ranges { + varint(payload, &mut pos)?; + varint(payload, &mut pos)?; + } + if kind == 0x03 { + for _ in 0..3 { + varint(payload, &mut pos)?; + } + } + } + 0x06 => { + let offset = varint(payload, &mut pos)?; + let len = usize::try_from(varint(payload, &mut pos)?).ok()?; + let data = payload.get(pos..pos.checked_add(len)?)?; + frames.push((offset, data.to_vec())); + pos += len; + } + // CONNECTION_CLOSE ends what there is to read. + 0x1c => break, + _ => return None, + } + } + Some(frames) +} + +/// The ALPN list of a ClientHello, once enough of it has arrived to have one. +/// +/// `Some(vec![])` for a complete hello that offers none — which QUIC does not allow, and +/// which the handshake is left to refuse. +fn client_hello_alpn(fragments: &[(u64, Vec)]) -> Option>> { + // What has arrived contiguously from the start, in whatever order it came. + let mut sorted: Vec<&(u64, Vec)> = fragments.iter().collect(); + sorted.sort_by_key(|(offset, _)| *offset); + let mut hello: Vec = Vec::new(); + for (offset, data) in sorted { + let offset = usize::try_from(*offset).ok()?; + if offset > hello.len() { + break; + } + let overlap = hello.len() - offset; + if overlap < data.len() { + hello.extend_from_slice(&data[overlap..]); + } + } + if hello.len() < 4 || hello[0] != 0x01 { + return None; + } + let len = (usize::from(hello[1]) << 16) | (usize::from(hello[2]) << 8) | usize::from(hello[3]); + let body = hello.get(4..4 + len)?; + Some(alpn_of(body).unwrap_or_default()) +} + +fn alpn_of(body: &[u8]) -> Option>> { + let u16_at = |pos: usize| -> Option { + Some(usize::from(u16::from_be_bytes(body.get(pos..pos + 2)?.try_into().ok()?))) + }; + // Legacy version and random, then the session id, the suites and the compressions. + let mut pos = 2 + 32; + pos += 1 + usize::from(*body.get(pos)?); + pos += 2 + u16_at(pos)?; + pos += 1 + usize::from(*body.get(pos)?); + let end = pos.checked_add(2 + u16_at(pos)?)?; + pos += 2; + while pos + 4 <= end { + let kind = u16_at(pos)?; + let len = u16_at(pos + 2)?; + let data = body.get(pos + 4..pos + 4 + len)?; + pos += 4 + len; + if kind != 16 { + continue; + } + let mut names = Vec::new(); + let mut at = 2; + while at < data.len() { + let name_len = usize::from(data[at]); + names.push(data.get(at + 1..at + 1 + name_len)?.to_vec()); + at += 1 + name_len; + } + return Some(names); + } + Some(Vec::new()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn push_varint(out: &mut Vec, value: u64) { + if value < 64 { + out.push(value as u8); + } else if value < 16384 { + out.extend_from_slice(&(0x4000 | value as u16).to_be_bytes()); + } else { + out.extend_from_slice(&(0x8000_0000 | value as u32).to_be_bytes()); + } + } + + /// A ClientHello offering `alpn`, and nothing else a parser would need. + fn hello_with(alpn: &[&[u8]]) -> Vec { + let mut names = Vec::new(); + for name in alpn { + names.push(name.len() as u8); + names.extend_from_slice(name); + } + let mut ext = vec![0x00, 0x10]; + ext.extend_from_slice(&((names.len() + 2) as u16).to_be_bytes()); + ext.extend_from_slice(&(names.len() as u16).to_be_bytes()); + ext.extend_from_slice(&names); + let mut body = vec![0x03, 0x03]; + body.extend_from_slice(&[0u8; 32]); + body.push(0); // no session id + body.extend_from_slice(&[0x00, 0x02, 0x13, 0x01]); // one suite + body.extend_from_slice(&[0x01, 0x00]); // no compression + body.extend_from_slice(&(ext.len() as u16).to_be_bytes()); + body.extend_from_slice(&ext); + let mut hello = vec![0x01]; + hello.extend_from_slice(&(body.len() as u32).to_be_bytes()[1..]); + hello.extend_from_slice(&body); + hello + } + + /// A client Initial carrying these CRYPTO fragments, protected the way a client + /// protects one — with rustls's own client-side keys, so what is read back is checked + /// against the other half of the library rather than against itself. + fn client_initial(dcid: &[u8], number: u64, crypto: &[(u64, &[u8])]) -> Vec { + let keys = Hellos::new().suite.keys(dcid, Side::Client, Version::V1); + let mut payload = Vec::new(); + for (offset, data) in crypto { + payload.push(0x06); + push_varint(&mut payload, *offset); + push_varint(&mut payload, data.len() as u64); + payload.extend_from_slice(data); + } + payload.push(0x01); // a PING, as Chrome scatters among the fragments + payload.resize(payload.len().max(1100), 0); // PADDING, to the size a client sends + let mut packet = vec![0xc1, 0, 0, 0, 1, dcid.len() as u8]; + packet.extend_from_slice(dcid); + packet.extend_from_slice(&[0, 0]); // no source id, no token + packet.extend_from_slice(&(0x4000 | (2 + payload.len() + 16) as u16).to_be_bytes()); + let pn_offset = packet.len(); + packet.extend_from_slice(&(number as u16).to_be_bytes()); + let tag = keys.local.packet.encrypt_in_place(number, &packet, &mut payload).unwrap(); + packet.extend_from_slice(&payload); + packet.extend_from_slice(tag.as_ref()); + let sample = packet[pn_offset + 4..pn_offset + 20].to_vec(); + let (first, rest) = packet.split_at_mut(1); + keys.local + .header + .encrypt_in_place(&sample, &mut first[0], &mut rest[pn_offset - 1..pn_offset + 1]) + .unwrap(); + packet + } + + fn from() -> SocketAddr { + "192.0.2.1:4433".parse().unwrap() + } + + #[test] + fn a_protected_initial_is_read_for_what_it_offers() { + let hellos = Hellos::new(); + let hello = hello_with(&[b"h3", b"fgex-test"]); + hellos.observe(from(), &client_initial(b"\x83\x94\xc8\xf0\x3e\x51\x57\x08", 0, &[(0, &hello)])); + assert_eq!(hellos.take(from()), Some(vec![b"h3".to_vec(), b"fgex-test".to_vec()])); + } + + /// Post-quantum key shares make a ClientHello bigger than one packet, and clients + /// scatter its pieces: two Initials, the second one first. + #[test] + fn a_hello_across_two_packets_arriving_backwards_is_reassembled() { + let hellos = Hellos::new(); + let hello = hello_with(&[b"doq"]); + let (a, b) = hello.split_at(30); + let dcid = b"\x01\x02\x03\x04\x05\x06\x07\x08"; + hellos.observe(from(), &client_initial(dcid, 1, &[(30, b)])); + assert_eq!(hellos.take(from()), None, "half a hello was read as a whole one"); + hellos.observe(from(), &client_initial(dcid, 0, &[(0, a)])); + assert_eq!(hellos.take(from()), Some(vec![b"doq".to_vec()])); + } + + /// A Retry answered is a new Initial under a new connection ID, and what was gathered + /// under the old one is not mixed into it. + #[test] + fn a_new_connection_id_starts_again() { + let hellos = Hellos::new(); + let old = hello_with(&[b"old"]); + let new = hello_with(&[b"new"]); + hellos.observe(from(), &client_initial(b"\x11\x11\x11\x11\x11\x11\x11\x11", 0, &[(0, &old[..20])])); + hellos.observe(from(), &client_initial(b"\x22\x22\x22\x22\x22\x22\x22\x22", 0, &[(0, &new)])); + assert_eq!(hellos.take(from()), Some(vec![b"new".to_vec()])); + } + + #[test] + fn anything_but_an_initial_is_passed_over() { + let hellos = Hellos::new(); + hellos.observe(from(), &[0x40, 1, 2, 3, 4, 5, 6, 7]); + hellos.observe(from(), &[0xe0, 0, 0, 0, 1, 0, 0]); + let mut tampered = client_initial(b"\x09\x09\x09\x09\x09\x09\x09\x09", 0, &[(0, &hello_with(&[b"h3"]))]); + let last = tampered.len() - 1; + tampered[last] ^= 0xff; + hellos.observe(from(), &tampered); + assert_eq!(hellos.take(from()), None); + } + + #[tokio::test] + async fn a_hello_still_arriving_is_waited_for() { + let hellos = Arc::new(Hellos::new()); + let hello = hello_with(&[b"h3"]); + let packet = client_initial(b"\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11", 0, &[(0, &hello)]); + let later = Arc::clone(&hellos); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + later.observe(from(), &packet); + }); + assert_eq!(hellos.offered(from(), Duration::from_secs(2)).await, Some(vec![b"h3".to_vec()])); + assert_eq!(hellos.offered(from(), Duration::from_millis(50)).await, None); + } +} diff --git a/backend/proxysrc/src/report.rs b/backend/proxysrc/src/report.rs new file mode 100644 index 0000000..ac83e07 --- /dev/null +++ b/backend/proxysrc/src/report.rs @@ -0,0 +1,84 @@ +//! Every line this engine sends the backend, written by a thread of its own. +//! +//! stdout is a pipe the backend reads, and a pipe fills. Each refused connection used to +//! be a `println!("BLOCKED …")` from the thread that refused it — so when the backend fell +//! behind, which a flood of refused connections makes it do, the pipe filled and those +//! writes blocked: the runtime threads carrying every other connection sat waiting on it. +//! Measured, 5000 refused connections took 74 seconds to get through, and a legitimate +//! client meanwhile waited a median 4.5 seconds and half the time not at all — an attacker +//! slowing a service to a halt by sending it exactly what firegex was there to refuse. +//! +//! So nothing on the datapath writes to stdout any more. Lines go into a bounded queue and +//! a thread of their own writes them out; a report of what happened (`BLOCKED`, `STATS`) +//! is dropped and counted when the queue is full, and a reply the backend is waiting for +//! (`PORT`, `UDP`, `ACK`) waits for room, so it is never lost and never overtakes a line +//! sent before it. + +use std::io::Write; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{sync_channel, SyncSender, TrySendError}; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +/// How many lines may wait for the backend. At a few dozen bytes each this is a megabyte; +/// what it buys is that a burst of refusals is reported whole rather than cut off. +const QUEUE: usize = 32 * 1024; + +/// How often a count of reports that had to be dropped is written out. +const DROPS_SAID_EVERY: Duration = Duration::from_secs(5); + +static SENDER: OnceLock> = OnceLock::new(); +static DROPPED: AtomicU64 = AtomicU64::new(0); + +fn sender() -> &'static SyncSender { + SENDER.get_or_init(|| { + let (tx, rx) = sync_channel::(QUEUE); + std::thread::Builder::new() + .name("fgex-report".to_string()) + .spawn(move || { + let stdout = std::io::stdout(); + let mut said = Instant::now(); + while let Ok(first) = rx.recv() { + let mut out = stdout.lock(); + let _ = writeln!(out, "{first}"); + // Whatever else is already waiting goes in the same write. + while let Ok(line) = rx.try_recv() { + let _ = writeln!(out, "{line}"); + } + let _ = out.flush(); + drop(out); + if said.elapsed() >= DROPS_SAID_EVERY { + let dropped = DROPPED.swap(0, Ordering::Relaxed); + if dropped > 0 { + eprintln!( + "[warn] [report] {dropped} refusals were not reported to the \ + backend: it was not reading fast enough. They were refused all \ + the same; only the counters and the log are short of them." + ); + } + said = Instant::now(); + } + } + }) + .expect("cannot start the thread that talks to the backend"); + tx + }) +} + +/// A reply the backend is waiting for. Waits for room rather than being dropped. +pub fn reply(line: String) { + let _ = sender().send(line); +} + +/// A report of something that happened. Never waits: dropped and counted when the backend +/// is too far behind, so the datapath is never held up by it. +pub fn event(line: String) { + if let Err(TrySendError::Full(_)) = sender().try_send(line) { + DROPPED.fetch_add(1, Ordering::Relaxed); + } +} + +/// A rule refused a connection. +pub fn blocked(id: &str) { + event(format!("BLOCKED {id}")); +} diff --git a/backend/proxysrc/src/rules.rs b/backend/proxysrc/src/rules.rs index ef28fbe..0d64cb1 100644 --- a/backend/proxysrc/src/rules.rs +++ b/backend/proxysrc/src/rules.rs @@ -199,6 +199,21 @@ impl FilterSession for HyperscanSession { /// All or nothing: a single bad rule rejects the update and leaves the running chain /// untouched, because half a ruleset is worse than the previous one. pub fn parse_ruleset(line: &str) -> Result>, String> { + parse_ruleset_reusing(line, &[]) +} + +/// [`parse_ruleset`], carrying over any filter of `previous` that the new ruleset +/// describes exactly as it was (`Filter::reuse_key`). +/// +/// A ruleset is sent whole on every edit, so without this a pattern changed in one +/// filter restarted the Python worker of another, and the new worker met every open +/// connection in the middle: its module globals were gone, its HTTP parser started on +/// the second half of whatever request was in flight, and the addresses it had been told +/// when each connection opened were never told again. +pub fn parse_ruleset_reusing( + line: &str, + previous: &[Arc], +) -> Result>, String> { let specs: Vec = serde_json::from_str(line).map_err(|e| format!("malformed ruleset: {e}"))?; @@ -306,13 +321,21 @@ pub fn parse_ruleset(line: &str) -> Result>, String> { } else { command }; - out.push(Arc::new(PyWorkerRule::new( - id, - command, - code_path, - enabled, - Duration::from_millis(timeout_ms), - ))) + let deadline = Duration::from_millis(timeout_ms); + let key = PyWorkerRule::key_for(&id, &command, &code_path, &enabled, deadline); + let kept = key.as_deref().and_then(|key| { + previous + .iter() + .find(|filter| filter.reuse_key() == Some(key)) + .cloned() + }); + out.push(match kept { + Some(filter) => filter, + None => Arc::new( + PyWorkerRule::new(id, command, code_path, enabled, deadline) + .reusable_as(key), + ), + }) } } } diff --git a/backend/proxysrc/src/udp.rs b/backend/proxysrc/src/udp.rs index c1acb7b..bff5a80 100644 --- a/backend/proxysrc/src/udp.rs +++ b/backend/proxysrc/src/udp.rs @@ -333,6 +333,13 @@ async fn replies( loop { let len = match upstream.recv(&mut buf).await { Ok(len) => len, + // The service's port answered a datagram with ICMP "unreachable", which a + // connected socket reports once, on its next call: the service was down for a + // moment — being restarted, most often. Nothing about the flow is over. This + // used to end the task, which left the flow in the relay's table with nobody + // reading the service's answers, so a client that went on talking never heard + // back again while its datagrams kept the flow from ever going idle. + Err(e) if e.kind() == io::ErrorKind::ConnectionRefused => continue, Err(e) => { eprintln!("[info] [udp] flow for {client} ended: {e}"); return; diff --git a/backend/proxysrc/tests/fail_open.rs b/backend/proxysrc/tests/fail_open.rs index 45bea7d..e51ee32 100644 --- a/backend/proxysrc/tests/fail_open.rs +++ b/backend/proxysrc/tests/fail_open.rs @@ -42,11 +42,16 @@ async fn spawn_echo() -> std::net::SocketAddr { } async fn spawn_proxy(filters: &str, deadline_ms: u64) -> (std::net::SocketAddr, ChainHandle) { - let upstream = spawn_echo().await; - let chain = ChainHandle::new(FilterChain::new( + spawn_proxy_with(FilterChain::new( parse_filters(filters).unwrap(), Duration::from_millis(deadline_ms), - )); + )) + .await +} + +async fn spawn_proxy_with(chain: FilterChain) -> (std::net::SocketAddr, ChainHandle) { + let upstream = spawn_echo().await; + let chain = ChainHandle::new(chain); let proxy = Proxy::bind( ProxyConfig::fixed("127.0.0.1:0".parse().unwrap(), upstream), chain.clone(), @@ -105,7 +110,12 @@ async fn panicking_filter_keeps_traffic_flowing() { /// process-wide, and the harness runs tests in parallel within one process. #[tokio::test] async fn hanging_filter_fails_open_then_loses_its_say() { - let (addr, chain) = spawn_proxy("hang", 50).await; + let rearm = Duration::from_millis(1500); + let (addr, chain) = spawn_proxy_with( + FilterChain::new(parse_filters("hang").unwrap(), Duration::from_millis(50)) + .rearming_after(rearm), + ) + .await; // The deadline, not the filter, decides when the chunk moves. let started = Instant::now(); @@ -139,8 +149,24 @@ async fn hanging_filter_fails_open_then_loses_its_say() { "a disabled filter is still being called, so the thread leak is unbounded" ); - // Let the abandoned threads exit, otherwise dropping the runtime waits on them. + // Slow is not broken: after a pause it is asked again, rather than staying out until + // somebody next edits a rule. It used to stay out for the life of the chain — which + // for a Python filter meant a flood, the one moment it was wanted, switched it off. + tokio::time::sleep(rearm + Duration::from_millis(300)).await; + let answered = roundtrip(addr, b"again").await.unwrap(); + let asked = live.stats.timeouts.load(Ordering::Relaxed) - leaked; + let back_in = live.disabled_filters().is_empty(); + let bypassed = live.is_bypassed(); + + // Let the abandoned threads exit, otherwise dropping the runtime waits on them — and + // before asserting, so a failure below fails the test rather than hanging it. fgex_proxy::spec::release_hangs(); + + assert_eq!(answered, b"again"); + // Once per direction: the echo comes back through the chain as well. + assert_eq!(asked, 2, "the filter was not asked again after its pause"); + assert!(back_in, "two misses after the pause took it out again"); + assert!(!bypassed); } /// Failing open is only worth anything if the filters do something when healthy. diff --git a/backend/proxysrc/tests/h3.rs b/backend/proxysrc/tests/h3.rs index ebd4ede..f558e5d 100644 --- a/backend/proxysrc/tests/h3.rs +++ b/backend/proxysrc/tests/h3.rs @@ -207,7 +207,7 @@ async fn spawn_relay_to( Duration::from_millis(500), )); let cfg = QuicConfig { - setup: Arc::new(QuicSetup::build(&cert, &key, vec![b"h3".to_vec()]).unwrap()), + setup: Arc::new(QuicSetup::build(&cert, &key).unwrap()), self_mark: None, spoof_source: false, max_connections: 0, diff --git a/backend/proxysrc/tests/pyworker.rs b/backend/proxysrc/tests/pyworker.rs index 808e493..e917f46 100644 --- a/backend/proxysrc/tests/pyworker.rs +++ b/backend/proxysrc/tests/pyworker.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; use fgex_proxy::filter::{ next_connection_id, ChainSessions, ConnectionMeta, Direction, FilterChain, Verdict, L4, }; -use fgex_proxy::rules::parse_ruleset; +use fgex_proxy::rules::{parse_ruleset, parse_ruleset_reusing}; /// Write a filter file and give back its path, kept for the process's lifetime. fn filter_file(name: &str, body: &str) -> String { @@ -537,3 +537,73 @@ def refuse_b(packet: RawPacket): assert_eq!(feed(&off, &[b"carrying AAA"]).await, vec![Verdict::Accept]); assert_eq!(feed(&off, &[b"carrying BBB"]).await, vec![Verdict::Accept]); } + +/// An edit elsewhere in the chain must not restart a Python filter that did not change. +/// +/// Every edit sends the whole ruleset, and a new worker meets every open connection +/// halfway: its module globals gone, its HTTP parser starting on the second half of the +/// request in flight. The same filter, described the same way, is carried over with the +/// state it keeps for each connection; new code is a new worker, as it has to be. +#[tokio::test] +async fn an_unchanged_worker_is_carried_into_the_next_ruleset_with_its_state() { + const REMEMBERS: &str = r#"from firegex.pyfilters import pyfilter, ACCEPT, REJECT +from firegex.pyfilters.models import RawPacket + +# Module level, so it is per connection: the library gives each stream its own globals. +said_first = False + +@pyfilter +def second_after_first(packet: RawPacket): + global said_first + if b"FIRST" in packet.data: + said_first = True + return REJECT if said_first and b"SECOND" in packet.data else ACCEPT +"#; + let path = filter_file("carried", REMEMBERS); + let json = format!( + r#"[{{"kind":"python","id":"py1","code_path":"{path}","timeout_ms":2000, + "command":["python3","{}"]}}]"#, + worker() + ); + let deadline = Duration::from_secs(10); + let connection = next_connection_id(); + + let before = FilterChain::new(parse_ruleset(&json).unwrap(), deadline); + let mut sessions = ChainSessions::new(connection); + assert_eq!( + before.run(Direction::ClientToServer, b"FIRST", &mut sessions).await, + Verdict::Accept + ); + + // The same ruleset again, as an edit to some other filter would send it. + let after = FilterChain::new( + parse_ruleset_reusing(&json, &before.filters()).unwrap(), + deadline, + ); + assert!( + std::sync::Arc::ptr_eq(&before.filters()[0], &after.filters()[0]), + "an unchanged Python filter was rebuilt" + ); + let mut sessions = ChainSessions::new(connection); + assert_eq!( + after.run(Direction::ClientToServer, b"SECOND", &mut sessions).await, + Verdict::Reject(Some("py1/second_after_first".to_string())), + "the connection's state did not survive an edit that did not touch this filter" + ); + + // New code is a new worker, and starts every connection afresh. + filter_file("carried", &format!("{REMEMBERS}\n# edited\n")); + let edited = FilterChain::new( + parse_ruleset_reusing(&json, &after.filters()).unwrap(), + deadline, + ); + assert!( + !std::sync::Arc::ptr_eq(&after.filters()[0], &edited.filters()[0]), + "edited code was not given a worker of its own" + ); + let mut sessions = ChainSessions::new(connection); + assert_eq!( + edited.run(Direction::ClientToServer, b"SECOND", &mut sessions).await, + Verdict::Accept + ); +} diff --git a/backend/proxysrc/tests/quic.rs b/backend/proxysrc/tests/quic.rs index 8e3b5f3..d968529 100644 --- a/backend/proxysrc/tests/quic.rs +++ b/backend/proxysrc/tests/quic.rs @@ -79,14 +79,14 @@ async fn spawn_quic_echo(cert: &str, key: &str, speaks: &[&str]) -> SocketAddr { } /// The relay, in front of that service. -async fn spawn_relay(upstream: SocketAddr, filters: &str, offers: &[&str]) -> SocketAddr { +async fn spawn_relay(upstream: SocketAddr, filters: &str) -> SocketAddr { let (cert, key) = self_signed(); let chain = ChainHandle::new(FilterChain::new( parse_filters(filters).unwrap(), Duration::from_millis(500), )); let cfg = QuicConfig { - setup: Arc::new(QuicSetup::build(&cert, &key, alpn(offers)).unwrap()), + setup: Arc::new(QuicSetup::build(&cert, &key).unwrap()), // No mark and no impersonation: `SO_MARK` and `IP_TRANSPARENT` both need // CAP_NET_ADMIN, which a test run has no business requiring. What they change is // which address the service sees, and nothing here is asking about that. @@ -188,7 +188,7 @@ async fn roundtrip(addr: SocketAddr, payload: &[u8], speaks: &[&str]) -> Exchang async fn carries_a_stream_through() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; - let addr = spawn_relay(upstream, "", &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "").await; let exchange = roundtrip(addr, b"hello over quic", &["fgex-raw"]).await; assert_eq!(exchange.echoed.unwrap(), b"hello over quic"); @@ -200,7 +200,7 @@ async fn carries_a_stream_through() { async fn rules_see_the_plaintext_inside_the_streams() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; - let addr = spawn_relay(upstream, "block:FLAG{", &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:FLAG{").await; let harmless = roundtrip(addr, b"harmless request", &["fgex-raw"]).await; assert_eq!(harmless.echoed.unwrap(), b"harmless request"); @@ -225,7 +225,7 @@ async fn rules_see_the_plaintext_inside_the_streams() { async fn a_refusal_takes_the_whole_connection() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; - let addr = spawn_relay(upstream, "block:FLAG{", &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:FLAG{").await; let mut config = tls::quic_client_config().unwrap(); config.alpn_protocols = alpn(&["fgex-raw"]); @@ -249,31 +249,47 @@ async fn a_refusal_takes_the_whole_connection() { ); } -/// The service picks the protocol, and the client is told that and nothing else. +/// The service picks from what the client offered, and the client is told that and +/// nothing else — the TLS path's mirroring, on QUIC. /// -/// This is the ordering that had to change from the TLS path: there the ClientHello is -/// held open and the service is asked what the *client* offered, which QUIC gives no -/// opportunity to do. So the relay offers its candidates to the service and passes the -/// answer on — and a service speaking something unusual is carried without firegex -/// having to guess. +/// The relay used to offer the service a list of its own, `h3` unless the operator named +/// others, because the client's hello was taken to be unreadable. It is read off the +/// Initial instead (`quic_hello`), so a service speaking something nobody configured is +/// carried as it is. #[tokio::test] -async fn the_service_chooses_the_protocol() { +async fn the_service_chooses_from_what_the_client_offered() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-test"]).await; - let addr = spawn_relay(upstream, "", &["h3", "fgex-test"]).await; + let addr = spawn_relay(upstream, "").await; let exchange = roundtrip(addr, b"ping", &["h3", "fgex-test"]).await; assert_eq!(exchange.protocol.as_deref(), Some("fgex-test")); assert_eq!(exchange.echoed.unwrap(), b"ping"); } -/// A client that cannot speak what the service chose is refused by the handshake, which -/// is the honest failure this ordering costs. +/// A hello too big for one packet — which post-quantum key shares make ordinary — is read +/// across the packets it came in. +#[tokio::test] +async fn a_hello_spanning_several_packets_is_read_whole() { + let (cert, key) = self_signed(); + let upstream = spawn_quic_echo(&cert, &key, &["fgex-last"]).await; + let addr = spawn_relay(upstream, "").await; + + let filler: Vec = (0..40).map(|i| format!("fgex-unspoken-{i:02}-{}", "x".repeat(40))).collect(); + let mut offers: Vec<&str> = filler.iter().map(String::as_str).collect(); + offers.push("fgex-last"); + let exchange = roundtrip(addr, b"ping", &offers).await; + assert_eq!(exchange.protocol.as_deref(), Some("fgex-last")); + assert_eq!(exchange.echoed.unwrap(), b"ping"); +} + +/// A client that speaks nothing the service does is refused by the service's handshake, +/// the way it would be with no firegex in the way. #[tokio::test] async fn a_client_that_speaks_something_else_is_refused() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-test"]).await; - let addr = spawn_relay(upstream, "", &["fgex-test"]).await; + let addr = spawn_relay(upstream, "").await; let exchange = roundtrip(addr, b"ping", &["h3"]).await; assert!( @@ -298,7 +314,7 @@ async fn connect(addr: SocketAddr, speaks: &[&str]) -> quinn::Connection { async fn datagrams_are_carried_both_ways() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; - let addr = spawn_relay(upstream, "", &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "").await; let connection = connect(addr, &["fgex-raw"]).await; assert!( @@ -324,7 +340,7 @@ async fn datagrams_are_carried_both_ways() { async fn a_refused_datagram_is_dropped_and_the_connection_lives() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; - let addr = spawn_relay(upstream, "block:forbidden", &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:forbidden").await; let connection = connect(addr, &["fgex-raw"]).await; connection @@ -361,7 +377,7 @@ async fn a_refused_datagram_is_dropped_and_the_connection_lives() { async fn a_datagram_flow_keeps_its_filter_state() { let (cert, key) = self_signed(); let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await; - let addr = spawn_relay(upstream, "block:secret", &["fgex-raw"]).await; + let addr = spawn_relay(upstream, "block:secret").await; let connection = connect(addr, &["fgex-raw"]).await; connection diff --git a/backend/proxysrc/tests/udp.rs b/backend/proxysrc/tests/udp.rs index 250054d..be17643 100644 --- a/backend/proxysrc/tests/udp.rs +++ b/backend/proxysrc/tests/udp.rs @@ -130,6 +130,61 @@ async fn flows_spend_the_same_budget_as_connections() { assert_eq!(stats.live.load(Ordering::Relaxed), 1, "a refused flow kept its place"); } +/// A service that goes away for a moment and comes back is answering again, for the +/// client that kept talking through it. +/// +/// A datagram sent while it was down is answered with ICMP "port unreachable", which the +/// flow's connected socket reports as an error on its next call. That error ended the task +/// carrying the answers back while the flow stayed in the table — and a client that kept +/// sending kept it from ever going idle, so it never heard from the service again. +#[tokio::test] +async fn a_flow_outlives_its_service_restarting() { + let first = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let upstream = first.local_addr().unwrap(); + let echo = |socket: UdpSocket| { + tokio::spawn(async move { + let mut buf = vec![0u8; 2048]; + loop { + let Ok((n, from)) = socket.recv_from(&mut buf).await else { + continue; + }; + let _ = socket.send_to(&buf[..n], from).await; + } + }) + }; + let service = echo(first); + let relay = spawn_relay(upstream, "", 0, false, Arc::new(ProxyStats::default())).await; + + // One source throughout: this is about a flow that already exists. + let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + assert_eq!(exchange(&client, relay, b"before").await.as_deref(), Some(&b"before"[..])); + + service.abort(); + let _ = service.await; + assert_eq!(exchange(&client, relay, b"while down").await, None); + + echo(UdpSocket::bind(upstream).await.unwrap()); + assert_eq!( + exchange(&client, relay, b"after").await.as_deref(), + Some(&b"after"[..]), + "the service came back and the flow that kept talking was not answered" + ); +} + +/// One datagram from `client`, and the answer if one comes. +async fn exchange( + client: &UdpSocket, + relay: std::net::SocketAddr, + payload: &[u8], +) -> Option> { + client.send_to(payload, relay).await.unwrap(); + let mut buf = vec![0u8; 2048]; + match tokio::time::timeout(Duration::from_millis(700), client.recv(&mut buf)).await { + Ok(Ok(n)) => Some(buf[..n].to_vec()), + _ => None, + } +} + /// Flows are counted in the unit the backend divides by: flows seen, flows refused. /// /// UDP counted neither, so a service refusing datagrams all day reported no connections diff --git a/backend/routers/firewall.py b/backend/routers/firewall.py index c463e87..591076d 100644 --- a/backend/routers/firewall.py +++ b/backend/routers/firewall.py @@ -50,6 +50,7 @@ async def startup(): await firewall.init() async def shutdown(): + firewall.stop_watching() if not firewall.keep_rules: await firewall.close() db.disconnect() diff --git a/backend/routers/services.py b/backend/routers/services.py index 79aa26d..9829753 100644 --- a/backend/routers/services.py +++ b/backend/routers/services.py @@ -21,7 +21,7 @@ import sys import time import traceback -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import PlainTextResponse from pydantic import BaseModel @@ -35,7 +35,7 @@ from modules.services.firewall import ( from modules.services import stats from modules.services.logs import log_for from modules.services.models import (KIND, L4, MODE, PROTO, STATUS, Service, - TRANSPORT, UPSTREAM, upstream_refusal) + TRANSPORT, UPSTREAM) from modules.services.nftables import FiregexTables, hijack_endpoint from modules.services import transports from modules.services.transports import PROXY_ENGINE, PYWORKER, UnsupportedChain @@ -50,7 +50,17 @@ from utils.models import ResetRequest, StatusMessageModel from utils.sqlite import SQLite -app = APIRouter() +def _counters_written() -> None: + """Every request sees the block counters as they are, and changes them from there. + + They are written once a second (`BLOCK_FLUSH` in the manager), not per block — so a + read would otherwise be up to a second behind, and an edit that resets a pattern's + counter would have the old pattern's last blocks written onto the new one after it. + """ + firewall.flush_blocks() + + +app = APIRouter(dependencies=[Depends(_counters_written)]) db = SQLite( "db/services.db", @@ -262,6 +272,15 @@ class AddressForm(BaseModel): proxy_port: PortType | None = None +class ProblemModel(BaseModel): + """The newest warning or error in a service's log. See `ServiceLog.last_problem`.""" + + #: Unix milliseconds, like every log line. + at: int + level: str + text: str + + class ServiceModel(BaseModel): service_id: str name: str @@ -288,6 +307,9 @@ class ServiceModel(BaseModel): addresses: list[AddressModel] = [] n_filters: int n_blocked: int + #: The newest warning or error still in its log, so the list can say something is + #: wrong without anybody having the service's page open. Cleared with the log. + problem: ProblemModel | None = None class ServiceAddForm(BaseModel): @@ -534,6 +556,9 @@ async def shutdown(): async def reset(params: ResetRequest): if not params.delete: + # Into the copy that is put back, or the last second of counters is written + # after it was taken and then overwritten by it. + firewall.flush_blocks() db.backup() await firewall.close() FiregexTables().reset() @@ -709,6 +734,7 @@ def _with_addresses(row: dict) -> dict: row["addresses"] = [ _address_row(a) for a in _addresses(row["service_id"]) ] + row["problem"] = log_for(row["service_id"]).last_problem() return row @@ -1052,14 +1078,7 @@ async def add_service(form: ServiceAddForm): # nobody asked for. for address in form.addresses: _address_target(form.transport, address) - edge = _address_edge(form.proto, address) - refusal = upstream_refusal( - form.proto, - _address_upstream(form.proto, edge, address), - L4.l4_of(edge) == L4.UDP, - ) - if refusal: - raise HTTPException(status_code=400, detail=refusal) + _address_upstream(form.proto, _address_edge(form.proto, address), address) # The transport gets a look before the row exists. A chain it could not host is # caught later, when there is one; what is caught here is what the *service* makes # impossible on its own — a protocol this layer cannot carry — and it is worth @@ -1254,6 +1273,11 @@ async def edit_service(service_id: str, form: ServiceSettingsForm): fields["tls_cert"] = form.tls_cert if form.tls_key is not None: fields["tls_key"] = form.tls_key + # Only what actually changes. The form sends every setting on every save, and a field + # that already holds the value it is sent is not an edit: counted as one, opening the + # dialog and saving it untouched restarted a running service and dropped every + # connection it was carrying. + fields = {key: value for key, value in fields.items() if row[key] != value} if not fields: return {"status": "ok"} @@ -1275,12 +1299,6 @@ async def edit_service(service_id: str, form: ServiceSettingsForm): + ", ".join(f"{_shown(a['ip_int'])}:{a['port']}" for a in published) + " is published on another port. Take that off the address first.", ) - for address in new_addresses: - refusal = upstream_refusal(proto, address["upstream"], - L4.l4_of(address["edge"]) == L4.UDP) - if refusal: - raise HTTPException(status_code=400, detail=refusal) - # The service it would become, asked what a start would ask of the layer and the # chain. Without its addresses, the way a service is checked when it is created: what # an address still lacks on the new layer — the port of the operator's own proxy, say @@ -1329,6 +1347,13 @@ async def edit_service(service_id: str, form: ServiceSettingsForm): detail=f"Another service already protects one of these addresses on {proto}", ) + if set(fields) == {"name"}: + # A name is the operator's, not the datapath's: nothing running reads it, so it + # is not a reason to take the service down and bring it back. + manager.srv.name = fields["name"] + await refresh_frontend() + return {"status": "ok"} + was_active = manager.active try: await manager.refresh(load_service(service_id)) @@ -1383,17 +1408,8 @@ async def add_address(service_id: str, form: AddressForm): detail="The external transport hands traffic to your own proxy and rewrites the " "source address on return, which requires a concrete IP address rather than an interface.", ) - # Before the row exists, because this is the address's own answer and it can be one - # this instance cannot honour. `ProxyTransport.check` says the same thing at start, - # and would leave the address behind to be undone. - edge = _address_edge(row["proto"], form) - refusal = upstream_refusal( - row["proto"], - _address_upstream(row["proto"], edge, form), - L4.l4_of(edge) == L4.UDP, - ) - if refusal: - raise HTTPException(status_code=400, detail=refusal) + # Before the row exists: what the address says has to mean something on this service. + _address_upstream(row["proto"], _address_edge(row["proto"], form), form) try: address_id = _insert_address(service_id, row["proto"], row["transport"], form) except sqlite3.IntegrityError as e: @@ -1469,9 +1485,6 @@ async def edit_address(service_id: str, address_id: str, form: AddressForm): form.upstream = UPSTREAM.SAME target_port = _address_target(row["transport"], form) upstream = _address_upstream(row["proto"], edge, form) - refusal = upstream_refusal(row["proto"], upstream, L4.l4_of(edge) == L4.UDP) - if refusal: - raise HTTPException(status_code=400, detail=refusal) def write(ip_int, port, proto, edge, target_port, upstream, proxy_ip, proxy_port): db.query( @@ -2356,6 +2369,8 @@ async def get_logs(service_id: str): async def clear_logs(service_id: str): _service_or_404(service_id) log_for(service_id).clear() + # The list shows the newest problem in the log, and clearing is how it is dismissed. + await refresh_frontend() return {"status": "ok"} diff --git a/backend/utils/__init__.py b/backend/utils/__init__.py index 91117da..2f0b9b1 100644 --- a/backend/utils/__init__.py +++ b/backend/utils/__init__.py @@ -311,15 +311,27 @@ class NFTableManager(Singleton): def reset(self): self.raw_cmd(*self.__reset_cmds) - def list_rules(self, tables = None, chains = None): - for filter in [ele["rule"] for ele in self.raw_list() if "rule" in ele ]: + def list_rules(self, tables = None, chains = None, family: str | None = None): + for filter in [ele["rule"] for ele in self.raw_list(tables, family) if "rule" in ele ]: if tables and filter["table"] not in tables: continue if chains and filter["chain"] not in chains: continue yield filter - - def raw_list(self): + + def raw_list(self, tables = None, family: str | None = None): + # One table of ours when that is all that is asked for, rather than the host's + # whole ruleset: on a node running Docker or Kubernetes that is thousands of rules + # of somebody else's, serialised and parsed on every lookup — and a lookup runs + # inside the backend's one event loop, on every start, stop, address change and + # statistics request. A table that is not there is simply no rules. + if tables and family: + out = [] + for table in tables: + code, listed, _ = self.raw_cmd({"list": {"table": {"family": family, "name": table}}}) + if code == 0: + out.extend(listed["nftables"]) + return out return self.cmd({"list": {"ruleset": None}})["nftables"] def _json_like(obj: BaseModel|List[BaseModel], unset=False, convert_keys:dict[str, str]=None, exclude:list[str]=None, mode:str="json"): diff --git a/docs/firewall.md b/docs/firewall.md index 78573f6..3d47811 100644 --- a/docs/firewall.md +++ b/docs/firewall.md @@ -27,7 +27,7 @@ Beyond the rule list and policy, a few toggles affect the whole module: - **allow_dhcp**: always accept DHCP traffic. - **multicast_dns**: always accept mDNS (multicast DNS) traffic. - **allow_upnp**: always accept UPnP traffic. -- **allow_dnat** (on by default): forwarded traffic whose destination another NAT rewrote — a port **Docker or podman published** for a container, a router's port forward — is left to the rules that published it when none of *your* rules matched it, instead of meeting the forward policy. Your `forward` rules still apply to it first, so a rule that drops it drops it. +- **allow_dnat** (on by default, "Leave container traffic to the container runtime"): forwarded traffic whose destination another NAT rewrote — a port **Docker or podman published** for a container, a router's port forward — and traffic **the containers send**, to each other or out (anything arriving from `docker0`, `br-*`, `docker_gwbridge`, `podman*` or `cni-podman*`), is left to the container runtime's own rules when none of *yours* matched it, instead of meeting the forward policy. Without it, a drop policy cuts a web container off from its database on the same bridge. Your `forward` rules still apply to it first, so a rule that drops it drops it. Each of the others, when enabled, inserts a small accept rule ahead of your own rules — they're conveniences for common cases you'd otherwise have to write by hand. **allow_dnat** is the exception: it sits after your rules, because it only decides what the *policy* does with what they did not match. @@ -48,6 +48,8 @@ Two things follow from Firegex owning those tables rather than writing into the - **`iptables` keeps working.** Earlier versions put their chains into the tables named `filter` and `mangle`, which is where `iptables-nft` — the default `iptables` on most current distributions — keeps its own. `iptables` rejects a whole table as soon as it contains a rule it cannot express in its own format, and the connection-tracking rules Firegex writes for **allow_established** and **drop_invalid** are exactly that. The result was `iptables -L` and `iptables-save` answering ``table `filter' is incompatible, use 'nft' tool`` on a host where Docker, ufw, fail2ban or the organisers' own scripts were the ones asking. That cannot happen now: `iptables` never looks at a table Firegex owns. - **Turning the firewall off no longer touches the host's own.** The default policy is a property of a base chain, and it used to be *your* `INPUT` chain that Firegex set to `drop` — and set back to `accept` on shutdown, silently undoing a default-deny an administrator had configured elsewhere. Firegex now sets the policy on its own base chain, so a reset is just deleting two tables. +**If something else removes those tables, Firegex puts them back within a few seconds.** `nft -f /etc/nftables.conf` starts with `flush ruleset` on Debian, and so does `systemctl restart nftables`; either one used to switch the Firegex firewall off with nothing to say so. The same watch covers the services' table: each running service is steered at its filters again without restarting, and its log records the gap as an error, since traffic was not filtered while it lasted. The lasting fix is still to make whatever manages your nftables leave the `fgex` tables alone. With `flush ruleset`, for example, you can delete only your own tables instead. + If you are upgrading from a version that used the shared tables, whatever it left behind is still there — Firegex does not go looking in tables it no longer owns. Clear it by hand once, and check the policy while you are there, since an older Firegex was the one driving it: ```bash diff --git a/docs/pyfilter.md b/docs/pyfilter.md index eadeb09..d0cbe72 100644 --- a/docs/pyfilter.md +++ b/docs/pyfilter.md @@ -404,7 +404,7 @@ compressed WebSocket message is bounded the same way. from firegex.pyfilters import ExceptionAction ``` -- `FGEX_INVALID_ENCODING_ACTION: ExceptionAction` — action taken when parsing hits an invalid/unsupported encoding (a parser-level failure). Default: `ExceptionAction.REJECT`. Values: `ACCEPT` (accept the packet that caused the error), `DROP` (drop the connection), `REJECT` (reject the connection), `NOACTION` (do nothing — the error is signaled and the stream is accepted without calling any more pyfilters on it). +- `FGEX_INVALID_ENCODING_ACTION: ExceptionAction` — action taken when parsing hits an invalid/unsupported encoding (a parser-level failure). Default: `ExceptionAction.ACCEPT` — the traffic is carried without the HTTP filters seeing it, and the service log says so once per connection; `REJECT` is the stricter choice against request smuggling, and yours to make. Values: `ACCEPT` (accept the packet that caused the error), `DROP` (drop the connection), `REJECT` (reject the connection), `NOACTION` (do nothing — the error is signaled and the stream is accepted without calling any more pyfilters on it). One case never reaches this setting: a connection the filter took over already open — it started, was rebuilt or got new code while the connection was running — whose first bytes it saw were the middle of a message, such as the rest of an upload. That is not malformed traffic, so that direction of the connection is carried without its HTTP filters until it closes, and the log says so once, as information. A connection taken over between two messages is filtered as usual. - `FGEX_MAX_HISTORY_SIZE: int` — max number of requests/responses kept per stream by [`HttpHistory`](#httphistory-alias-httpstreamhistory--http-only). Default: `100`. The entries of one direction together are also held under `FGEX_STREAM_MAX_SIZE` bytes. ## Testing a filter locally diff --git a/docs/services.md b/docs/services.md index c9ae141..96aa51f 100644 --- a/docs/services.md +++ b/docs/services.md @@ -162,9 +162,9 @@ allowed to do to the traffic. Pick by those. | | **Proxy** | **NFQUEUE** | |---|---|---| | Unit of work | a connection | a packet | -| Transparency | full — dials your service **from the client's own address** | full — the real packets, on their way to your service | +| Transparency | dials your service **from the client's own address** (not its source port) | full — the real packets, on their way to your service | | Reassembly | the kernel's, on two real sockets | rebuilt in userspace with libtins | -| Rewriting | patterns: no. Python: exact, any length | patterns: no. Python: unstable on TCP, exact on UDP | +| Rewriting | none — a filter answers with a verdict | none — a filter answers with a verdict | | If a filter dies | the engine rebuilds fail-open by hand | **the kernel keeps forwarding**, by itself | | Cost | terminating a connection, once | **a userspace round trip per packet**, per filter | | Measured ([how](../tests/bench/README.md#performance)) | **4035 MB/s** at 1 thread, **13 984** at 8 | 1820 at 1 thread, 2956 at 8 | @@ -207,14 +207,28 @@ halves. - **The kernel's fail-open backstop is gone**, and the engine has to rebuild it by hand: `catch_unwind`, a deadline per filter, a filter that misbehaves losing its say rather - than the traffic being held. It works — and it is code, where the other layer has a - kernel guarantee. + than the traffic being held. One that keeps missing its deadline — a Python filter + under a flood, usually — is asked again after half a minute rather than staying out; + one that crashed stays out until you next save a change to the filters. It works — + and it is code, where the other layer has a kernel guarantee. It stays invisible: it always dials your service **from the client's own address** on both TCP and UDP, so anything that logs, rate-limits or bans by IP keeps working. That is not a setting, because a service that suddenly saw one address for the whole internet would be a regression nobody would attribute to us. +What it cannot keep is the client's **source port**: the connection to your service is a +new one, from the client's address and a port the kernel picks. And a reset from either +end reaches the other as an ordinary close. + +**A service in a container is the ordinary case.** Protect the port the container is +published on (`docker run -p 8080:80` → this host's address, port `8080`), on either layer. +Firegex's rules run before the container runtime rewrites the destination, so it sees the +traffic first; the service is reached, filtered and shown the real client address whether +the client is on another host or on this one. Both were measured with a container behind +Docker's own rewrite, and `tests/integration/test_remote_clients.py` asks it of every +release with a stand-in that needs no Docker. + ### NFQUEUE Packets are lifted to userspace, inspected, and a verdict is handed back to the kernel. @@ -228,6 +242,11 @@ Nothing is terminated. timestamps intact. - **No connection termination.** Suitable for protocols that cannot be proxied or where terminating the transport is prohibited. +- **Connections already open are protected too.** Starting a service, adding a filter or + rebuilding the chain inspects the connections that were open at that moment from their + next packet on. The proxy layer cannot do this: a connection made before its redirect + existed was never sent to it, and goes on straight to the service, unfiltered but + uninterrupted, until it closes. **What it costs** @@ -427,10 +446,10 @@ its own connection to the service — dialled from the client's own address like dial firegex makes. What cannot be done, and is a limit rather than a missing setting: a QUIC edge carrying -anything **other than HTTP/3**. Opaque bytes on a QUIC stream have no HTTP/1.1 form, so -there is nothing to send a service that speaks one. In practice that means an instance -whose `FGEX_PROXY_QUIC_ALPN` is not `h3`: the choice is refused there, on `QUIC` and -`HTTPS` services alike, with the ALPN named. +anything **other than HTTP/3** to a service that speaks HTTP/1.1. Opaque bytes on a QUIC +stream have no HTTP/1.1 form, so there is nothing to send it. On such an address firegex +answers the client itself, in HTTP/3, and a client offering only something else is +refused by that handshake. #### What is carried through, and what cannot be @@ -613,31 +632,16 @@ bytes are real and the framing is reconstructed.** Concretely — #### What is carried through, and what cannot be -**ALPN is the service's answer, carried — asked in the other order.** On TLS over TCP the -client's hello is held open, the service is asked with exactly the list the client -offered, and the client is told what came back. In QUIC the hello arrives inside an -encrypted Initial packet whose processing *is* the handshake, so there is nothing to hold -it at. The order is therefore reversed: firegex opens its connection to the service -first, offering the protocols it was told to offer, and tells the client the one thing -the service agreed to. The invariant that matters is unchanged — the client is never told -a protocol the service did not choose — and what is lost is knowing in advance whether -the client would have accepted it. When it would not, its handshake fails and the log -says which protocol the service picked. - -That list is `h3`, which is what a QUIC service speaks nine times in ten. An instance in -front of something else sets `FGEX_PROXY_QUIC_ALPN` as a comma-separated list in -preference order: - -```bash -python3 run.py restart --env FGEX_PROXY_QUIC_ALPN=h3,doq -``` - -`run.py` stores it and puts it back on every later start — editing the generated compose -file by hand does not survive one, because `run.py` rewrites that file each time. The -list is **per instance, not per service**, and it is a superset rather than a choice: -every QUIC service is offered all of it and each one picks what it actually speaks, so -listing every protocol the instance carries is the way to run more than one kind of QUIC -service at once. +**ALPN is the service's answer, carried — and there is nothing to configure.** On TLS +over TCP the client's hello is held open, the service is asked with exactly the list the +client offered, and the client is told what came back. QUIC does the same. Its hello +travels inside an Initial packet, but Initial packets are protected with keys anyone can +derive from the packet itself — on purpose, so something in the path can read exactly +this — and firegex reads the client's list off it before answering. Your service is then +offered precisely what the client offered, and the client is told the one protocol your +service chose. HTTP/3, DNS over QUIC, a game's protocol or a CTF's own all work the same +way, with no list to keep in step. If a hello cannot be read (a QUIC version firegex does +not know), `h3` is offered instead and the service log says so. **0-RTT is not offered.** Early data is replayable by anyone who watched it go past, and a filter that refused a request has no way to un-deliver the copy your service already @@ -959,6 +963,12 @@ as it happens rather than polled: missing its deadline, a fallback, a ruleset that was refused. These used to go to the backend's stderr, where nobody looks during a competition. +Warnings and errors do not wait for you to open the service: each one also pops up as a +notification wherever you are in the interface (one per service every twenty seconds, +the rest counted into it — except that an error always gets through a warning), and the service's row in the list carries a **WARNING** or +**ERROR** badge for an hour, with the line itself on hover. Clearing the service's log +dismisses it. + It is bounded at both ends. A service under attack refuses thousands of connections a minute; the tail keeps the last few hundred lines and the rest fall off, and lines are coalesced before being sent so a burst arrives as one message instead of thousands. @@ -1003,15 +1013,34 @@ would. Adding, removing, reordering, enabling and disabling filters, patterns and individual `@pyfilter` functions all take effect on live connections **without dropping any of -them**. Both layers swap their configuration -underneath the traffic. +them**. Both layers swap their configuration underneath the traffic. The exception is +adding, removing or reordering a whole filter on the NFQUEUE layer, where each filter +is a process of its own: there the chain of processes is rebuilt, traffic passes +unfiltered for the moment that takes, and the new processes take over the connections +already open from their next packet. + +An edit reaches only the filter it was made to. A Python filter keeps what it knows about +each open connection — its module globals, and how far into a request it has read — +through edits to the other filters; its own new code starts each connection afresh. A +filter that takes over a connection in the middle of a message (a request half uploaded) +does not judge the half it sees: HTTP filters leave that direction alone until the +connection closes, rather than refusing a valid request or calling it malformed. Adding an address does not drop anything either, and removing or moving one costs only the connections on that address. -Changing the service's own definition — its protocol, network layer, or TLS — is -different: that changes what the datapath *is*, so the service is stopped and started -around it, and the connections it was carrying are dropped. The UI says so before you do it. +Changing the service's own definition — its protocol, network layer, TLS or limits — +changes what the datapath *is*, so the service is restarted around it. The connections it +was carrying are not cut: new ones get the new settings, and the ones already open go on. + +Stopping a service does not cut them either. On the NFQUEUE layer the kernel simply stops +asking the filters. The proxy layer terminates the connections it carries, so its engine +is not killed: it is taken out of the rules at once, and new connections go straight to +the service. It carries the ones it already had, unfiltered, until they close, and then +exits. After a restart the old engine does the same, and keeps filtering those connections +the way it did before. Either way it gives up after ten minutes, closing whatever is still +open, and the service's log says so. Only shutting firegex itself down cuts them +straight away. ## See also diff --git a/fgex-lib/firegex/pyfilters/internals/__init__.py b/fgex-lib/firegex/pyfilters/internals/__init__.py index 5978bca..b57a0c8 100644 --- a/fgex-lib/firegex/pyfilters/internals/__init__.py +++ b/fgex-lib/firegex/pyfilters/internals/__init__.py @@ -182,11 +182,14 @@ def get_code_proto(code:str) -> str: #: What a model raises while being built, and the verdict it means. `NotReadyToRun` is #: not here: it is the ordinary "there is nothing to give this filter yet", handled by #: skipping the call rather than by ending the packet. +#: Each is credited to what actually decided it, never to the filter function: those two +#: are raised only when the parser could not read the traffic, and a function the +#: operator wrote was listed as having refused a connection it was never shown. _RAISED_VERDICT = { StreamFullDrop: (Action.DROP, "@MAX_STREAM_SIZE_REACHED"), StreamFullReject: (Action.REJECT, "@MAX_STREAM_SIZE_REACHED"), - DropPacket: (Action.DROP, None), - RejectConnection: (Action.REJECT, None), + DropPacket: (Action.DROP, "@INVALID_ENCODING"), + RejectConnection: (Action.REJECT, "@INVALID_ENCODING"), } diff --git a/fgex-lib/firegex/pyfilters/internals/data.py b/fgex-lib/firegex/pyfilters/internals/data.py index e078aeb..3427953 100644 --- a/fgex-lib/firegex/pyfilters/internals/data.py +++ b/fgex-lib/firegex/pyfilters/internals/data.py @@ -210,7 +210,13 @@ class DataStreamCtx: filter_call_info = _Kept(list) stream_max_size = _Kept(DEFAULT_STREAM_MAX_SIZE, int) full_stream_action = _Kept(FullStreamAction.FLUSH, FullStreamAction) - invalid_encoding_action = _Kept(ExceptionAction.REJECT, ExceptionAction) + #: Traffic the HTTP parser cannot read is carried, and said so, unless the filter file + #: asks for it to be refused. It was refused by default — so a request the service + #: would have taken and llhttp would not (a quirk of one client, a protocol that only + #: resembles HTTP) closed the connection with nothing chosen by anybody, and the block + #: was credited to the filter function as if it had decided it. Refusing is the + #: stricter answer to request smuggling and stays one line away; it is the operator's. + invalid_encoding_action = _Kept(ExceptionAction.ACCEPT, ExceptionAction) data_handler_context = _Kept(dict) def __init__(self, glob: dict, init_pkt: bool = True): diff --git a/fgex-lib/firegex/pyfilters/models/http.py b/fgex-lib/firegex/pyfilters/models/http.py index 58c752b..9b9c1f4 100644 --- a/fgex-lib/firegex/pyfilters/models/http.py +++ b/fgex-lib/firegex/pyfilters/models/http.py @@ -21,6 +21,26 @@ from websockets.extensions.permessage_deflate import PerMessageDeflate from firegex._llhttp import PAUSED_H2_UPGRADE, PAUSED_UPGRADE +#: Whether this process has said that it met connections in the middle of a message. +#: Once per process rather than per connection: a filter taking over from another meets +#: every connection that was open at that moment, and a line each would bury the log. +_told_met_mid_message = False + + +def _met_mid_message() -> None: + global _told_met_mid_message + if _told_met_mid_message: + return + _told_met_mid_message = True + print( + "[info] [http] connections already open when this filter took over were met in " + "the middle of a message; whatever could not be read from where it came in is " + "carried without its HTTP filters until that connection closes. Connections " + "opened from now on are filtered from their first byte.", + flush=True, + ) + + @dataclass class InternalHTTPMessage: """Internal class to handle HTTP messages""" @@ -754,6 +774,11 @@ class InternalBasicHttpMetaClass: ) parser_key = f"{cls._parser_class()}_{'in' if internal_data.current_pkt.is_input else 'out'}" + # A connection this filter met already open, and in the middle of a message: it + # cannot be read from where it came in, and it did nothing wrong. See below. + if internal_data.data_handler_context.get(f"{parser_key}_met_mid_message"): + raise NotReadyToRun() + parser = internal_data.data_handler_context.get(parser_key, None) if parser is None or parser.raised_error: parser: InternalHttpRequest | InternalHttpResponse = ParserType() @@ -816,8 +841,47 @@ class InternalBasicHttpMetaClass: try: parser.parse_data(internal_data.current_pkt.data) except Exception as e: - traceback.print_exc() - match internal_data.invalid_encoding_action: + # **Met already open, and not yet read a single thing cleanly.** The + # datapath says when this filter's state began after its connection did: + # a queued filter that started, or was rebuilt, with the connection open, + # or new code taking over one. Its first bytes here can be the middle of a + # request — the rest of an upload — and llhttp reads that as a malformed + # message. It is not one, and judging it as one refused valid uploads in + # flight whenever a filter changed (with REJECT), or reported a working + # client as "not HTTP" (without). That direction is carried unfiltered + # until the connection closes instead. A connection met at the start of + # a message parses, is in step from then on, and is judged as usual. + if ( + internal_data.filter_glob.get("__firegex_joined_late") + and not internal_data.data_handler_context.get(f"{parser_key}_in_step") + ): + internal_data.data_handler_context[f"{parser_key}_met_mid_message"] = True + _met_mid_message() + raise NotReadyToRun() + action = internal_data.invalid_encoding_action + # Said once per connection and direction, and in one line: the parser is + # started afresh on the next packet, so traffic it cannot read fails on + # every packet — and a traceback each was the whole log, several hundred + # times, saying less than this does. + told = f"{parser_key}_unreadable_told" + if not internal_data.data_handler_context.get(told): + internal_data.data_handler_context[told] = True + print( + f"[warn] [http] this connection's " + f"{'requests' if internal_data.current_pkt.is_input else 'responses'}" + f" are not HTTP this filter can read ({e}): " + + { + ExceptionAction.ACCEPT: "they are carried without its HTTP " + "filters seeing them. FGEX_INVALID_ENCODING_ACTION = " + "ExceptionAction.REJECT refuses such connections instead.", + ExceptionAction.REJECT: "the connection is refused " + "(FGEX_INVALID_ENCODING_ACTION).", + ExceptionAction.DROP: "the packet is dropped " + "(FGEX_INVALID_ENCODING_ACTION).", + }.get(action, "reported as the filter raising."), + flush=True, + ) + match action: case ExceptionAction.REJECT: raise RejectConnection() case ExceptionAction.DROP: @@ -827,6 +891,9 @@ class InternalBasicHttpMetaClass: case ExceptionAction.ACCEPT: raise NotReadyToRun() + # Read cleanly once: from here on this direction is a stream like any other. + internal_data.data_handler_context[f"{parser_key}_in_step"] = True + if any(msg.decoded_too_large for msg in parser.messages): # Too large once decoded is too large: the operator's answer to a stream # past the cap is the answer here too, rather than a second policy nobody diff --git a/fgex-lib/requirements.txt b/fgex-lib/requirements.txt index ff5abd6..7f20801 100644 --- a/fgex-lib/requirements.txt +++ b/fgex-lib/requirements.txt @@ -1,4 +1,7 @@ typer +# Imported directly by the CLI and both simulators for their output. `typer` happens to +# bring it today; `typer-slim` does not, and nothing here should rest on which one it is. +rich pydantic>=2 typing-extensions>=4.7.1 pycryptodome diff --git a/frontend/src/components/MainLayout.tsx b/frontend/src/components/MainLayout.tsx index d142f91..f303179 100644 --- a/frontend/src/components/MainLayout.tsx +++ b/frontend/src/components/MainLayout.tsx @@ -7,6 +7,7 @@ import { getMainPath, HOME_SECTIONS } from '../js/utils'; import { useLocation } from 'react-router'; import { useNavbarStore, useSessionStore } from '../js/store'; import { HiMenu } from "react-icons/hi"; +import ProblemNotifier from './Services/ProblemNotifier'; function MainLayout({ children }:{ children:any }) { @@ -26,6 +27,7 @@ function MainLayout({ children }:{ children:any }) { > + {children} diff --git a/frontend/src/components/Services/AddEditService.tsx b/frontend/src/components/Services/AddEditService.tsx index 3f460bc..0483f42 100644 --- a/frontend/src/components/Services/AddEditService.tsx +++ b/frontend/src/components/Services/AddEditService.tsx @@ -440,8 +440,8 @@ export default function AddEditService({ opened, onClose, edit }: { {edit ? - Changing any of this restarts the service, which drops the connections it - is carrying. Adding an address, or adding and reordering filters, does not. + Changing any of this restarts the service. The connections it is carrying + are not cut; new ones get the new settings. : null} diff --git a/frontend/src/components/Services/ProblemNotifier.tsx b/frontend/src/components/Services/ProblemNotifier.tsx new file mode 100644 index 0000000..929e2f9 --- /dev/null +++ b/frontend/src/components/Services/ProblemNotifier.tsx @@ -0,0 +1,68 @@ +import { showNotification } from '@mantine/notifications'; +import { useQueryClient } from '@tanstack/react-query'; +import { useEffect, useRef } from 'react'; +import { useNavigate } from 'react-router'; +import { socketio } from '../../js/utils'; +import { LogEntry, serviceQueryKey, services } from './utils'; + +/** How long one service's warnings are folded into the one already on screen. */ +const QUIET_MS = 20_000 + +/** + * Says so, wherever the operator is, when a service reports a problem. + * + * The live log is on the service's own page, and it was the only place a warning went: + * an engine that died and came back, a filter raising on every packet, a fallback that + * cost the service its clients' addresses — each of them waited for somebody to open that + * page. Warnings and errors now also arrive as a notification, one per service per + * `QUIET_MS` with the rest counted into the next, because a flood arriving twice — once + * in the log and once as a stack of toasts — informs nobody. Blocks are not problems and + * never notify. + */ +export default function ProblemNotifier() { + const queryClient = useQueryClient() + const navigate = useNavigate() + const told = useRef(new Map()) + + useEffect(() => { + const onLog = (payload: { service_id: string, entries: LogEntry[] }) => { + const problems = payload.entries.filter(e => e.level === "warn" || e.level === "error") + if (problems.length === 0) return + const now = Date.now() + const worst = problems.some(e => e.level === "error") ? "error" : "warn" + const was = told.current.get(payload.service_id) ?? { at: 0, held: 0, worst } + // An error is not folded into a warning already on screen: "restarting it" + // followed by "could not restart it" is the second one that matters, and + // folded it waited for a third that need never come. + const escalates = worst === "error" && was.worst !== "error" + if (now - was.at < QUIET_MS && !escalates) { + told.current.set(payload.service_id, { ...was, held: was.held + problems.length }) + return + } + told.current.set(payload.service_id, { at: now, held: 0, worst }) + const last = problems[problems.length - 1] + const more = was.held + problems.length - 1 + // By name, which on a page that never listed the services means asking for + // them first: an id is not something an operator recognises mid-round. + queryClient.ensureQueryData({ queryKey: serviceQueryKey, queryFn: services.list }) + .catch(() => []) + .then(listed => show( + listed.find(s => s.service_id === payload.service_id)?.name + ?? payload.service_id, + worst, last.text, more, payload.service_id)) + } + const show = (name: string, worst: string, text: string, more: number, id: string) => + showNotification({ + autoClose: 8000, + color: worst === "error" ? "red" : "orange", + title: `${name}: ${worst === "error" ? "something went wrong" : "a warning"}`, + message: text + (more > 0 ? ` (and ${more} more — see its log)` : ""), + style: { cursor: "pointer" }, + onClick: () => navigate(`/services/${id}`), + }) + socketio.on("log", onLog) + return () => { socketio.off("log", onLog) } + }, [queryClient, navigate]) + + return null +} diff --git a/frontend/src/components/Services/ServiceRow.tsx b/frontend/src/components/Services/ServiceRow.tsx index 1d9fb34..85b43d0 100644 --- a/frontend/src/components/Services/ServiceRow.tsx +++ b/frontend/src/components/Services/ServiceRow.tsx @@ -4,7 +4,7 @@ import { useState } from 'react'; import { FaPlay, FaStop, FaTrash } from 'react-icons/fa'; import { IoSettingsSharp } from 'react-icons/io5'; import { MdChevronRight, MdMoreHoriz } from 'react-icons/md'; -import { TbHexagon, TbShieldLock } from 'react-icons/tb'; +import { TbAlertTriangle, TbHexagon, TbShieldLock } from 'react-icons/tb'; import { bareAddress, errorNotify, isMediumScreen, okNotify } from '../../js/utils'; import YesNoModal from '../YesNoModal'; import AddEditService from './AddEditService'; @@ -38,6 +38,29 @@ export const transportSummary = (transport: string, proto: string) => { : "The connection is terminated and reopened, so the kernel reassembles and the chain has no length limit. It also carries bulk traffic several times faster than NFQUEUE, which pays a userspace round trip per packet; what it costs is fail-open being rebuilt in userspace rather than guaranteed by the kernel. The service still sees the real client address." } +/** How long a warning stays on the row after it happened, unless the log is cleared first. */ +const PROBLEM_SHOWN_MS = 60 * 60 * 1000 + +/** + * The newest warning or error in the service's log, on the row, for an hour. + * + * The log is on the service's own page, and a problem there waits for somebody to open + * it — the notification says it once, this keeps saying it for whoever looks at the list + * afterwards. Clearing the log is how it is dismissed. + */ +export function ProblemBadge({ service }: { service: Service }) { + const problem = service.problem + if (!problem || Date.now() - problem.at > PROBLEM_SHOWN_MS) return null + const error = problem.level === "error" + return + }> + {error ? "ERROR" : "WARNING"} + + +} + /** Where a service is reachable, short enough to sit on one line. */ export const addressSummary = (addresses: Address[]) => { if (!addresses || addresses.length === 0) return "no address yet" @@ -135,6 +158,7 @@ export default function ServiceRow({ service, onClick }: { service: Service, onC {service.status.toUpperCase()} + diff --git a/frontend/src/components/Services/utils.ts b/frontend/src/components/Services/utils.ts index 5a104b5..6e16c8c 100644 --- a/frontend/src/components/Services/utils.ts +++ b/frontend/src/components/Services/utils.ts @@ -218,6 +218,20 @@ export type Service = { addresses: Address[], n_filters: number, n_blocked: number, + /** + * The newest warning or error still in the service's log. The log lives on the + * service's own page, and this is what lets the list say something went wrong without + * that page being open. Clearing the log dismisses it. + */ + problem?: Problem | null, +} + +export type Problem = { + /** Unix milliseconds. */ + at: number, + /** warn | error */ + level: string, + text: string, } export type ServiceAddForm = { diff --git a/frontend/src/js/demo.ts b/frontend/src/js/demo.ts index 6fe74c5..d495b7d 100644 --- a/frontend/src/js/demo.ts +++ b/frontend/src/js/demo.ts @@ -234,6 +234,10 @@ const addLog = (service_id: string, level: string, text: string) => { logs[service_id] = [...(logs[service_id] ?? []), entry].slice(-500) for (const cb of listeners["log"] ?? []) cb({ service_id, entries: [entry] }) } +// One warning already in the tail, so the list shows what a problem looks like: the line a +// service at its connection limit really writes. +addLog(svcShop, "warn", "the 512 connection limit turned away 37 more: they were refused. " + + "Raise the limit, or find out who is holding connections open.") let ticker: ReturnType | null = null const emit = (...tags: string[][]) => { @@ -261,6 +265,13 @@ const startTicker = () => { } if (changed) emit(["services"]) }, 4000) + // And one warning a little after the start, the way a real one arrives: as a + // notification wherever you are, and as the badge on the service's row. + setTimeout(() => { + addLog(svcShop, "warn", "the 512 connection limit turned away 12 more: they were refused. " + + "Raise the limit, or find out who is holding connections open.") + emit(["services"]) + }, 10_000) } let welcomed = false @@ -325,6 +336,9 @@ const decorate = (s: Json) => ({ addresses: addressesOf(s.service_id), n_filters: state.filters.filter(f => f.service_id === s.service_id).length, n_blocked: state.filters.filter(f => f.service_id === s.service_id).reduce((acc, f) => acc + f.blocked, 0), + // The newest warning or error in its tail, as the backend reports it. + problem: [...(logs[s.service_id] ?? [])].reverse() + .find(e => e.level === "warn" || e.level === "error") ?? null, }) /** @@ -589,7 +603,7 @@ const routes: [string, RegExp, Handler][] = [ } }], ["GET", /^services\/([^/]+)\/logs$/, m => { service(m[1]); return logs[m[1]] ?? [] }], - ["DELETE", /^services\/([^/]+)\/logs$/, m => { service(m[1]); logs[m[1]] = []; return ok }], + ["DELETE", /^services\/([^/]+)\/logs$/, m => { service(m[1]); logs[m[1]] = []; emit(["services"]); return ok }], // ---- services: the filter chain ["GET", /^services\/([^/]+)\/filters$/, m => { diff --git a/frontend/src/pages/Firewall/SettingsModal.tsx b/frontend/src/pages/Firewall/SettingsModal.tsx index 9fe3a33..2170024 100644 --- a/frontend/src/pages/Firewall/SettingsModal.tsx +++ b/frontend/src/pages/Firewall/SettingsModal.tsx @@ -43,8 +43,8 @@ export function SettingsModal({ opened, onClose }:{ opened:boolean, onClose:()=> setSettings({...settings, allow_upnp:v.target.checked})}/> setSettings({...settings, drop_invalid:v.target.checked})}/> setSettings({...settings, allow_dhcp:v.target.checked})}/> - setSettings({...settings, allow_dnat:v.target.checked})}/> diff --git a/run.py b/run.py index 96d1e69..7ae9d7f 100755 --- a/run.py +++ b/run.py @@ -149,10 +149,9 @@ def load_config(): "proxy_ip_header": None, "unsafe_disable_auth": False, # Anything the engine or the backend reads straight out of the environment and - # run.py has no opinion about: the filter deadline, the QUIC ALPN list, the - # stream size cap. Kept here because run.py **rewrites** the compose file on + # run.py has no opinion about: the filter deadline, the stream size cap. Kept here because run.py **rewrites** the compose file on # every start, so a variable hand-edited into it is gone by the next one — which - # is what the documentation telling an operator to set FGEX_PROXY_QUIC_ALPN + # is what the documentation telling an operator to set an engine variable # "in the container's environment" silently ran into. "env": {}, } @@ -216,7 +215,7 @@ def gen_args(args_to_parse: list[str]|None = None): parser_start.add_argument('--port', "-p", type=int, required=False, help=f'Port where open the web service of the firewall (default from config: {config["port"]})', default=config["port"]) parser_start.add_argument('--host', required=False, help=f'Host IP address to bind the service to (default from config: {config["host"]})', default=config["host"]) parser_start.add_argument('--socket-dir', required=False, type=str, help=f'Listen on socket_dir/firegex.sock instead of TCP (default from config: {config["socket_dir"]})', default=config["socket_dir"]) - parser_start.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_QUIC_ALPN. Stored, so later runs keep it.') + parser_start.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_FILTER_TIMEOUT_MS. Stored, so later runs keep it.') parser_start.add_argument('--allowed-ips', required=False, type=str, help=f'Comma-separated list of CIDR addresses allowed to contact firegex (default from config: {config.get("allowed_ips")})', default=config.get("allowed_ips")) parser_start.add_argument('--proxy-ip-header', required=False, type=str, help=f'Header name to read the client IP from (default from config: {config.get("proxy_ip_header")})', default=config.get("proxy_ip_header")) parser_start.add_argument('--unsafe-disable-auth', action=argparse.BooleanOptionalAction, default=config.get("unsafe_disable_auth", False), help='UNSAFE: disable Firegex password/JWT authentication, making every request that reaches firegex a full administrator (for a trusted reverse proxy only)') @@ -234,7 +233,7 @@ def gen_args(args_to_parse: list[str]|None = None): parser_restart.add_argument('--port', "-p", type=int, required=False, help=f'Port where open the web service of the firewall (default from config: {config["port"]})', default=config["port"]) parser_restart.add_argument('--host', required=False, help=f'Host IP address to bind the service to (default from config: {config["host"]})', default=config["host"]) parser_restart.add_argument('--socket-dir', required=False, type=str, help=f'Listen on socket_dir/firegex.sock instead of TCP (default from config: {config["socket_dir"]})', default=config["socket_dir"]) - parser_restart.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_QUIC_ALPN. Stored, so later runs keep it.') + parser_restart.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_FILTER_TIMEOUT_MS. Stored, so later runs keep it.') parser_restart.add_argument('--allowed-ips', required=False, type=str, help=f'Comma-separated list of CIDR addresses allowed to contact firegex (default from config: {config.get("allowed_ips")})', default=config.get("allowed_ips")) parser_restart.add_argument('--proxy-ip-header', required=False, type=str, help=f'Header name to read the client IP from (default from config: {config.get("proxy_ip_header")})', default=config.get("proxy_ip_header")) parser_restart.add_argument('--unsafe-disable-auth', action=argparse.BooleanOptionalAction, default=config.get("unsafe_disable_auth", False), help='UNSAFE: disable Firegex password/JWT authentication, making every request that reaches firegex a full administrator (for a trusted reverse proxy only)') @@ -254,7 +253,7 @@ def gen_args(args_to_parse: list[str]|None = None): parser_config.add_argument('--host', required=False, help='Set default host IP address to bind the service to') parser_config.add_argument('--socket-dir', required=False, type=str, help=f'Listen on socket_dir/firegex.sock instead of TCP (default from config: {config["socket_dir"]})', default=config["socket_dir"]) parser_config.add_argument('--password', required=False, type=str, nargs='?', const='', help='Change the password of the firewall (omit the value to be prompted for it interactively)') - parser_config.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_QUIC_ALPN. Stored, so later runs keep it.') + parser_config.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_FILTER_TIMEOUT_MS. Stored, so later runs keep it.') parser_config.add_argument('--allowed-ips', required=False, type=str, help=f'Comma-separated list of CIDR addresses allowed to contact firegex (default from config: {config.get("allowed_ips")})', default=config.get("allowed_ips")) parser_config.add_argument('--proxy-ip-header', required=False, type=str, help=f'Header name to read the client IP from (default from config: {config.get("proxy_ip_header")})', default=config.get("proxy_ip_header")) parser_config.add_argument('--unsafe-disable-auth', action=argparse.BooleanOptionalAction, default=None, help='UNSAFE: turn Firegex password/JWT authentication off or back on, on the running instance and for the next start') @@ -410,6 +409,19 @@ def get_web_interface_url(): display_host = "localhost" if args.host == "0.0.0.0" else args.host return f"http://{display_host}:{args.port}" +def compose_env(key: str, value) -> str: + """One operator-supplied `KEY=VALUE` for the compose file, as compose will read it back. + + Quoted, because `dict_to_yaml` writes scalars as they come and this value is the + operator's: a `: ` in it made the entry a mapping compose refuses, and a ` #` cut it + short as a comment. And every `$` doubled, because compose interpolates variables in its + own file — a value naming `$HOME` would have arrived as a path. The standalone start + passes the same entries through `shlex.quote` for the neighbouring reason. + """ + import json + return json.dumps(f"{key}={value}".replace("$", "$$")) + + def write_compose(skip_password = True): psw_set = get_password() if not skip_password else None with open(g.composefile,"wt") as compose: @@ -434,7 +446,7 @@ def write_compose(skip_password = True): *(["UNSAFE_DISABLE_AUTH=1"] if getattr(args, 'unsafe_disable_auth', False) else []), # Whatever the operator asked to be here and run.py has no # opinion about. Last, so it reads as the addition it is. - *(f"{k}={v}" for k, v in sorted(getattr(args, 'env_vars', {}).items())), + *(compose_env(k, v) for k, v in sorted(getattr(args, 'env_vars', {}).items())), ], "volumes": [ "firegex_data:/execute/db", @@ -497,7 +509,7 @@ def write_compose(skip_password = True): *([f"ALLOWED_IPS={args.allowed_ips}"] if getattr(args, 'allowed_ips', None) else []), *([f"PROXY_IP_HEADER={args.proxy_ip_header}"] if getattr(args, 'proxy_ip_header', None) else []), *(["UNSAFE_DISABLE_AUTH=1"] if getattr(args, 'unsafe_disable_auth', False) else []), - *(f"{k}={v}" for k, v in sorted(getattr(args, 'env_vars', {}).items())), + *(compose_env(k, v) for k, v in sorted(getattr(args, 'env_vars', {}).items())), ], "volumes": [ "firegex_data:/execute/db" diff --git a/tests/helpers/quicserver.py b/tests/helpers/quicserver.py index be4dd87..359b936 100644 --- a/tests/helpers/quicserver.py +++ b/tests/helpers/quicserver.py @@ -25,7 +25,7 @@ try: from aioquic.h3.connection import H3_ALPN, H3Connection from aioquic.h3.events import DataReceived, HeadersReceived from aioquic.quic.configuration import QuicConfiguration - from aioquic.quic.events import ConnectionTerminated + from aioquic.quic.events import ConnectionTerminated, StreamDataReceived HAVE_AIOQUIC = True except ImportError: # pragma: no cover - depends on what is installed HAVE_AIOQUIC = False @@ -49,11 +49,16 @@ class QuicEcho: the suite is not. Stopping it closes the loop, which is what ends the server. """ - def __init__(self, port: int, cert: str, key: str, ipv6: bool = False): + def __init__(self, port: int, cert: str, key: str, ipv6: bool = False, + alpn: list[str] | None = None): self.port = port self.cert = cert self.key = key self.ipv6 = ipv6 + #: What it negotiates. HTTP/3 unless told otherwise; anything else is a service + #: that echoes each stream back as it is — QUIC carrying a protocol of its own, + #: which is what a CTF's QUIC service usually is when it is not a web one. + self.alpn = alpn #: Every address a connection arrived from. What the *service* saw, which is the #: only way to check that the client's own address survived the relay. self.seen_peers: list[tuple] = [] @@ -89,11 +94,12 @@ class QuicEcho: handle.write(material) os.chmod(path, 0o600) paths.append(path) - config = QuicConfiguration(is_client=False, alpn_protocols=H3_ALPN) + config = QuicConfiguration(is_client=False, alpn_protocols=self.alpn or H3_ALPN) config.load_cert_chain(paths[0], paths[1]) seen = self.seen_peers + protocol = _EchoProtocol if not self.alpn else _RawEchoProtocol await serve(self.host, self.port, configuration=config, - create_protocol=lambda *a, **kw: _EchoProtocol(seen, *a, **kw)) + create_protocol=lambda *a, **kw: protocol(seen, *a, **kw)) for path in paths: os.unlink(path) self._ready.set() @@ -144,6 +150,22 @@ if HAVE_AIOQUIC: self._http.send_data(stream_id, answer, end_stream=True) self.transmit() + class _RawEchoProtocol(QuicConnectionProtocol): + """Each stream back as it came, once the client has finished sending it.""" + + def __init__(self, seen_peers, *args, **kwargs): + super().__init__(*args, **kwargs) + self._streams: dict[int, bytes] = {} + + def quic_event_received(self, event): + if isinstance(event, StreamDataReceived): + self._streams[event.stream_id] = \ + self._streams.get(event.stream_id, b"") + event.data + if event.end_stream: + self._quic.send_stream_data( + event.stream_id, self._streams.pop(event.stream_id), end_stream=True) + self.transmit() + class _RequestingProtocol(QuicConnectionProtocol): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -208,6 +230,27 @@ async def _ask(host: str, port: int, method: str, path: str, ) +async def _raw(host: str, port: int, alpn: str, payload: bytes, timeout: float) -> bytes: + config = QuicConfiguration(is_client=True, alpn_protocols=[alpn]) + config.verify_mode = ssl.CERT_NONE + async with connect(host, port, configuration=config) as client: + reader, writer = await client.create_stream() + writer.write(payload) + writer.write_eof() + return await asyncio.wait_for(reader.read(), timeout) + + +def quic_exchange(host: str, port: int, alpn: str, payload: bytes, + timeout: float = 8.0) -> bytes | None: + """One stream over a QUIC connection negotiating `alpn`: what came back, or `None` + when no connection could be made or nothing came back in time.""" + try: + return asyncio.run(asyncio.wait_for(_raw(host, port, alpn, payload, timeout), + timeout + 2)) + except (ConnectionError, OSError, asyncio.TimeoutError, TimeoutError): + return None + + def authority(host: str, port: int) -> str: """What goes in `:authority`, bracketed where the host needs it. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 00906d7..46c3d29 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -301,11 +301,12 @@ def quic_stand_in(certificate): from helpers.quicserver import QuicEcho started = [] - def _serve(ipv6: bool = False, port: int | None = None) -> "QuicEcho": + def _serve(ipv6: bool = False, port: int | None = None, + alpn: list[str] | None = None) -> "QuicEcho": cert, key = certificate("::1" if ipv6 else "127.0.0.1") # A UDP probe: this one is going to bind a QUIC endpoint, and a port free for TCP # says nothing about whether it is free for UDP. - echo = QuicEcho(port or free_port(ipv6, udp=True), cert, key, ipv6) + echo = QuicEcho(port or free_port(ipv6, udp=True), cert, key, ipv6, alpn=alpn) echo.start() echo.material = (cert, key) started.append(echo) diff --git a/tests/integration/test_filters_python.py b/tests/integration/test_filters_python.py index f6ba826..5818988 100644 --- a/tests/integration/test_filters_python.py +++ b/tests/integration/test_filters_python.py @@ -7,6 +7,7 @@ taking `(data, direction)`, so a filter written from the docs worked on one tran failed on the other. """ +import socket import time import pytest @@ -201,6 +202,67 @@ def test_switching_one_function_off_stops_it_deciding_without_touching_the_code( assert "refuse_traversal" in stored, "the code was edited to switch a function off" +def test_new_code_reaches_a_connection_already_open(api, protected, inspecting_layer): + """An edit is for the attack in progress, which is on a connection already open. + + The proxy layer hands a connection's next chunk to the new chain, and `cppregex` + resets its matchers on a new configuration. `cpproxy` kept a version number for the + same purpose and never read it, so a stream went on running the code it had started + with for as long as it stayed open: a check added against one connection did not reach + that connection. + """ + service_id, server, port = protected(inspecting_layer, name="py-live") + filter_id = add_python_filter(api, service_id, filter_code.BLOCK_MARKER) + start_and_settle(api, service_id) + + server.connect_client(timeout=3) + try: + server.send_packet(b"carrying NEWRULE") + assert server.recv_packet() == b"carrying NEWRULE", \ + "the connection did not work to begin with" + assert api.services_set_code( + service_id, filter_id, filter_code.BLOCK_MARKER.replace("PYBLOCK", "NEWRULE")) + time.sleep(RELOAD) + server.send_packet(b"carrying NEWRULE") + assert server.recv_packet() != b"carrying NEWRULE", \ + "the connection that was already open went on running the old code" + finally: + server.close_client() + + +def test_a_python_filter_added_later_reaches_a_connection_already_open( + api, protected, inspecting_layer): + """The same as for patterns, for the binary that runs code. + + `cpproxy` read each stream's direction from which end it had seen open it, so besides + dropping a stream it met halfway it would have had nothing to go on. It takes the + direction from the packet now, which the kernel knows either way. + """ + service_id, server, port = protected(inspecting_layer, name="py-late") + add_regex_filter(api, service_id, "UNRELATED") + start_and_settle(api, service_id) + + server.connect_client(timeout=3) + try: + server.send_packet(b"carrying PYBLOCK") + assert server.recv_packet() == b"carrying PYBLOCK", \ + "the connection did not work to begin with" + add_python_filter(api, service_id, filter_code.BLOCK_MARKER) + time.sleep(RELOAD + 1) + server.send_packet(b"still fine") + assert server.recv_packet() == b"still fine", \ + "the connection stopped working when the new filter picked it up" + try: + server.send_packet(b"carrying PYBLOCK") + got = server.recv_packet() + except OSError: + got = False + assert got != b"carrying PYBLOCK", \ + "a connection that was already open was never inspected by the new filter" + finally: + server.close_client() + + def test_saving_different_code_reconciles_the_list_of_functions(api, http_filter): """The code is what decides which functions exist. @@ -273,3 +335,130 @@ def test_removing_a_filter_leaves_the_rest_of_the_chain_running(api, protected, time.sleep(1.0) assert channel.gets_through(b"carrying PYBLOCK"), "the removed filter still decides" assert channel.is_blocked(b"carrying BLOCKME"), "the rest of the chain stopped" + + +#: Strict about what it cannot read, as an operator guarding against request smuggling +#: would be — the setting under which judging half a request refused it. +STRICT_HTTP = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT, ExceptionAction +from firegex.pyfilters.models import HttpRequest + +FGEX_INVALID_ENCODING_ACTION = ExceptionAction.REJECT + +@pyfilter +def refuse_marked_url(req: HttpRequest): + return REJECT if "PYBLOCK" in (req.url or "") else ACCEPT +""" + + +def _answer(sock: socket.socket) -> bytes: + """One HTTP response off `sock`, or what arrived before it closed or went quiet.""" + got = b"" + try: + while b"\r\n\r\n" not in got: + chunk = sock.recv(65536) + if not chunk: + return got + got += chunk + head, _, body = got.partition(b"\r\n\r\n") + length = next((int(line.split(b":")[1]) for line in head.split(b"\r\n") + if line.lower().startswith(b"content-length:")), 0) + while len(body) < length: + chunk = sock.recv(65536) + if not chunk: + break + body += chunk + return head + b"\r\n\r\n" + body + except OSError: + return got + + +def test_a_request_in_flight_when_the_filter_changes_is_not_judged_by_its_second_half( + api, service, http_stand_in, inspecting_layer): + """A filter meeting a connection halfway leaves what it cannot read alone. + + Half an upload is sent, the filter changes — a queued one is added, which rebuilds the + chain, or a proxied one gets new code — and the other half follows. Whatever runs the + filter now meets that connection in the middle of a body. It used to hand the body to + the HTTP parser as if it were the start of a request, which it is not: with the + strict setting on, a valid upload was refused in flight, and without it the log said + a working client was not speaking HTTP. Connections opened afterwards are filtered + from their first byte, as always. + """ + http = http_stand_in(inspecting_layer.ipv6) + service_id = service(f"py-inflight-{http.port}", inspecting_layer.ip, http.port, + inspecting_layer.transport) + queued = inspecting_layer.transport == "nfqueue" + if queued: + add_regex_filter(api, service_id, "UNRELATED") + else: + filter_id = add_python_filter(api, service_id, STRICT_HTTP) + start_and_settle(api, service_id) + + body = b"x" * 3000 + with socket.create_connection((inspecting_layer.ip, http.port), timeout=3) as sock: + sock.sendall(b"POST /upload HTTP/1.1\r\nHost: svc\r\nContent-Length: 3000\r\n\r\n" + + body[:1500]) + time.sleep(0.3) + if queued: + add_python_filter(api, service_id, STRICT_HTTP) + else: + assert api.services_set_code(service_id, filter_id, STRICT_HTTP + "\n# edited\n") + time.sleep(RELOAD + 1) + sock.sendall(body[1500:]) + assert b"sent 3000 bytes" in _answer(sock), \ + "a valid upload in flight was refused when the filter changed" + + said = [e["text"] for e in api.services_logs(service_id) + if e["level"] in ("warn", "error") and "not HTTP" in e["text"]] + assert not said, f"a working client was reported as not speaking HTTP: {said}" + + with socket.create_connection((inspecting_layer.ip, http.port), timeout=3) as sock: + sock.sendall(b"GET /PYBLOCK HTTP/1.1\r\nHost: svc\r\n\r\n") + assert b"200" not in _answer(sock), "a new connection was not filtered" + + +#: Remembers, per connection, whether the client has said FIRST — so a verdict on +#: SECOND says whether the filter's state for the connection survived. +REMEMBERS = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT +from firegex.pyfilters.models import RawPacket + +said_first = False + +@pyfilter +def second_after_first(packet: RawPacket): + global said_first + if packet.is_input and b"FIRST" in packet.data: + said_first = True + return REJECT if said_first and b"SECOND" in packet.data else ACCEPT +""" + + +def test_an_edit_to_another_filter_leaves_a_python_filters_connections_alone( + api, protected, inspecting_layer): + """A Python filter keeps its per-connection state through edits that are not its own. + + Every edit reloads the whole chain. On both layers that restarted the Python filters + too — a new worker on the proxy, a clean slate in `cpproxy` — so changing a pattern + in one filter wiped what another had kept about every open connection: its module + globals, and the place its HTTP parser had reached in a request. + """ + service_id, server, port = protected(inspecting_layer, name="py-kept") + add_python_filter(api, service_id, REMEMBERS) + regex_filter, _ = add_regex_filter(api, service_id, "UNRELATED") + start_and_settle(api, service_id) + + server.connect_client(timeout=3) + try: + server.send_packet(b"FIRST") + assert server.recv_packet() == b"FIRST", "the connection did not work to begin with" + assert api.services_add_regex(service_id, regex_filter, "ALSOUNRELATED", mode="C") + time.sleep(RELOAD + 1) + try: + server.send_packet(b"SECOND") + got = server.recv_packet() + except OSError: + got = False + assert got != b"SECOND", \ + "a pattern added to another filter wiped this filter's state for the connection" + finally: + server.close_client() diff --git a/tests/integration/test_filters_regex.py b/tests/integration/test_filters_regex.py index 10a0f4e..f269d55 100644 --- a/tests/integration/test_filters_regex.py +++ b/tests/integration/test_filters_regex.py @@ -173,3 +173,66 @@ def test_switching_a_pattern_off_stops_it_deciding(api, blocking): assert api.services_edit_regex(service_id, filter_id, pattern_id, active=True) time.sleep(RELOAD) assert channel.is_blocked(b"carrying BLOCKME"), "it did not start deciding again" + + +def _held_open(server, payload: bytes) -> bytes | bool: + """Send `payload` on the connection the test is holding and return the answer.""" + try: + server.send_packet(payload) + except OSError: + return False + return server.recv_packet() + + +def test_a_filter_added_to_a_running_service_reaches_a_connection_already_open( + api, protected, inspecting_layer): + """A new filter is for the attack in progress, which is on a connection already open. + + On the NFQUEUE layer a new filter is a new process, and it meets that connection + halfway through. The stream follower it used threw away any stream whose handshake it + had not seen, so every connection open when a filter was added — or when the chain was + rebuilt for any other reason — went uninspected until it closed. + """ + service_id, server, port = protected(inspecting_layer, name="rx-late") + add_regex_filter(api, service_id, "UNRELATED", name="first") + start_and_settle(api, service_id) + + server.connect_client(timeout=3) + try: + assert _held_open(server, b"hello") == b"hello", \ + "the connection did not work to begin with" + add_regex_filter(api, service_id, "BLOCKME", name="second") + time.sleep(RELOAD + 1) + # Picked up, not broken: the connection has to go on working for everything else. + assert _held_open(server, b"still fine") == b"still fine", \ + "the connection stopped working when the new filter picked it up" + assert _held_open(server, b"carrying BLOCKME") != b"carrying BLOCKME", \ + "a connection that was already open was never inspected by the new filter" + finally: + server.close_client() + + +def test_a_connection_open_before_the_service_started_is_inspected( + api, protected, inspecting_layer): + """Starting a service protects the connections it finds, not only the ones after. + + Only the queued layer can: it filters packets, so a connection that predates it is as + visible as any other. The proxy layer takes connections as they are made, and one + already made goes on straight to the service. + """ + if inspecting_layer.transport != "nfqueue": + pytest.skip("this is the queued layer's guarantee") + service_id, server, port = protected(inspecting_layer, name="rx-before") + add_regex_filter(api, service_id, "BLOCKME") + + server.connect_client(timeout=3) + try: + assert _held_open(server, b"hello") == b"hello", \ + "the connection did not work to begin with" + start_and_settle(api, service_id) + assert _held_open(server, b"still fine") == b"still fine", \ + "the connection stopped working when the service started" + assert _held_open(server, b"carrying BLOCKME") != b"carrying BLOCKME", \ + "a connection open before the service started was never inspected" + finally: + server.close_client() diff --git a/tests/integration/test_quic.py b/tests/integration/test_quic.py index 635a83d..06f07f6 100644 --- a/tests/integration/test_quic.py +++ b/tests/integration/test_quic.py @@ -276,3 +276,36 @@ def test_what_the_service_speaks_can_be_changed_while_it_runs(api, service, quic answer = h3_request(quic.host, quic.port, "/which") assert answer.endswith("bytes"), \ f"the edit was saved and the relay went on speaking HTTP/3: {answer!r}" + + +# --- what a QUIC service negotiates -------------------------------------------------- + + +@needs_quic +def test_a_quic_service_that_is_not_http3_is_reached_with_nothing_to_configure( + api, service, quic_stand_in): + """QUIC carries whatever its two ends agree on, and the engine has to say it to both. + + It offered the service a list of its own — one environment variable for the whole + instance, `h3` unless firegex was restarted with another — because the client's hello + was taken to be unreadable. So a QUIC service speaking anything but HTTP/3 was not + reachable through firegex until somebody found that. The hello is read off the + client's first packet now and offered to the service as it is, the way the TLS path + has always mirrored it; there is nothing to set. + """ + from helpers.quicserver import quic_exchange + + server = quic_stand_in(alpn=["fgx-echo"]) + cert, key = server.material + service_id = service(f"quic-alpn-{server.port}", "127.0.0.1", server.port, "proxy", + proto="quic", tls_cert=cert, tls_key=key) + add_regex_filter(api, service_id, "BLOCKME") + start_and_settle(api, service_id) + + assert quic_exchange(server.host, server.port, "fgx-echo", b"hello") == b"hello", \ + "a QUIC service that is not HTTP/3 was not reached" + assert quic_exchange(server.host, server.port, "fgx-echo", b"x BLOCKME", timeout=3) \ + != b"x BLOCKME", "a stream carrying a blocked pattern reached the service" + # And one speaking something the service does not is refused by the service, as it + # would be with nothing in the way. + assert quic_exchange(server.host, server.port, "fgx-other", b"hello", timeout=3) is None diff --git a/tests/integration/test_remote_clients.py b/tests/integration/test_remote_clients.py new file mode 100644 index 0000000..41f641f --- /dev/null +++ b/tests/integration/test_remote_clients.py @@ -0,0 +1,419 @@ +"""A client on another host. + +Every other test here dials from the host firegex runs on, and a reply to such a client is +delivered locally whether or not the rules brought it home — so a return path that only +works for a client on this host passes all of them, and fails for every client a CTF +service actually has. Here the client is in a network namespace behind a veth pair: its +packets arrive on an interface, its address is none of this host's, and a reply that is +not brought home is routed out to it, past the engine waiting for it. +""" + +import os +import socket +import subprocess +import threading +import time + +import pytest + +from integration.conftest import add_regex_filter, start_and_settle +from helpers.host import as_root +from helpers.net import free_port + +pytestmark = [pytest.mark.instance, pytest.mark.root] + +NETNS = "fgextest" +HOST_SIDE, CLIENT_SIDE = "fgxt0", "fgxt1" +HOST_IP, CLIENT_IP = "10.199.77.1", "10.199.77.2" + + +def _take_down() -> None: + as_root("ip", "netns", "del", NETNS) + as_root("ip", "link", "del", HOST_SIDE) + + +@pytest.fixture +def elsewhere(datapath_in_our_path): + """Run a snippet of Python on a client that is not this host, and read what it printed.""" + if not datapath_in_our_path: + pytest.skip("firegex's datapath is not in front of this host") + _take_down() + for step in ( + ("ip", "netns", "add", NETNS), + ("ip", "link", "add", HOST_SIDE, "type", "veth", "peer", "name", CLIENT_SIDE), + ("ip", "link", "set", CLIENT_SIDE, "netns", NETNS), + ("ip", "addr", "add", f"{HOST_IP}/24", "dev", HOST_SIDE), + ("ip", "link", "set", HOST_SIDE, "up"), + ("ip", "netns", "exec", NETNS, "ip", "addr", "add", f"{CLIENT_IP}/24", + "dev", CLIENT_SIDE), + ("ip", "netns", "exec", NETNS, "ip", "link", "set", CLIENT_SIDE, "up"), + ("ip", "netns", "exec", NETNS, "ip", "link", "set", "lo", "up"), + ): + if as_root(*step) is None: + _take_down() + pytest.skip("cannot stage a second host here: no root, or no network namespaces") + + def run(code: str) -> str: + return (as_root("ip", "netns", "exec", NETNS, "python3", "-c", code) or "").strip() + + yield run + _take_down() + + +SERVICE_NETNS = "fgextest-svc" +SERVICE_HOST_SIDE, SERVICE_SIDE = "fgxs0", "fgxs1" +CONTAINER_IP = "10.199.78.2" +CONTAINER_TABLE = "fgextest_dnat" + +#: What the "container" runs: TCP and UDP on 8080, echoing — except `WHO`, answered with +#: the address the service saw, which is how transparency is asked about. +CONTAINER_SERVICE = r""" +import socket, threading +def tcp(): + s = socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("0.0.0.0", 8080)); s.listen(64) + def serve(c, peer): + with c: + while True: + d = c.recv(4096) + if not d: return + c.sendall(peer[0].encode() if d == b"WHO" else d) + while True: + c, peer = s.accept() + threading.Thread(target=serve, args=(c, peer), daemon=True).start() +def udp(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(("0.0.0.0", 8080)) + while True: + d, peer = s.recvfrom(4096) + s.sendto(peer[0].encode() if d == b"WHO" else d, peer) +threading.Thread(target=udp, daemon=True).start() +tcp() +""" + +#: Forwarding between the client's link and the container's, both ways. A host running +#: Docker drops forwarded traffic by policy, and this is how a runtime lets its own +#: published ports through it. +FORWARD_RULES = [("-i", HOST_SIDE, "-o", SERVICE_HOST_SIDE), + ("-i", SERVICE_HOST_SIDE, "-o", HOST_SIDE)] + + +def _root_command(*argv: str) -> list[str]: + return list(argv) if os.geteuid() == 0 else ["sudo", "-n", *argv] + + +def _rewrites(published: int) -> list[tuple[str, ...]]: + """Docker's own rule for a published port, in its own tool: `iptables` in `nat`.""" + return [("-t", "nat", chain, "-d", HOST_IP, "-p", proto, "--dport", str(published), + "-j", "DNAT", "--to-destination", f"{CONTAINER_IP}:8080") + for chain in ("PREROUTING", "OUTPUT") for proto in ("tcp", "udp")] + + +_staged: list[tuple[str, ...]] = [] + + +def _undo_rewrites() -> None: + while _staged: + rule = _staged.pop() + as_root("iptables", *rule[:2], "-D", *rule[2:]) + + +def _take_container_down() -> None: + as_root("pkill", "-f", "fgextest-container") + as_root("nft", "delete", "table", "ip", CONTAINER_TABLE) + _undo_rewrites() + for rule in FORWARD_RULES: + as_root("iptables", "-D", "FORWARD", *rule, "-j", "ACCEPT") + as_root("ip", "netns", "del", SERVICE_NETNS) + as_root("ip", "link", "del", SERVICE_HOST_SIDE) + + +@pytest.fixture +def container(elsewhere): + """A service the way a container runtime publishes one, on the far side of a DNAT. + + The service lives in a namespace of its own, and a port of this host's is rewritten to + it at `dstnat` in `nat PREROUTING` and `nat OUTPUT` — the priority, the hooks and the + shape of Docker's own rule — without needing Docker. It is the deployment a CTF + service almost always has, and the rewrite is what the rest of the suite never puts + in the way: every other service here listens on this host, where a reply is local. + """ + _take_container_down() + for step in ( + ("ip", "netns", "add", SERVICE_NETNS), + ("ip", "link", "add", SERVICE_HOST_SIDE, "type", "veth", "peer", "name", SERVICE_SIDE), + ("ip", "link", "set", SERVICE_SIDE, "netns", SERVICE_NETNS), + ("ip", "addr", "add", "10.199.78.1/24", "dev", SERVICE_HOST_SIDE), + ("ip", "link", "set", SERVICE_HOST_SIDE, "up"), + ("ip", "netns", "exec", SERVICE_NETNS, "ip", "addr", "add", f"{CONTAINER_IP}/24", + "dev", SERVICE_SIDE), + ("ip", "netns", "exec", SERVICE_NETNS, "ip", "link", "set", SERVICE_SIDE, "up"), + ("ip", "netns", "exec", SERVICE_NETNS, "ip", "link", "set", "lo", "up"), + ("ip", "netns", "exec", SERVICE_NETNS, "ip", "route", "add", "default", + "via", "10.199.78.1"), + ): + if as_root(*step) is None: + _take_container_down() + pytest.skip("cannot stage a container's network here") + # Best effort: without them the client elsewhere cannot reach the container even with + # no firegex in the way, and the test says so rather than blaming firegex. + for rule in FORWARD_RULES: + as_root("iptables", "-I", "FORWARD", "1", *rule, "-j", "ACCEPT") + + published = free_port() + # `iptables` first — it is what Docker writes with, and it is on every machine Docker + # is; `nft` for a host with no iptables at all. + for rule in _rewrites(published): + if as_root("iptables", *rule[:2], "-A", *rule[2:]) is None: + _undo_rewrites() + break + _staged.append(rule) + ruleset = ( + f"table ip {CONTAINER_TABLE} {{\n" + + "".join( + f" chain {hook} {{\n" + f" type nat hook {hook} priority dstnat; policy accept;\n" + f" ip daddr {HOST_IP} meta l4proto {{ tcp, udp }} th dport {published} " + f"dnat to {CONTAINER_IP}:8080\n" + f" }}\n" + for hook in ("prerouting", "output") + ) + + "}\n" + ) + if not _staged and subprocess.run( + _root_command("nft", "-f", "-"), input=ruleset, text=True, + capture_output=True).returncode != 0: + _take_container_down() + pytest.skip("cannot stage a container runtime's rewrite here: neither iptables " + "nor nft took it") + + serving = subprocess.Popen( + _root_command("ip", "netns", "exec", SERVICE_NETNS, "python3", "-c", + "# fgextest-container\n" + CONTAINER_SERVICE), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(0.5) + yield published + serving.terminate() + _take_container_down() + + +def _local(port: int, payload: str, udp: bool = False) -> str: + """The same question as `_ask`, from this host.""" + kind = socket.SOCK_DGRAM if udp else socket.SOCK_STREAM + with socket.socket(socket.AF_INET, kind) as s: + s.settimeout(4) + try: + if udp: + s.sendto(payload.encode(), (HOST_IP, port)) + return repr(s.recvfrom(64)[0]) + s.connect((HOST_IP, port)) + s.sendall(payload.encode()) + return repr(s.recv(64)) + except OSError as e: + return type(e).__name__ + + +@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"]) +@pytest.mark.parametrize("transport", ["proxy", "nfqueue"]) +def test_a_service_behind_a_container_runtimes_rewrite_is_protected_from_anywhere( + api, service, elsewhere, container, transport, udp): + """Answered, filtered and transparent, from another host and from this one. + + Two ways this failed, one per layer, and neither was visible from this host alone. + + **Proxy, a client elsewhere:** the engine dials the published address as the client, + the runtime's DNAT sends it on to the container, and the container answers from its + own address. The rule that brings the engine's answers home looked the socket up by + that address, found none, and the answer went on to the real client, which reset it: + every connection waited out the dial and was retried as this host — slowly, and with + the client's address lost. + + **NFQUEUE, a client on this host:** the runtime rewrites this host's own connection in + `nat OUTPUT`, so it leaves towards the container without passing prerouting, where the + queues are, and the container's answer arrives for a local socket without passing + postrouting. Neither half was inspected, and a payload refused from anywhere else + reached the service from here. + """ + clients = { + "another host": (lambda payload: elsewhere(_ask(container, udp, payload)), CLIENT_IP), + "this host": (lambda payload: _local(container, payload, udp), HOST_IP), + } + # Each path has to work with nothing of firegex in it before firegex can be judged on + # it: a host that does not forward to the container's network is not a firegex bug. + for where, (ask, _) in clients.items(): + if ask("hello") != "b'hello'": + pytest.skip(f"the stand-in container is not reachable from {where} even " + f"without firegex: this host does not forward to it") + + service_id = service(f"container-{transport}-{container}", HOST_IP, container, transport, + proto="udp" if udp else "tcp") + add_regex_filter(api, service_id, "BLOCKME") + start_and_settle(api, service_id, wait=1.5) + + # A datagram refused is one that is not answered; a connection refused is closed. + refused = ("TimeoutError",) if udp else ("b''", "ConnectionResetError", "TimeoutError") + for where, (ask, client_ip) in clients.items(): + assert ask("hello") == "b'hello'", f"the service did not answer {where}" + assert ask("WHO") == repr(client_ip.encode()), \ + f"the service did not see the address of a client on {where}" + assert ask("x BLOCKME") in refused, f"a blocked payload from {where} reached the service" + + +def _echo_on(host: str, udp: bool) -> int: + """A service answering with what it is sent, on this host's side of the link.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM if udp else socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((host, 0)) + + def serve(): + if udp: + while True: + data, peer = sock.recvfrom(2048) + sock.sendto(data, peer) + sock.listen(8) + while True: + conn, _ = sock.accept() + with conn: + conn.sendall(conn.recv(2048)) + + threading.Thread(target=serve, daemon=True).start() + return sock.getsockname()[1] + + +def _ask(port: int, udp: bool, payload: str = "hello") -> str: + kind = "SOCK_DGRAM" if udp else "SOCK_STREAM" + exchange = ( + f"s.sendto({payload.encode()!r}, ('{HOST_IP}', {port})); print(s.recvfrom(64)[0])" + if udp else + f"s.connect(('{HOST_IP}', {port})); s.sendall({payload.encode()!r}); print(s.recv(64))" + ) + return ( + "import socket\n" + f"s = socket.socket(socket.AF_INET, socket.{kind}); s.settimeout(4)\n" + f"try:\n {exchange}\n" + "except OSError as e:\n print(type(e).__name__)\n" + ) + + +@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"]) +def test_a_published_address_answers_a_client_on_another_host(api, service, elsewhere, udp): + """Published: the engine dials the service on a port the address does not name. + + The rule that brings the service's answers home knew the service only by the address + it is protected on, which is where the engine dials it while the address *is* the + service. Published elsewhere, the answers came from a port nothing matched and were + routed straight to the client — who was expecting them from the engine and dropped + them. A client on this host never noticed, the reply being local either way. + + The service's own port is not protected here, which is the case that broke: protected + too, its rule happened to cover the answers. And it keeps answering its own clients + directly, which is what a fix matching every reply from that port would have broken. + """ + served = _echo_on(HOST_IP, udp) + extra = free_port(udp=udp) + service_id = service( + f"remote-pub-{served}", HOST_IP, extra, "proxy", proto="udp" if udp else "tcp", + addresses=[{"ip_int": HOST_IP, "port": extra, "target_port": served}], + ) + start_and_settle(api, service_id, wait=1.2) + + assert elsewhere(_ask(extra, udp)) == "b'hello'", \ + "a client on another host was not answered through the published address" + assert elsewhere(_ask(served, udp)) == "b'hello'", \ + "the service stopped answering its own port directly" + + +def _echo_all_on(host: str, udp: bool) -> int: + """`_echo_on`, answering every message of a connection rather than the first.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM if udp else socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((host, 0)) + + def answer(conn): + with conn: + try: + while data := conn.recv(2048): + conn.sendall(data) + except OSError: + pass + + def serve(): + if udp: + while True: + data, peer = sock.recvfrom(2048) + sock.sendto(data, peer) + sock.listen(8) + while True: + conn, _ = sock.accept() + threading.Thread(target=answer, args=(conn,), daemon=True).start() + + threading.Thread(target=serve, daemon=True).start() + return sock.getsockname()[1] + + +def _held_across(elsewhere, udp: bool, port: int, between) -> list[str]: + """Ask once from the other host, run `between`, and ask again on the same socket.""" + kind = "SOCK_DGRAM" if udp else "SOCK_STREAM" + exchange = (f"s.sendto(p, ('{HOST_IP}', {port})); print(s.recvfrom(64)[0])" if udp + else "s.sendall(p); print(s.recv(64))") + held = ( + "import socket, time\n" + f"s = socket.socket(socket.AF_INET, socket.{kind}); s.settimeout(4)\n" + + ("" if udp else f"s.connect(('{HOST_IP}', {port}))\n") + + "def ask(p):\n" + + " try:\n" + + " " + exchange + "\n" + + " except OSError as e:\n" + + " print(type(e).__name__)\n" + + "ask(b'before')\n" + + "time.sleep(4)\n" + + "ask(b'after')\n" + ) + said: list[str] = [] + client = threading.Thread(target=lambda: said.append(elsewhere(held))) + client.start() + time.sleep(1.5) + between() + client.join(timeout=20) + return said[0].splitlines() if said else [] + + +@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"]) +def test_a_connection_through_the_engine_survives_the_service_stopping( + api, service, elsewhere, udp): + """The engine is taken out of the rules and carries what it has; see `retire`. + + From another host because that is where the return path can fail: the answers to the + engine's own dial have to keep coming home after the service's rules are gone. + """ + served = _echo_all_on(HOST_IP, udp) + service_id = service(f"remote-stop-{served}", HOST_IP, served, "proxy", + proto="udp" if udp else "tcp") + start_and_settle(api, service_id, wait=1.2) + + lines = _held_across(elsewhere, udp, served, lambda: api.services_stop(service_id)) + assert lines[:1] == ["b'before'"], f"the connection did not work to begin with: {lines}" + assert lines[1:2] == ["b'after'"], \ + f"stopping the service cut a connection the engine was carrying: {lines}" + + +@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"]) +@pytest.mark.parametrize("transport", ["proxy", "nfqueue"]) +def test_a_connection_already_open_survives_the_service_starting( + api, service, elsewhere, transport, udp): + """Starting protection must not cut the clients a service already has. + + A connection made before the service started is not the datapath's — nothing + redirected it, and nothing will: conntrack translates a connection once, at its first + packet. It goes on straight to the service. What it must not do is stop working, and on + the proxy layer it did for a client on another host: the rule that brings the service's + answers home to the engine matched every answer from the service's port, these + included, and delivered them here instead of sending them to the client. + """ + served = _echo_all_on(HOST_IP, udp) + service_id = service(f"remote-open-{served}", HOST_IP, served, transport, + proto="udp" if udp else "tcp") + lines = _held_across(elsewhere, udp, served, + lambda: start_and_settle(api, service_id, wait=0.5)) + assert lines[:1] == ["b'before'"], f"the connection did not work to begin with: {lines}" + assert lines[1:2] == ["b'after'"], \ + f"a connection already open stopped working when the service started: {lines}" diff --git a/tests/integration/test_resilience.py b/tests/integration/test_resilience.py index cb8af05..af6852f 100644 --- a/tests/integration/test_resilience.py +++ b/tests/integration/test_resilience.py @@ -16,6 +16,7 @@ import pytest from integration import filter_code from integration.conftest import add_python_filter, add_regex_filter, start_and_settle +from helpers.host import as_root, ruleset from helpers.traffic import Channel pytestmark = pytest.mark.instance @@ -185,3 +186,33 @@ def test_the_kernel_keeps_forwarding_when_a_queued_filter_dies(api, protected, assert channel.gets_through(b"traffic after the binary was killed"), \ "the kernel stopped forwarding when the filter died" + + +@pytest.mark.root +def test_rules_removed_from_under_firegex_are_put_back_and_said(api, protected, + inspecting_layer): + """Something else managing the host's nftables must not quietly switch firegex off. + + `nft -f /etc/nftables.conf` begins with `flush ruleset` on Debian, and so does + restarting its unit. Every service went on reading ACTIVE with nothing reaching a + filter, and nothing said so. They are put back within a few seconds now — the + datapath never stopped, so nothing restarts — and the service's log says what + happened, as an error, because traffic went through unfiltered in between. + """ + service_id, server, port = protected(inspecting_layer, name="flushed") + add_regex_filter(api, service_id, "BLOCKME") + start_and_settle(api, service_id) + channel = Channel(server, port, inspecting_layer.ipv6) + assert channel.is_blocked(b"carrying BLOCKME"), "it was not filtering to begin with" + + if as_root("nft", "delete", "table", "inet", "fgex") is None: + pytest.skip("cannot remove the table from here") + deadline = time.time() + 10 + while f"dport {port}" not in (ruleset() or "") and time.time() < deadline: + time.sleep(0.5) + + assert channel.gets_through(b"harmless"), "the service stopped answering" + assert channel.is_blocked(b"carrying BLOCKME"), "the rules were not put back" + said = [e for e in api.services_logs(service_id) + if e["level"] == "error" and "removed from the kernel" in e["text"]] + assert said, "the rules were put back without a word about the gap" diff --git a/tests/integration/test_service_edit.py b/tests/integration/test_service_edit.py index 01cad7f..89aabb0 100644 --- a/tests/integration/test_service_edit.py +++ b/tests/integration/test_service_edit.py @@ -5,9 +5,11 @@ and written all at once. Each test here is a way it used to leave the service so between the old definition and the new one. """ +import time + import pytest -from integration.conftest import start_and_settle +from integration.conftest import RELOAD, start_and_settle from helpers.certs import UNPARSEABLE_CERT, UNPARSEABLE_KEY from helpers.traffic import Channel @@ -49,6 +51,33 @@ def test_an_edit_that_collides_leaves_the_service_as_it_was(api, service, stand_ assert [a["proto"] for a in api.services_addresses(udp)] == ["udp"] +def test_saving_a_service_as_it_was_or_renaming_it_drops_nothing(api, protected, proxy_layer): + """The form sends every setting on every save, so most saves change nothing. + + Each of them was taken for a change of definition anyway: a running service was + stopped and started around it, and every connection it carried was reset — for opening + the dialog and pressing save, or for a new name, which nothing running reads. + """ + service_id, server, _ = protected(proxy_layer, name="edit-same") + start_and_settle(api, service_id) + was = api.services_get(service_id) + + server.connect_client(timeout=3) + try: + server.send_packet(b"before") + assert server.recv_packet() == b"before", "the connection did not work to begin with" + assert api.services_edit(service_id, **{key: was[key] for key in ( + "name", "proto", "transport", "fail_open", "max_connections", + "over_limit_forwards", "first_byte_timeout")}) + assert api.services_edit(service_id, name=f"{was['name']}-renamed") + time.sleep(RELOAD) + server.send_packet(b"after") + assert server.recv_packet() == b"after", "saving the service dropped its connections" + finally: + server.close_client() + assert api.services_get(service_id)["name"] == f"{was['name']}-renamed" + + def test_a_running_service_whose_edit_will_not_start_is_put_back(api, protected, proxy_layer): """Refused by the engine, not by the form: nothing short of starting it can tell. diff --git a/tests/integration/test_service_lifecycle.py b/tests/integration/test_service_lifecycle.py index 8ac37ca..93857cd 100644 --- a/tests/integration/test_service_lifecycle.py +++ b/tests/integration/test_service_lifecycle.py @@ -4,9 +4,11 @@ Parametrised over every layer, because a service is a service whichever one it i the layer is a choice inside it, not a different kind of object. """ +import time + import pytest -from integration.conftest import start_and_settle +from integration.conftest import RELOAD, add_regex_filter, start_and_settle from helpers.traffic import Channel pytestmark = pytest.mark.instance @@ -66,3 +68,80 @@ def test_a_tls_service_occupies_no_port_of_its_own(api, protected, tls_layer): start_and_settle(api, service_id) address = api.services_addresses(service_id)[0] assert "ssl_port" not in address and "clear_port" not in address, str(address) + + +def _held_open(server, payload: bytes) -> bytes | bool: + try: + server.send_packet(payload) + except OSError: + return False + return server.recv_packet() + + +def _log_says(api, service_id: str, words: str, wait: float = 10) -> bool: + deadline = time.monotonic() + wait + while time.monotonic() < deadline: + if any(words in e["text"] for e in api.services_logs(service_id)): + return True + time.sleep(0.3) + return False + + +def test_stopping_a_proxied_service_does_not_cut_its_connections(api, protected, proxy_layer): + """Stopping takes the filters away, not the service. + + The proxy terminates every connection it carries, so killing it — which a stop did — + cut them all: a download, a websocket, a request half answered. It is taken out of the + rules instead, and carries what it already has until it closes. Unfiltered: a stop is + how a filter that is hurting the service is taken out of the way, and it would not be + if it went on judging the connections already open. New ones go straight to the service. + """ + service_id, server, port = protected(proxy_layer, name="drain-stop") + add_regex_filter(api, service_id, "BLOCKME") + start_and_settle(api, service_id) + + server.connect_client(timeout=3) + try: + assert _held_open(server, b"hello") == b"hello", "the connection did not work to begin with" + assert api.services_stop(service_id) + assert _held_open(server, b"still here") == b"still here", \ + "stopping the service cut a connection it was carrying" + assert _held_open(server, b"carrying BLOCKME") == b"carrying BLOCKME", \ + "a stopped service went on filtering the connections it had" + time.sleep(3) + assert _held_open(server, b"a while later") == b"a while later" + assert _log_says(api, service_id, "carry on unfiltered"), \ + "the log did not say connections were still being carried" + finally: + server.close_client() + assert _log_says(api, service_id, "have all closed"), \ + "the stopped engine did not notice its last connection had closed" + channel = Channel(server, port, proxy_layer.ipv6) + assert channel.gets_through(b"carrying BLOCKME"), "a new connection did not reach the service" + + +def test_a_restart_hands_over_without_cutting_connections(api, protected, proxy_layer): + """A change that rebuilds the engine starts a new one beside the old. + + New connections go to the new engine; the ones the old engine carries stay with it, + under the filters they started with, until they close. Killing it — which a restart + did — cut them all for an edit as small as the connection limit. + """ + service_id, server, port = protected(proxy_layer, name="drain-restart") + add_regex_filter(api, service_id, "BLOCKME") + start_and_settle(api, service_id) + + server.connect_client(timeout=3) + try: + assert _held_open(server, b"hello") == b"hello", "the connection did not work to begin with" + assert api.services_edit(service_id, max_connections=100) + time.sleep(RELOAD) + assert _held_open(server, b"still here") == b"still here", \ + "a restart cut a connection the old engine was carrying" + assert _held_open(server, b"carrying BLOCKME") != b"carrying BLOCKME", \ + "a connection carried through a restart lost its filters" + finally: + server.close_client() + channel = Channel(server, port, proxy_layer.ipv6) + assert channel.gets_through(b"harmless"), "the new engine did not take new connections" + assert channel.is_blocked(b"carrying BLOCKME"), "the new engine came up without the filters" diff --git a/tests/integration/test_udp.py b/tests/integration/test_udp.py index bed2492..b4867b0 100644 --- a/tests/integration/test_udp.py +++ b/tests/integration/test_udp.py @@ -238,9 +238,12 @@ def test_a_queued_udp_service_keeps_flows_up_to_its_own_limit(api, service, udp_ assert _from(first, echo, b"MARK") == b"MARK" assert _from(first, echo, b"again") is None, "the flow did not keep its state at all" # Enough of them that some land on the first flow's thread: flows are shared out - # between the queue threads by a hash, and so is the limit. + # between the queue threads by a hash, and so is the limit. Each from a port handed + # out once, rather than one the kernel picks: that can be the first flow's own, now + # that nothing holds it, and a datagram sent from it is the first flow — the one the + # filter was told to refuse — which failed this one run in a few hundred. for _ in range(200): - assert echo.exchange(b"another flow") == b"another flow" + assert _from(free_port(udp=True), echo, b"another flow") == b"another flow" answered = _from(first, echo, b"once more") if kept: diff --git a/tests/unit/test_datapath_acks.py b/tests/unit/test_datapath_acks.py new file mode 100644 index 0000000..58d857a --- /dev/null +++ b/tests/unit/test_datapath_acks.py @@ -0,0 +1,79 @@ +"""What a datapath process's answer to a configuration push leaves running. + +Both NFQUEUE binaries answer `ACK FAIL` with the filters they had still in force, so a +refusal is an error for the operator and nothing more. A push that is not answered at all +is different: whatever the process ends up doing is unknown, and its answer, if it comes, +would be read as the answer to the next command. +""" + +import asyncio + +import pytest + +from modules.services import transports +from modules.services.models import KIND, Filter, Service, TRANSPORT +from modules.services.transports import ChainLink, NfqueueTransport, _QueueStage + + +class _Stdin: + def write(self, data: bytes) -> None: + self.data = data + + async def drain(self) -> None: + pass + + +class _Process: + def __init__(self): + self.returncode = None + self.stdin = _Stdin() + self.killed = False + + def kill(self) -> None: + self.killed = True + self.returncode = -9 + + +def _stage() -> tuple[_QueueStage, _Process, list[str]]: + said: list[str] = [] + srv = Service(service_id="s", name="n", status="active", proto="tcp", + transport=TRANSPORT.NFQUEUE) + owner = NfqueueTransport(srv, on_engine=said.append) + flt = Filter(filter_id="f1", service_id="s", position=0, kind=KIND.REGEX, name="p", + active=True) + stage = _QueueStage(srv, ChainLink(flt), owner) + stage.process = _Process() + return stage, stage.process, said + + +def test_a_refused_push_leaves_the_binary_running(): + """It used to be stopped, and stopped *on purpose*, so its watchdog said nothing: the + service went on reading active with no process behind its queue.""" + stage, process, _ = _stage() + + async def run(): + pushing = asyncio.create_task(stage._push(b"payload\n")) + await asyncio.sleep(0) + stage._ack.set_result((False, "Failed to compile hyperscan db")) + with pytest.raises(Exception, match="rejected the filters"): + await pushing + + asyncio.run(run()) + assert not process.killed + assert stage.process is process and not stage._stopped + + +def test_an_unanswered_push_hands_the_binary_to_its_watchdog(monkeypatch): + """Killed, and **not** marked stopped: its watchdog is what rebuilds the service from + the saved configuration, and what the process had applied cannot be known.""" + monkeypatch.setattr(transports, "LOAD_TIMEOUT", 0.05) + stage, process, said = _stage() + + async def run(): + with pytest.raises(Exception, match="did not acknowledge"): + await stage._push(b"payload\n") + + asyncio.run(run()) + assert process.killed + assert not stage._stopped + assert any("restarting it" in line for line in said), said diff --git a/tests/unit/test_firewall_rules.py b/tests/unit/test_firewall_rules.py new file mode 100644 index 0000000..703ad30 --- /dev/null +++ b/tests/unit/test_firewall_rules.py @@ -0,0 +1,66 @@ +"""What the plain firewall writes into nftables, for the rules an operator lists. + +Nothing here talks to the kernel: the commands are built and read back as data. The two +things pinned are both about traffic a rule was meant to let through and did not. +""" + +from modules.firewall.models import Action, FirewallSettings, Mode, Protocol, Rule, Table +from modules.firewall.nftables import CONTAINER_BRIDGES, FiregexTables + + +def rule(**kw) -> Rule: + fields = dict(proto=Protocol.TCP, src="", dst="", port_src_from=1, port_dst_from=80, + port_src_to=65535, port_dst_to=80, action=Action.ACCEPT, mode=Mode.IN, + table=Table.FILTER) + fields.update(kw) + return Rule(**fields) + + +def families_of(commands: list[dict]) -> list[str]: + return [c["add"]["rule"]["family"] for c in commands] + + +def test_a_rule_naming_no_address_is_installed_for_both_families_wherever_it_is(): + """The family was narrowed once and never widened again, so every rule after the first + one naming an IPv4 address was IPv4 only — an "accept port 80" below it did not + exist for IPv6, and under a drop policy every IPv6 client of that port was refused.""" + commands = FiregexTables().get_rules( + rule(src="10.0.0.0/8", port_dst_from=22, port_dst_to=22), + rule(), + ) + assert families_of(commands) == ["ip", "ip", "ip6"], families_of(commands) + + +def test_a_rule_for_both_protocols_is_not_rewritten_underneath_its_owner(): + both = rule(proto=Protocol.BOTH) + commands = FiregexTables().get_rules(both) + assert both.proto == Protocol.BOTH, "the caller's rule was turned into a TCP one" + protocols = {c["add"]["rule"]["expr"][0]["match"]["left"]["payload"]["protocol"] + for c in commands} + assert protocols == {"tcp", "udp"}, protocols + + +def settings(**kw) -> FirewallSettings: + fields = dict(keep_rules=False, allow_loopback=True, allow_established=True, + allow_icmp=True, multicast_dns=True, allow_upnp=True, drop_invalid=True, + allow_dhcp=True) + fields.update(kw) + return FirewallSettings(**fields) + + +def test_container_traffic_is_left_to_the_container_runtime(): + """Under a drop policy, a chain of firegex's own is evaluated on its own — and while the + rules lived in iptables' `FORWARD`, Docker's accepts beside them let containers talk to + each other and out. Only published ports were given back at first, which cut a web + container off from its database on the same bridge.""" + commands = FiregexTables().dnat_rules(settings()) + matched = [c["add"]["rule"]["expr"][0]["match"] for c in commands] + bridges = {m["right"] for m in matched if m["left"] == {"meta": {"key": "iifname"}}} + assert set(CONTAINER_BRIDGES) <= bridges, bridges + assert {"docker0", "br-*"} <= bridges + assert any(m["left"] == {"ct": {"key": "status"}} for m in matched), \ + "published ports are no longer left to the runtime" + assert set(families_of(commands)) == {"ip", "ip6"} + assert all(c["add"]["rule"]["chain"] == "fgex_forward" for c in commands) + + assert FiregexTables().dnat_rules(settings(allow_dnat=False)) == [] diff --git a/tests/unit/test_http_parsing.py b/tests/unit/test_http_parsing.py index 63011d5..9eb90e2 100644 --- a/tests/unit/test_http_parsing.py +++ b/tests/unit/test_http_parsing.py @@ -83,20 +83,25 @@ def run(*chunks: tuple[bytes, bool]) -> list[dict]: return _feed(*chunks)[0] -def run_with_verdict(*chunks: tuple[bytes, bool]) -> tuple[list[dict], int]: +def run_with_verdict(*chunks: tuple[bytes, bool], settings: str = "") -> tuple[list[dict], int]: """The same, plus what the chain answered — which is the whole story when the parser refuses the traffic and the filter is therefore never called.""" - return _feed(*chunks) + return _feed(*chunks, settings=settings)[:2] -def _feed(*chunks: tuple[bytes, bool]) -> tuple[list[dict], int]: +def _feed(*chunks: tuple[bytes, bool], settings: str = "", + joined_late: bool = False) -> tuple[list[dict], int, str]: glob = {"__firegex_pyfilter_enabled": ["watch_request", "watch_response"]} - exec(FILTER_CODE, glob, glob) + exec(FILTER_CODE + settings, glob, glob) compile(glob) + if joined_late: + # What the datapath sets on a context that began after its connection did. + glob["__firegex_joined_late"] = True for payload, is_input in chunks: glob["__firegex_packet_info"] = packet(payload, is_input) handle_packet(glob) - return glob["seen"], glob.get("__firegex_pyfilter_result", {}).get("action") + result = glob.get("__firegex_pyfilter_result") or {} + return glob["seen"], result.get("action"), result.get("matched_by") def final(seen: list[dict], kind: str) -> dict: @@ -348,21 +353,101 @@ def test_an_identity_encoding_is_left_alone(): # answers changed between 9.2.1 and 9.4.3, and both are decisions about what a filter is # shown rather than details of how it is parsed. +EMPTY_TRANSFER_ENCODING = b"POST / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: \r\n\r\n" + + def test_an_empty_transfer_encoding_never_reaches_a_filter(): - """`Transfer-Encoding:` with nothing after it is refused, and the connection with it. + """`Transfer-Encoding:` with nothing after it is not parsed into a request. This is the shape request smuggling is built out of: a header that one parser reads as "chunked follows" and another as "no framing here" is two readings of where the next request begins. llhttp accepted it until 9.4.3 — which means firegex parsed it, showed a filter an ordinary-looking request, and forwarded whatever the service then - made of it. Now the parse fails, `invalid_encoding_action` decides, and its default - is to reject: the filter is never called, because there is no one message here to - call it with. + made of it. Now the parse fails and `invalid_encoding_action` decides. The filter is + never called, because there is no one message here to call it with. + + **By default the traffic is carried**, and said so: refusing whatever the parser cannot + read broke a service over any quirk of a client it would have accepted, with nobody + having chosen that. Refusing is one setting away. """ - seen, action = run_with_verdict( - (b"POST / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: \r\n\r\n", True)) + seen, action = run_with_verdict((EMPTY_TRANSFER_ENCODING, True)) + assert seen == [] + assert action == 0 # ACCEPT + + +def test_traffic_the_parser_cannot_read_is_refused_when_asked_and_not_blamed_on_a_filter(): + """The stricter answer, chosen in the file — and credited to the parser. + + It used to be credited to the filter function, which had never been shown the + request: the operator was told their own code refused a connection it did not see. + """ + seen, action, matched_by = _feed( + (EMPTY_TRANSFER_ENCODING, True), + settings="\nfrom firegex.pyfilters import ExceptionAction\n" + "FGEX_INVALID_ENCODING_ACTION = ExceptionAction.REJECT\n") assert seen == [] assert action == 2 # REJECT + assert matched_by == "@INVALID_ENCODING" + + +def test_unreadable_traffic_is_reported_once_per_connection(capsys): + """The parser starts afresh on every packet, so every packet fails the same way. + + A traceback each was the whole log, hundreds of times; one line says it and what was + done about it. + """ + run_with_verdict(*[(EMPTY_TRANSFER_ENCODING, True)] * 4) + said = [line for line in capsys.readouterr().out.splitlines() if "[warn] [http]" in line] + assert len(said) == 1, said + assert "carried" in said[0], said + + +STRICT = ("\nfrom firegex.pyfilters import ExceptionAction\n" + "FGEX_INVALID_ENCODING_ACTION = ExceptionAction.REJECT\n") + + +def test_a_connection_met_in_the_middle_of_a_body_is_carried_not_refused(capsys, monkeypatch): + """What a filter taking over an open connection meets first can be half a request. + + The rest of an upload is not a malformed request, and judging it as one refused valid + uploads in flight whenever a filter changed — under the strict setting, which is + exactly the one an operator worried about smuggling chooses. That direction of the + connection is carried, and said once, as information rather than as a problem. + """ + from firegex.pyfilters.models import http + monkeypatch.setattr(http, "_told_met_mid_message", False) + seen, action, _ = _feed( + (b"x" * 200, True), + (b"GET /next HTTP/1.1\r\nHost: a\r\n\r\n", True), + settings=STRICT, joined_late=True) + assert action == 0, "a request met halfway was refused" + assert seen == [], "a direction met halfway was judged after all" + out = capsys.readouterr().out + assert "[warn] [http]" not in out, "a working client was reported as not speaking HTTP" + assert out.count("[info] [http]") == 1, out + + +def test_a_connection_met_at_the_start_of_a_message_is_filtered_as_usual(): + """Met between two requests — the ordinary idle keep-alive — nothing is lost.""" + seen = _feed((b"GET /first HTTP/1.1\r\nHost: a\r\n\r\n", True), + settings=STRICT, joined_late=True)[0] + assert final(seen, "request")["url"] == "/first" + + +def test_once_in_step_a_connection_met_late_answers_to_the_usual_rules(): + """Only the first thing met is given the benefit of the doubt; garbage after a clean + request is garbage, and the strict setting refuses it as it would anywhere.""" + _, action, matched_by = _feed( + (b"GET /first HTTP/1.1\r\nHost: a\r\n\r\n", True), + (EMPTY_TRANSFER_ENCODING, True), + settings=STRICT, joined_late=True) + assert action == 2 and matched_by == "@INVALID_ENCODING" + + +def test_a_connection_seen_from_its_start_gets_no_such_benefit(): + """Unreadable from the first byte is not a connection met halfway.""" + _, action, _ = _feed((b"x" * 200, True), settings=STRICT) + assert action == 2 def test_tabs_around_a_content_length_are_tolerated(): diff --git a/tests/unit/test_nftables_addresses.py b/tests/unit/test_nftables_addresses.py index ad3f8b3..dd61ac0 100644 --- a/tests/unit/test_nftables_addresses.py +++ b/tests/unit/test_nftables_addresses.py @@ -206,7 +206,7 @@ def test_installed_rule_parsing(): # Simulate get() parsing logic table = detached_table() - table.list_rules = lambda tables, chains: [raw_rule] + table.list_rules = lambda tables, chains, family=None: [raw_rule] rules = table.get() assert len(rules) == 1 r = rules[0] @@ -244,7 +244,7 @@ def test_an_interfaces_output_rule_is_read_back_as_the_interfaces(): ], } table = detached_table() - table.list_rules = lambda tables, chains: [raw_rule] + table.list_rules = lambda tables, chains, family=None: [raw_rule] rule = table.get()[0] assert rule.ip_int == "wg0", "the rule was read back under the address it matches" @@ -292,9 +292,12 @@ def test_proxy_rule_generation_interface_vs_ip(monkeypatch): # each, which is what makes a service protected on `lo` filter a local client. chains = [c["add"]["rule"]["chain"] for c in commands] assert chains.count("fgex_nat") == 1 - assert chains.count("fgex_route") == 1 assert chains.count("fgex_nat_out") == 2, "one per address the interface carries" - assert len(commands) == 4 + # Nothing on the way back: the engine's answers come home by the conntrack mark on + # the connections it opened. A per-service rule there caught the answers on + # connections open before the service started, and a remote client lost them. + assert "fgex_route" not in chains + assert len(commands) == 3 # Inbound stays the interface and nothing narrower. Pinning it to the addresses the # interface carries was tried and reverted: a name is what an operator reaches for @@ -328,22 +331,13 @@ def test_proxy_rule_generation_interface_vs_ip(monkeypatch): for e in nat_rule["expr"] ) - # Check expressions in route rule - route_rule = [c["add"]["rule"] for c in commands if c["add"]["rule"]["chain"] == "fgex_route"][0] - assert any( - e.get("match", {}).get("left", {}).get("meta", {}).get("key") == "oifname" - and e["match"]["right"] == "eth0" - for e in route_rule["expr"] - ) - # 2. IP target commands.clear() table._add_proxy(srv, "127.0.0.1/32", 80, "ip", "tcp", 38000) - # Should write to nat_chain, route_chain, AND nat_output_chain - assert len(commands) == 3 + # Both hooks of the redirect, and nothing on the way back. + assert len(commands) == 2 chains_ip = [c["add"]["rule"]["chain"] for c in commands] assert "fgex_nat" in chains_ip - assert "fgex_route" in chains_ip assert "fgex_nat_out" in chains_ip diff --git a/tests/unit/test_proxy_drain.py b/tests/unit/test_proxy_drain.py new file mode 100644 index 0000000..c2236a3 --- /dev/null +++ b/tests/unit/test_proxy_drain.py @@ -0,0 +1,137 @@ +"""A proxy engine taken out of service carries its connections until they close. + +The proxy terminates every connection it carries, so a stop or a restart that killed it +cut them all. `ProxyTransport.retire` lets it go on instead, and ends it as soon as it +reports none left — or at `DRAIN_LIMIT`, or when too many engines of one service are +draining at once. Driven here against a stand-in process, because the interesting part +is the bookkeeping, and the integration suite cannot wait ten minutes. +""" + +import asyncio +import time + +import pytest + +from modules.services import transports +from modules.services.models import TRANSPORT, Service + + +class _Process: + def __init__(self): + self.returncode = None + + def kill(self): + self.returncode = -9 + + async def wait(self): + return self.returncode + + +@pytest.fixture +def engine(monkeypatch): + monkeypatch.setattr(transports, "DRAIN_FRESH_AFTER", 0) + monkeypatch.setattr(transports, "_draining", {}) + said: list[str] = [] + + def _build(live: int): + srv = Service(service_id="drain", name="n", status="active", proto="tcp", + transport=TRANSPORT.PROXY) + t = transports.ProxyTransport(srv, on_engine=said.append) + t.process = _Process() + t.counters["live"] = live + return t + + return _build, said + + +def _report(t, live: int): + """What a `STATS` line arriving now does.""" + t.counters["live"] = live + t._stats_at = time.monotonic() + + +def test_it_goes_as_soon_as_its_last_connection_does(engine): + build, said = engine + gone = [] + + async def run(): + t = build(live=2) + process = t.process + await t.retire(keep_filtering=True, then=lambda: gone.append(True)) + _report(t, 2) + await asyncio.sleep(1.2) + assert process.returncode is None, "killed while it still carried connections" + _report(t, 0) + await asyncio.sleep(1.2) + assert process.returncode is not None, "left running with nothing to carry" + + asyncio.run(run()) + assert gone == [True], "its ports were never released" + assert any("under the filters they started with" in line for line in said), said + assert any("have all closed" in line for line in said), said + + +def test_a_count_from_before_it_was_taken_out_does_not_end_it(engine, monkeypatch): + """`live=0` said before the rules came off says nothing about what arrived since.""" + build, _ = engine + monkeypatch.setattr(transports, "DRAIN_FRESH_AFTER", 60) + + async def run(): + t = build(live=0) + _report(t, 0) + await t.retire(keep_filtering=False) + await asyncio.sleep(1.2) + assert t.process.returncode is None + await t._end_drain(None) + + asyncio.run(run()) + + +def test_connections_that_never_close_are_closed_at_the_limit(engine, monkeypatch): + build, said = engine + monkeypatch.setattr(transports, "DRAIN_LIMIT", 1) + + async def run(): + t = build(live=3) + process = t.process + await t.retire(keep_filtering=True) + _report(t, 3) + await asyncio.sleep(2) + assert process.returncode is not None + + asyncio.run(run()) + assert any(line.startswith("[warn]") and "still open" in line for line in said), said + + +def test_only_so_many_engines_of_a_service_drain_at_once(engine): + build, said = engine + + async def run(): + engines = [build(live=1) for _ in range(transports.MAX_DRAINING + 1)] + processes = [t.process for t in engines] + for t in engines: + await t.retire(keep_filtering=True) + _report(t, 1) + assert processes[0].returncode is not None, "the oldest was kept" + assert all(p.returncode is None for p in processes[1:]) + assert len(transports._draining["drain"]) == transports.MAX_DRAINING + await transports.stop_draining() + assert all(p.returncode is not None for p in processes) + assert not transports._draining + + asyncio.run(run()) + assert any("were closed" in line for line in said), said + + +def test_an_engine_already_dead_is_simply_cleaned_up(engine): + build, _ = engine + gone = [] + + async def run(): + t = build(live=0) + t.process.returncode = 1 + await t.retire(keep_filtering=True, then=lambda: gone.append(True)) + assert not transports._draining + + asyncio.run(run()) + assert gone == [True] diff --git a/tests/unit/test_proxysim.py b/tests/unit/test_proxysim.py new file mode 100644 index 0000000..50e448c --- /dev/null +++ b/tests/unit/test_proxysim.py @@ -0,0 +1,137 @@ +"""`fgex pyfilters`: the way an operator tries a filter before it goes near a service. + +Run as they would run it — the `fgex` script, as a process of its own, in front of a +real service — because every part of that path has broken on its own at some point: the +file watcher that starts the proxy, the child process it runs in (Python 3.14 starts +those with `forkserver`, which imports the script again), and the filtering itself. +Nothing else in the suite runs it, so a release could ship a simulator that no longer +simulates and nobody would find out until they needed it. + +What it has to agree with is production. A filter that refuses here refuses there, clean +traffic comes back, and traffic the HTTP parser cannot read is carried — the library's +default — rather than refused as it used to be. +""" + +import os +import signal +import socket +import subprocess +import sys +import threading +import time + +import pytest + +from helpers.net import free_port + +FGEX = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "fgex-lib", "fgex")) + +RAW = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT +from firegex.pyfilters.models import RawPacket + +@pyfilter +def refuse(packet: RawPacket): + return REJECT if b"BLOCKME" in packet.data else ACCEPT +""" + +HTTP = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT +from firegex.pyfilters.models import HttpRequest + +@pyfilter +def refuse(req: HttpRequest): + return REJECT if "BLOCKME" in (req.url or "") else ACCEPT +""" + + +@pytest.fixture +def echo(): + """A service that answers with whatever it was sent. Returns its port.""" + server = socket.socket() + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", free_port())) + server.listen(16) + + def answer(conn): + try: + while data := conn.recv(4096): + conn.sendall(data) + except OSError: + pass + finally: + conn.close() + + def serve(): + while True: + try: + conn, _ = server.accept() + except OSError: + return + threading.Thread(target=answer, args=(conn,), daemon=True).start() + + threading.Thread(target=serve, daemon=True).start() + yield server.getsockname()[1] + server.close() + + +@pytest.fixture +def simulator(echo, tmp_path): + """Start `fgex pyfilters` on a filter file in front of `echo`. Returns an asker.""" + started = [] + + def _start(code: str): + path = tmp_path / "filter.py" + path.write_text(code) + port = free_port() + proc = subprocess.Popen( + [sys.executable, FGEX, "pyfilters", str(path), "127.0.0.1", str(echo), + "--from-port", str(port)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + started.append(proc) + deadline = time.time() + 20 + while time.time() < deadline: + if proc.poll() is not None: + pytest.fail(f"fgex pyfilters exited: {proc.stdout.read()}") + try: + socket.create_connection(("127.0.0.1", port), timeout=0.2).close() + break + except OSError: + time.sleep(0.1) + else: + pytest.fail("fgex pyfilters never started listening") + + def ask(payload: bytes) -> bytes: + try: + with socket.create_connection(("127.0.0.1", port), timeout=3) as conn: + conn.sendall(payload) + return conn.recv(4096) + except OSError: + return b"" + + return ask + + yield _start + for proc in started: + # Interrupted rather than killed: that is how an operator stops it, and it is the + # path that takes the proxy's own process down with it. + proc.send_signal(signal.SIGINT) + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +def test_a_raw_filter_refuses_what_it_should_and_carries_the_rest(simulator): + ask = simulator(RAW) + assert ask(b"hello") == b"hello" + assert ask(b"carrying BLOCKME") == b"" + + +def test_an_http_filter_reads_requests_and_carries_what_is_not_http(simulator): + ask = simulator(HTTP) + clean = b"GET /fine HTTP/1.1\r\nHost: x\r\n\r\n" + assert ask(clean) == clean + assert ask(b"GET /BLOCKME HTTP/1.1\r\nHost: x\r\n\r\n") == b"" + # Not a refusal: `invalid_encoding_action` is ACCEPT unless the file says otherwise. + garbage = b"\x16\x03\x01 not a request\r\n\r\n" + assert ask(garbage) == garbage diff --git a/tests/unit/test_pyfilter_inference.py b/tests/unit/test_pyfilter_inference.py index 6c3b733..f58dad2 100644 --- a/tests/unit/test_pyfilter_inference.py +++ b/tests/unit/test_pyfilter_inference.py @@ -198,7 +198,7 @@ def test_a_size_that_is_not_a_size_is_ignored_too(): def test_the_other_two_knobs_want_their_own_enum(): assert settings_of("").full_stream_action is FullStreamAction.FLUSH - assert settings_of("").invalid_encoding_action is ExceptionAction.REJECT + assert settings_of("").invalid_encoding_action is ExceptionAction.ACCEPT chosen = settings_of("from firegex.pyfilters import FullStreamAction\n" "FGEX_FULL_STREAM_ACTION = FullStreamAction.REJECT\n") diff --git a/tests/unit/test_rule_round_trip.py b/tests/unit/test_rule_round_trip.py index 5a056d6..8833aaf 100644 --- a/tests/unit/test_rule_round_trip.py +++ b/tests/unit/test_rule_round_trip.py @@ -121,7 +121,10 @@ def test_the_nfqueue_layers_rules_are_all_taken_back(tables): srv = service(TRANSPORT.NFQUEUE) handle.add(srv, queue_nums=[1000, 1001]) - assert len(fake.rules) == 4, "two positions, one pair of rules each" + # Two for traffic arriving, and two for this host's own to the address after a + # container runtime rewrote it — the ones matched through conntrack, which `get()` + # only finds by their comment. + assert len(fake.rules) == 8, "two positions, two pairs of rules each" handle.delete(srv) assert fake.rules == [], str(fake.rules) diff --git a/tests/unit/test_service_lifecycle_lock.py b/tests/unit/test_service_lifecycle_lock.py index 2352a6e..08c0d0d 100644 --- a/tests/unit/test_service_lifecycle_lock.py +++ b/tests/unit/test_service_lifecycle_lock.py @@ -27,6 +27,11 @@ class _Transport: self.seen["stop"] += 1 await asyncio.sleep(0.05) + async def retire(self, keep_filtering, then=None): + await self.stop() + if then: + then() + class _Quiet: """Stands in for the rules, the log and the database: none of them is the question.""" @@ -35,6 +40,9 @@ class _Quiet: pass def delete(self, *args, **kwargs): + return set() + + def release_guards(self, *args, **kwargs): pass def query(self, *args, **kwargs): @@ -73,3 +81,59 @@ def test_two_stops_at_once_take_it_down_once(manager): asyncio.run(run()) assert seen["stop"] == 1, "the second stop tore down a datapath that was already gone" assert not service.active + + +class _Rules(_Quiet): + """The rules, counted: how many times each service was steered, and un-steered.""" + + def __init__(self): + self.added = 0 + self.deleted = 0 + + def add(self, *args, **kwargs): + self.added += 1 + + def delete(self, *args, **kwargs): + self.deleted += 1 + return set() + + +def test_rules_are_not_put_back_behind_a_stop(manager, monkeypatch): + """The table watchdog and a stop, at once. + + A stop takes the rules away before the datapath, and is suspended in between. Put + back in that gap, they would outlive the stop — steering traffic at an engine that is + gone — so putting them back waits for the same lock and finds nothing running. + """ + service, _ = manager + rules = _Rules() + monkeypatch.setattr(firewall, "nft", rules) + + async def run(): + await service.enable() + stopping = asyncio.create_task(service.disable()) + await asyncio.sleep(0.01) # inside `disable`, waiting on the datapath + await service.put_rules_back() + await stopping + + asyncio.run(run()) + assert not service.active + assert rules.added == 1, "the rules were put back behind the stop" + + +def test_rules_put_back_on_a_running_service_say_so(manager, monkeypatch): + service, _ = manager + rules = _Rules() + monkeypatch.setattr(firewall, "nft", rules) + said = [] + service.log = type("Log", (), {"add": lambda self, level, text: said.append((level, text))})() + + async def run(): + await service.enable() + await service.put_rules_back() + await service.put_rules_back() + + asyncio.run(run()) + assert rules.added == 3 + errors = [text for level, text in said if level == firewall.LEVEL.ERROR] + assert len(errors) == 1, f"said once per quiet period, not once per time: {errors}" diff --git a/tests/unit/test_service_validation.py b/tests/unit/test_service_validation.py index c3bd9d2..8ee7908 100644 --- a/tests/unit/test_service_validation.py +++ b/tests/unit/test_service_validation.py @@ -308,3 +308,4 @@ def test_the_index_name_is_still_understood_if_it_is_ever_reported(): assert "proxy endpoint" in _address_taken(sqlite3.IntegrityError( "UNIQUE constraint failed: index 'unique_hijack_target'")) +