ad/firegex
1
0
Fork 0
mirror of https://github.com/Av0idD4rk/firegex.git synced 2026-10-23 16:31:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

general fixes and old/new connection resilience for proxy filters

Browse Source
This commit is contained in:
Domingo Dirutigliano 2026-09-26 09:52:20 +02:00
parent 70373eb4f6
commit 8ce9b21400
No known key found for this signature in database
69 changed files with 3939 additions and 493 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
.github/workflows/release.yml vendored
View File

@ -35,8 +35,31 @@ jobs:
- name: Run tests
run: cd tests && sudo ./run_tests.sh
# The proxy engine's own suite: HTTP/2, HTTP/3 and QUIC termination, the Python worker's
# framing and deadlines, the UDP relay — much of it reachable from no integration test,
# and none of it was run before a release. In the image stage built for it, which is
# the compiler plus the filter library the worker tests start. `backend/` is mounted
# rather than baked in because the worker tests launch `../modules/services/pyworker.py`.
engine_tests:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, ubuntu-24.04-arm]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Build the engine's test image
run: docker build --target enginetest -t firegex-enginetest .
- name: Run the engine's tests
run: >-
docker run --rm -v "$PWD/backend:/execute" -w /execute/proxysrc
firegex-enginetest cargo test
docker_build:
needs: run_tests
needs: [run_tests, engine_tests]
runs-on: ${{ matrix.os }}
strategy:
matrix:
@ -232,7 +255,7 @@ jobs:
# installed is not, which is why the parser ships as source rather than as a `ctypes`
# binding against the distribution's `libllhttp`.
build_firegex_wheels:
needs: run_tests
needs: [run_tests, engine_tests]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@ -329,7 +352,7 @@ jobs:
password: ${{ secrets.PYPI_API_TOKEN }}
pypi_publish_fgex_alias:
needs: run_tests
needs: [run_tests, engine_tests]
runs-on: ubuntu-latest
permissions:
contents: read

38
AGENTS.md
View File

File diff suppressed because one or more lines are too long

9
Dockerfile
View File

@ -45,6 +45,15 @@ RUN g++ binsrc/pyfilter.cpp -o cpproxy -std=c++23 -O3 -Wall -lnetfilter_queue -l
COPY ./backend/proxysrc /execute/proxysrc
RUN cd /execute/proxysrc && cargo build --release
# Where the engine's own suite runs (`cargo test` in proxysrc). It starts real Python
# workers through the filter library, so it needs the compiler stage *and* that library
# with what it imports — without them the pyworker tests fail with a ModuleNotFoundError
# buried in captured output. Nothing below copies from it, and it sits before `final` so a
# build with no target (what compose does) is not handed this as the image.
FROM --platform=$TARGETARCH compiler AS enginetest
COPY ./fgex-lib /fgex-lib
RUN python3.14 -m pip install --no-cache-dir --break-system-packages /fgex-lib
#Building main conteiner
FROM --platform=$TARGETARCH base AS final

40
backend/binsrc/pyproxy/pyproxy.cpp
View File

@ -30,7 +30,22 @@ namespace PyProxy {
class PyProxyQueue: public NfQueue::ThreadNfQueue<PyProxyQueue> {
private:
u_int16_t latest_config_ver = 0;
// The configuration this thread's contexts were built from.
uint32_t latest_config_ver = 0;
// A new configuration reaches the connections already open, the way it does on the
// other two engines: `cppregex` resets its matchers on a new version and the proxy
// layer hands the next chunk to the new chain. This was declared and never read, so
// here a stream kept the code it had started with for as long as it stayed open — a
// function switched off, or a check added against the attack in progress, went on not
// reaching the one connection it was meant for. Its state starts again, as it does
// there.
void follow_config(){
const uint32_t current = config.load()->version;
if (current == latest_config_ver) return;
sctx.clean_filters();
latest_config_ver = current;
}
public:
stream_ctx sctx;
StreamFollower follower;
@ -71,6 +86,14 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue<PyProxyQueue> {
}
handle_packet_code = unmarshal_code(py_handle_packet_code);
// Streams whose start this process did not see are followed from the packet it
// does see. They are otherwise not followed at all, and a packet of one was simply
// accepted: every connection open before the service started, or before its chain
// was rebuilt — which adding a filter does — and every one quiet for longer than
// the follower keeps a stream, carried whatever it said afterwards. A persistent
// connection was a way past any filter added after it opened. Recovery mode is
// switched on for them in `on_new_stream`.
follower.follow_partial_streams(true);
// Setting callbacks for the stream follower
follower.new_stream_callback(bind(on_new_stream, placeholders::_1, this));
follower.stream_termination_callback(bind(on_stream_close, placeholders::_1, this));
@ -168,6 +191,13 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue<PyProxyQueue> {
stream.ignore_server_data();
return pkt->accept();
}
// This context starts after its connection did: the process met the
// stream halfway, or new code replaced the context it had. What arrives
// first can be the middle of a message, and the library must not judge
// a message by its second half — see `_met_mid_message` there.
if (stream.is_partial_stream() || sctx.taken_over.erase(pkt->sid) > 0){
stream_match->set_item_to_glob("__firegex_joined_late", PyBool_FromLong(1));
}
sctx.streams_ctx.insert_or_assign(pkt->sid, stream_match);
}
}else{
@ -204,7 +234,12 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue<PyProxyQueue> {
}
static void on_data_recv(Stream& stream, PyProxyQueue* pyq, const string& data, bool is_client) {
// The direction is the packet's own, from the mark the rules put on it, rather than
// which side of the stream the follower calls the client: on a stream followed from
// the middle that is whoever it happened to see first, and the service answering
// before the client speaks again made the filter read the answer as the request.
static void on_data_recv(Stream& stream, PyProxyQueue* pyq, const string& data, bool) {
const bool is_client = pyq->pkt->is_input;
pyq->pkt->fix_data_payload();
pyq->filter_action(pyq->pkt, stream, data, is_client); //Only here the rebuilt_tcp_data is set
}
@ -255,6 +290,7 @@ class PyProxyQueue: public NfQueue::ThreadNfQueue<PyProxyQueue> {
void handle_next_packet(NfQueue::PktRequest<PyProxyQueue>* _pkt) override{
pkt = _pkt; // Setting packet context
follow_config();
if (pkt->l4_proto == NfQueue::L4Proto::UDP){
// Straight to the filter: the stream follower is TCP's, and so is every

7
backend/binsrc/pyproxy/settings.cpp
View File

@ -27,8 +27,14 @@ PyObject* unmarshal_code(string encoded_code){
}
class PyCodeConfig{
private:
// Only the updater thread builds configurations, so this needs no lock.
static inline uint32_t glob_seq = 0;
public:
string encoded_code;
// Which configuration this is, so a queue thread can tell that the contexts it
// holds were built from code no longer in force. 0 is the empty one it starts on.
uint32_t version = 0;
PyCodeConfig(const string& pycode){
PyObject* compiled_code = Py_CompileStringExFlags(pycode.c_str(), "<pyfilter>", Py_file_input, NULL, 2);
@ -61,6 +67,7 @@ class PyCodeConfig{
}
encoded_code = string(PyBytes_AsString(code_dump), PyBytes_Size(code_dump));
Py_DECREF(code_dump);
version = ++glob_seq;
}
PyObject* compiled_code(){

52
backend/binsrc/pyproxy/stream_ctx.cpp
View File

@ -6,6 +6,7 @@
#include <tins/tcp_ip/stream_identifier.h>
#include <map>
#include <list>
#include <set>
#include <cstdlib>
#include <atomic>
#include <chrono>
@ -85,6 +86,11 @@ inline bool traceback_is_due() {
previous, now, std::memory_order_relaxed);
}
// How many dropped filter contexts may wait for a full collection, and how many have on
// this queue thread — each thread runs an interpreter of its own, so each counts its own.
constexpr unsigned COLLECT_EVERY = 64;
static thread_local unsigned dropped_since_collect = 0;
struct pyfilter_ctx {
PyObject * glob = nullptr;
@ -108,8 +114,19 @@ struct pyfilter_ctx {
~pyfilter_ctx(){
Py_DECREF(glob);
Py_DECREF(py_handle_packet);
// The module's functions hold these globals and the globals hold the functions,
// so a context is a cycle only the collector frees. Left to the interpreter's
// own schedule, one that lived long enough to be promoted waits for a full pass,
// which is triggered by object counts rather than by bytes — measured, a filter
// building a table at module level held ~200 MB more across a stream of short
// connections. A full pass per context held memory flat and cost ~1 ms each, a
// ceiling of about two hundred connections a second per queue thread. One pass
// every `COLLECT_EVERY` contexts keeps both.
if (++dropped_since_collect >= COLLECT_EVERY){
dropped_since_collect = 0;
PyGC_Collect();
}
}
inline void set_item_to_glob(const char* key, PyObject* value){
set_item_to_dict(glob, key, value);
@ -160,8 +177,11 @@ struct pyfilter_ctx {
// Set packet info to the global context
set_item_to_glob("__firegex_packet_info", packet_info);
// No collection here. The interpreter's own collector is on (`before_loop`
// makes sure), and a full pass per packet cost about a millisecond with the
// library loaded — a ceiling of a thousand packets a second per queue thread,
// spent finding nothing.
PyObject * result = PyEval_EvalCode(py_handle_packet, glob, glob);
PyGC_Collect();
del_item_from_glob("__firegex_packet_info");
if (PyErr_Occurred()){
@ -315,7 +335,13 @@ struct stream_ctx {
list<pair<stream_id, int64_t>> udp_recent;
map<stream_id, list<pair<stream_id, int64_t>>::iterator> udp_where;
// The TCP streams whose filter context was thrown away while they were open — new
// code arrived — so the context built for their next packet knows it is taking over
// halfway, possibly in the middle of a message.
set<stream_id> taken_over;
void clean_stream_by_id(stream_id sid){
taken_over.erase(sid);
auto stream_search = streams_ctx.find(sid);
if (stream_search != streams_ctx.end()){
auto stream_match = stream_search->second;
@ -359,6 +385,22 @@ struct stream_ctx {
}
}
// Every filter context, TCP and UDP alike, for when the code they were built from is
// no longer the code in force. The sequence bookkeeping stays: it belongs to the
// connection, not to the filter, and a stream whose payload was already cut would
// have its acknowledgements go wrong without it.
void clean_filters(){
for (auto ele: streams_ctx){
if (udp_where.find(ele.first) == udp_where.end()){
taken_over.insert(ele.first);
}
delete ele.second;
}
streams_ctx.clear();
udp_recent.clear();
udp_where.clear();
}
void clean_tcp_ack_by_id(stream_id sid){
auto tcp_ack_search = tcp_ack_ctx.find(sid);
if (tcp_ack_search != tcp_ack_ctx.end()){
@ -369,16 +411,12 @@ struct stream_ctx {
}
void clean(){
for (auto ele: streams_ctx){
delete ele.second;
}
clean_filters();
taken_over.clear();
for (auto ele: tcp_ack_ctx){
delete ele.second;
}
tcp_ack_ctx.clear();
streams_ctx.clear();
udp_recent.clear();
udp_where.clear();
}
};

8
backend/binsrc/regex/regexfilter.cpp
View File

@ -181,6 +181,14 @@ public:
}
void before_loop() override{
// Streams whose start this process did not see are followed from the packet it
// does see. They are otherwise not followed at all, and a packet of one was simply
// accepted: every connection open before the service started, or before its chain
// was rebuilt — which adding a filter does — and every one quiet for longer than
// the follower keeps a stream, carried whatever it said afterwards. A persistent
// connection was a way past any filter added after it opened. Recovery mode is
// switched on for them in `on_new_stream`.
follower.follow_partial_streams(true);
follower.new_stream_callback(bind(on_new_stream, placeholders::_1, this));
follower.stream_termination_callback(bind(on_stream_close, placeholders::_1, this));
}

27
backend/modules/firewall/firewall.py
View File

@ -1,4 +1,5 @@
import asyncio
import traceback
from modules.firewall.nftables import FiregexTables
from modules.firewall.models import Rule, FirewallSettings
from utils.sqlite import SQLite
@ -6,18 +7,44 @@ from modules.firewall.models import Action
nft = FiregexTables()
#: How often the kernel is asked whether the firewall is still there. The services
#: module asks the same of its own table, for the same reason: `nft flush ruleset` —
#: the first line of a Debian `/etc/nftables.conf` — takes this with it, and a
#: default-deny firewall silently became no firewall at all.
TABLE_WATCH = 3.0
class FirewallManager:
def __init__(self, db:SQLite):
self.db = db
self.lock = asyncio.Lock()
self._watch: asyncio.Task | None = None
async def close(self):
self.stop_watching()
async with self.lock:
nft.reset()
async def init(self):
nft.init()
await self.reload()
if self._watch is None or self._watch.done():
self._watch = asyncio.create_task(self._watch_table())
async def _watch_table(self):
while True:
await asyncio.sleep(TABLE_WATCH)
try:
if self.enabled and not nft.intact():
print("[error] [firewall] the firewall's nftables tables were removed "
"by something else; putting them back", flush=True)
await self.reload()
except Exception:
traceback.print_exc()
def stop_watching(self) -> None:
if self._watch is not None:
self._watch.cancel()
self._watch = None
async def reload(self):
async with self.lock:

58
backend/modules/firewall/nftables.py
View File

@ -27,16 +27,23 @@ practice. Everything installed is visible under one prefix and removed by deleti
tables per family.
One thing did change, and `dnat_rules` is what puts it back: sharing iptables' own
`FORWARD` chain, traffic Docker or podman accepted for a published port never met
firegex's policy, while a chain of our own is evaluated on its own — so a drop policy
dropped every connection to a published container port. `allow_dnat` (on by default)
leaves destination-NATed traffic no rule of ours matched to the rules that published it.
`FORWARD` chain, traffic Docker or podman accepted never met firegex's policy, while a
chain of our own is evaluated on its own — so a drop policy dropped every connection to a
published container port, every connection between two containers and every one a
container made to the outside. `allow_dnat` (on by default) leaves that traffic, when no
rule of ours matched it, to the container runtime's rules as it always was.
"""
from modules.firewall.models import FirewallSettings, Action, Rule, Protocol, Mode, Table
from utils import nftables_int_to_json, ip_family, NFTableManager, is_ip_parse
import copy
#: The bridges a container runtime forwards through: Docker's default one and its user
#: networks (`br-<id>`), and podman's. Traffic arriving from one of them is a container
#: talking — to another container, or out — and iptables' `FORWARD`, where Docker and
#: podman accept it, is where firegex's rules used to sit too.
CONTAINER_BRIDGES = ("docker0", "br-*", "docker_gwbridge", "podman*", "cni-podman*")
#: Where a rule's `table` — which is the operator's word, stored in the database and
#: shown in the interface — actually lands. The two are deliberately not the same string
#: any more: "filter" and "mangle" describe what a rule *does*, and used to double as the
@ -257,6 +264,17 @@ class FiregexTables(NFTableManager):
)
])
def intact(self) -> bool:
"""Whether the firewall firegex installed is still in the kernel.
The input chain of the IPv4 filter table always holds its jump, so it is the one
place a deleted table and an emptied one both show.
"""
code, listed, _ = self.raw_cmd({"list": {"chain": {
"family": "ip", "table": self.filter_table, "name": "fgex_input",
}}})
return code == 0 and any("rule" in item for item in listed["nftables"])
def set(self, srvs:list[Rule], policy:str=Action.ACCEPT, opt:FirewallSettings = None):
srvs = list(srvs)
self.reset()
@ -298,15 +316,23 @@ class FiregexTables(NFTableManager):
"""
if opt is None or not opt.allow_dnat:
return []
matches = [
{"match": {"op": "in", "left": {"ct": {"key": "status"}}, "right": "dnat"}},
# What the containers send, to each other and out. Leaving only the published
# ports to the runtime was the first version, and with the policy at drop it
# cut a web container off from its database on the same bridge, and every
# container off from the network — neither of which met firegex's policy
# while its rules lived in iptables' `FORWARD` beside Docker's own accepts.
*({"match": {"op": "==", "left": {"meta": {"key": "iifname"}}, "right": bridge}}
for bridge in CONTAINER_BRIDGES),
]
return [
{"add": {"rule": {
"family": family, "table": self.filter_table, "chain": "fgex_forward",
"expr": [
{"match": {"op": "in", "left": {"ct": {"key": "status"}}, "right": "dnat"}},
{"accept": None},
],
"expr": [match, {"accept": None}],
}}}
for family in ("ip", "ip6")
for match in matches
]
def get_rules(self,*srvs:Rule):
@ -314,15 +340,23 @@ class FiregexTables(NFTableManager):
final_srvs:list[Rule] = []
for ele in srvs:
if ele.proto == Protocol.BOTH:
# Copies both ways: the rule handed in is the caller's, and rewriting its
# protocol in place turned a stored "both" into "tcp" for whoever read it
# next.
udp_rule = copy.deepcopy(ele)
udp_rule.proto = Protocol.UDP.value
ele.proto = Protocol.TCP.value
final_srvs.append(udp_rule)
tcp_rule = copy.deepcopy(ele)
tcp_rule.proto = Protocol.TCP.value
final_srvs.extend((udp_rule, tcp_rule))
continue
final_srvs.append(ele)
families = ["ip", "ip6"]
for srv in final_srvs:
# Per rule. It was set once, before the loop, and narrowed by the first rule
# naming an address — so every rule after that one was installed for that
# family only: an "accept port 80" written below an IPv4 rule did not exist
# for IPv6, and with the policy at drop every IPv6 client was refused.
families = ["ip", "ip6"]
ip_filters = []
if srv.src != "":

217
backend/modules/services/firewall.py
View File

@ -59,6 +59,24 @@ OVER_LIMIT_QUIET = 30
#: was in the log pushed out behind it.
RAISED_QUIET = 30
#: How long a block's counters may wait in memory before being written. They used to be
#: three writes per refused connection, each its own commit — about 17 ms a block, so the
#: backend fell behind any flood of them, the engine's pipe filled, and the flood slowed
#: every other client of the service. Once a second, in one transaction, whatever the
#: rate. Anything reading the counters through the API writes what is waiting first.
BLOCK_FLUSH = 1.0
#: How often the kernel is asked whether firegex's rules are still there.
#:
#: Something else can remove them: `nft -f /etc/nftables.conf` begins with `flush
#: ruleset` on Debian, and so does restarting the `nftables` unit. Every service then
#: went on reading `ACTIVE` while nothing reached a filter, and nothing said so. Every few
#: seconds is a short window and a cheap question — one small chain.
TABLE_WATCH = 3.0
#: And how long between two log lines about it, per service. Something removing them on a
#: timer would otherwise write one every `TABLE_WATCH`.
TABLE_TOLD_QUIET = 60
def code_path(filter_id: str) -> str:
return os.path.join(CODE_DIR, f"{filter_id}.py")
@ -113,6 +131,16 @@ class ServiceManager:
#: Carried between lines so a multi-line traceback keeps its severity.
self._in_traceback = False
self._engine_level = LEVEL.INFO
#: Blocks counted and not yet written, by the token the datapath reported.
self._unwritten_blocks: dict[str, int] = {}
#: Rules put back since the log last said so, and when it did.
self._rules_lost = 0
self._rules_lost_told = 0.0
self._block_flush: asyncio.TimerHandle | None = None
#: What a reported token is called and which filter owns it, looked up once per
#: token rather than once per block, and forgotten whenever the chain changes.
self._named: dict[tuple[str, str], str] = {}
self._owners: dict[str, str] = {}
# --- reading the configuration -------------------------------------------
@ -165,23 +193,13 @@ class ServiceManager:
# token both network layers report. Split once, here, so there is a single place
# that knows the format.
rule_id, _, function = reported.partition("/")
if function:
self.db.query(
"UPDATE pyfilters SET blocked = blocked + 1 WHERE filter_id = ? AND name = ?;",
rule_id,
function,
)
# What is left is a regex rule or a whole filter, depending on which refused.
# Only one of these matches, and a blind update is cheaper than asking first.
self.db.query(
"UPDATE regexes SET blocked = blocked + 1 WHERE regex_id = ?;", rule_id
)
self.db.query(
"UPDATE filters SET blocked = blocked + 1 WHERE filter_id = ? OR filter_id = "
"(SELECT filter_id FROM regexes WHERE regex_id = ?);",
rule_id,
rule_id,
)
self._unwritten_blocks[reported] = self._unwritten_blocks.get(reported, 0) + 1
if self._block_flush is None:
try:
self._block_flush = asyncio.get_running_loop().call_later(
BLOCK_FLUSH, self.flush_blocks)
except RuntimeError:
self.flush_blocks()
# The chart is attributed to the filter, not to the pattern or the function: it
# is about which link in the chain is carrying the traffic, and a chain of twenty
# patterns would be a chart nobody can read. The finer totals are listed
@ -192,20 +210,62 @@ class ServiceManager:
stats.flush(self.db)
self.log.add(LEVEL.BLOCK, f"connection refused by {self._describe(rule_id, function)}")
def flush_blocks(self) -> None:
"""Write the counters of every block reported since the last time."""
if self._block_flush is not None:
self._block_flush.cancel()
self._block_flush = None
pending, self._unwritten_blocks = self._unwritten_blocks, {}
if not pending:
return
queries = []
for reported, count in pending.items():
rule_id, _, function = reported.partition("/")
if function:
queries.append((
"UPDATE pyfilters SET blocked = blocked + ? "
"WHERE filter_id = ? AND name = ?;",
count, rule_id, function,
))
# What is left is a regex rule or a whole filter, depending on which refused.
# Only one of these matches, and a blind update is cheaper than asking first.
queries.append((
"UPDATE regexes SET blocked = blocked + ? WHERE regex_id = ?;",
count, rule_id,
))
queries.append((
"UPDATE filters SET blocked = blocked + ? WHERE filter_id = ? OR "
"filter_id = (SELECT filter_id FROM regexes WHERE regex_id = ?);",
count, rule_id, rule_id,
))
try:
self.db.queries(queries)
except Exception:
traceback.print_exc()
def _owning_filter(self, rule_id: str) -> str:
"""Which filter a block belongs to. The id is a filter's or a pattern's."""
owner = self._owners.get(rule_id)
if owner is None:
found = self.db.query(
"SELECT filter_id FROM regexes WHERE regex_id = ?;", rule_id
)
return found[0]["filter_id"] if found else rule_id
owner = self._owners[rule_id] = found[0]["filter_id"] if found else rule_id
return owner
def _describe(self, rule_id: str, function: str = "") -> str:
"""Name what refused a connection, rather than echoing an opaque id.
One query per block, on a path that only runs when something was already
refused — a block is rare compared to the traffic that is not blocked, and an
operator reading `refused by 4f2a91c8` learns nothing.
Looked up once per rule and kept until the chain changes: a flood of refusals is
one rule refusing thousands of times, and an operator reading `refused by
4f2a91c8` learns nothing.
"""
named = self._named.get((rule_id, function))
if named is None:
named = self._named[(rule_id, function)] = self._look_up_name(rule_id, function)
return named
def _look_up_name(self, rule_id: str, function: str) -> str:
if function:
# A file holds several functions, so naming the file alone would leave the
# operator to guess which of them refused the connection.
@ -433,25 +493,72 @@ class ServiceManager:
f"{len([link for link in self.chain() if link.filter.active])} filter(s) active",
)
async def disable(self, persist: bool = True):
async def disable(self, persist: bool = True, handing_over: bool = False,
drain: bool = True):
"""Stop filtering this service.
The connections the datapath is carrying are not cut (`drain`): the rules come
off first, so nothing new reaches it, and the transport carries what it already
has until it closes — see `ProxyTransport.retire`. `handing_over` is a restart,
whose connections carry on under the filters they started with; a stop carries
them unfiltered. Only shutting firegex down cuts them, since the rules that bring
their answers home go with it.
"""
# Under the lock for the same reason: a stop arriving while a start was still in
# progress saw a service that was not active yet, returned at once, and the start
# then finished — so the stop the operator asked for was simply lost.
async with self.lock:
self.flush_blocks()
if not self.active:
return
nft.delete(self.srv)
guarded = nft.delete(self.srv, keep_guards=drain)
self._steer = {}
if self.transport:
if drain:
await self.transport.retire(
keep_filtering=handing_over,
then=lambda: nft.release_guards(guarded),
)
else:
await self.transport.stop()
self.transport = None
else:
nft.release_guards(guarded)
self._set_status(False, persist=persist)
self.log.add(LEVEL.INFO, "stopped")
async def restart(self):
await self.disable()
await self.disable(handing_over=True)
await self.enable()
async def put_rules_back(self) -> None:
"""Steer this service at its datapath again, after something else removed the rules.
The datapath never went anywhere — the engine or the queue binaries are still
running and still listening where `_steer` says — so this is the second half of
`enable()` alone. Under the lock, like `disable()`, so a stop in progress cannot
have its rules put back behind it.
"""
async with self.lock:
if not self.active or self.transport is None:
return
nft.delete(self.srv)
nft.add(self.srv, **self._steer)
self._rules_lost += 1
now = time.time()
if now - self._rules_lost_told < TABLE_TOLD_QUIET:
return
times = f" ({self._rules_lost} times since the last report)" \
if self._rules_lost > 1 else ""
self._rules_lost, self._rules_lost_told = 0, now
self.log.add(
LEVEL.ERROR,
"firegex's rules were removed from the kernel by something else — an "
"`nft flush ruleset`, or a firewall being reloaded — and traffic reached "
f"this service unfiltered until they were put back{times}. Make whatever "
"manages this host's nftables leave the `fgex` tables alone.",
)
async def update_chain(self):
"""Push the current chain to a running datapath.
@ -466,6 +573,11 @@ class ServiceManager:
silently enforcing the old order.
"""
restart = False
# What was refused before this edit is written under the names it had, and what
# is refused after it is looked up again.
self.flush_blocks()
self._named.clear()
self._owners.clear()
async with self.lock:
if not self.active or not self.transport:
return
@ -633,7 +745,7 @@ class ServiceManager:
"""The service's own definition changed, which the datapath cannot absorb."""
was_active = self.active
if was_active:
await self.disable()
await self.disable(handing_over=True)
self.srv = srv
if was_active:
await self.enable()
@ -660,6 +772,7 @@ class FirewallManager:
self.db = db
self.services: dict[str, ServiceManager] = {}
self.lock = asyncio.Lock()
self._watch: asyncio.Task | None = None
async def init(self):
nft.init()
@ -670,6 +783,38 @@ class FirewallManager:
# takes the tool with it.
mirror.ensure()
await self.reload()
if self._watch is None or self._watch.done():
self._watch = asyncio.create_task(self._watch_table())
async def _watch_table(self):
"""Put the rules back whenever something outside firegex takes them away."""
while True:
await asyncio.sleep(TABLE_WATCH)
try:
if not nft.intact():
await self._put_rules_back()
except Exception:
traceback.print_exc()
async def _put_rules_back(self):
async with self.lock:
# Asked again under the lock, which may have been held by something that
# rebuilt the table while this waited for it.
if nft.intact():
return
print("[error] [backend] firegex's nftables table was removed by something "
"else; putting it back", flush=True)
nft.init()
for manager in list(self.services.values()):
try:
await manager.put_rules_back()
except Exception as e:
manager.log.add(LEVEL.ERROR, f"could not put its rules back: {e}")
def stop_watching(self) -> None:
if self._watch is not None:
self._watch.cancel()
self._watch = None
async def reload(self):
async with self.lock:
@ -690,10 +835,10 @@ class FirewallManager:
# of it working.
traceback.print_exc()
async def remove(self, srv_id: str, persist: bool = True):
async def remove(self, srv_id: str, persist: bool = True, drain: bool = True):
async with self.lock:
if srv_id in self.services:
await self.services[srv_id].disable(persist=persist)
await self.services[srv_id].disable(persist=persist, drain=drain)
del self.services[srv_id]
if persist:
# A deleted service's log has nothing left to be about.
@ -706,16 +851,32 @@ class FirewallManager:
releasing here would make the device disappear and come back underneath whatever
was capturing from it — the exact failure it was made persistent to avoid.
"""
# First, or a table deleted on the way down is one it would put back.
self.stop_watching()
# Cut, not drained: what brings a drained connection's answers home is this
# table's, and it is about to go — as is the process that would end the drain.
for key in list(self.services.keys()):
try:
await self.remove(key, persist=False)
await self.remove(key, persist=False, drain=False)
except Exception:
self.services.pop(key, None)
await transports.stop_draining()
stats.flush(self.db, force=True)
def release_capture(self) -> None:
"""Take the capture interface away, at process shutdown and nowhere else."""
mirror.release()
def flush_blocks(self) -> None:
"""Write every service's waiting block counters, before anything reads them.
The history behind the chart too: it waits up to `stats.FLUSH_INTERVAL` for the
next block, and at shutdown there is no next block.
"""
for manager in self.services.values():
manager.flush_blocks()
stats.flush(self.db, force=True)
def get(self, srv_id: str) -> ServiceManager:
if srv_id not in self.services:
raise ServiceNotFoundException()

18
backend/modules/services/logs.py
View File

@ -141,7 +141,10 @@ class ServiceLog:
final count — so the chart sat at whatever it happened to reach two seconds in.
The last block of a burst is precisely the one worth hearing about.
"""
if not any(entry["level"] == LEVEL.BLOCK for entry in entries):
# A warning or an error changes the service list too: it carries the latest one
# (`last_problem`).
if not any(entry["level"] in (LEVEL.BLOCK, LEVEL.WARN, LEVEL.ERROR)
for entry in entries):
return
waited = time.monotonic() - self._counters_told
if waited >= COUNTER_INTERVAL:
@ -170,6 +173,19 @@ class ServiceLog:
def entries(self) -> list[dict]:
return list(self.history)
def last_problem(self) -> dict | None:
"""The newest warning or error still in the tail, for the service list to show.
The live log is on the service's own page, and a warning nobody has that page open
for is a warning nobody reads: a datapath that died and came back, a filter that
raised, a fallback that lost the client's address. The list carries the latest one
beside the service, and clearing the log is how it is acknowledged.
"""
for entry in reversed(self.history):
if entry["level"] in (LEVEL.WARN, LEVEL.ERROR):
return entry
return None
def clear(self) -> None:
self.history.clear()
self._pending.clear()

53
backend/modules/services/models.py
View File

@ -12,7 +12,6 @@ that can host it, and the service does not have to be recreated to change either
"""
import base64
import os
import uuid
from utils import is_ip_parse
@ -319,58 +318,6 @@ class Address:
return f"<Address {self.ip_int}:{self.port}/{self.proto}>"
def quic_alpn() -> tuple[str, ...]:
"""What this instance offers a QUIC service's clients, in order.
Instance-wide (`FGEX_PROXY_QUIC_ALPN`) rather than per service, because nothing has
yet wanted a column for it: `h3` is what a QUIC service speaks nine times in ten, and
an instance in front of something else sets it in its own environment the way the
filter deadline beside it is set. Read here as well as where the engine is launched
so that the one question it decides — whether an exchange can be rendered at all —
has a single answer.
"""
raw = os.getenv("FGEX_PROXY_QUIC_ALPN", "h3")
listed = tuple(p.strip() for p in raw.split(",") if p.strip())
return listed or ("h3",)
def upstream_refusal(proto: str, upstream: str, edge_is_udp: bool) -> str | None:
"""Why one address cannot be forwarded the way it was told to, or `None`.
Asked of an **address** rather than of the service, because that is where it is
chosen: what is behind `:443` is the same daemon as what is behind `:80`, but the two
reach it differently — one may be re-encrypted and the other handed the plaintext, and
on an `http` service the two are separate ports with separate answers. Asked from the
three places that would otherwise drift: when an address is created, when one is
added, and in `ProxyTransport.check`.
One thing is impossible, and it is impossible rather than unimplemented: a QUIC edge
carrying something that is **not HTTP/3**. What lets an HTTP/3 client reach a service
that speaks HTTP/1.1 is that the exchange is already rendered as HTTP/1.1 for the
chain, so sending that rendering on invents nothing. Opaque bytes on a QUIC stream
have no such form — there is no HTTP/1.1 spelling of "some bytes" — so an address
whose clients speak anything else has nothing that could be forwarded.
Which is a fact about the **ALPN**, not about the protocol: a `quic` service offering
`h3`, which is the default and the usual case, is exactly as renderable as an `http`
one, and an instance that has set `FGEX_PROXY_QUIC_ALPN` to something else is the one
that cannot do this — on both.
"""
if str(upstream) == UPSTREAM.SAME:
return None
if str(proto) not in (L4.TLS, L4.QUIC, L4.HTTP):
return None
alpn = quic_alpn()
if edge_is_udp and "h3" not in alpn:
return (
f"This instance offers QUIC clients {', '.join(alpn)}, and only HTTP/3 can be "
f"rendered to a service that speaks HTTP/1.1 — anything else on a QUIC stream "
f"is opaque bytes with no HTTP/1.1 form to forward. Let this address be "
f"forwarded as it arrives, or take its HTTP/3 edge away."
)
return None
class Service:
"""One protected endpoint and the transport in front of it."""

260
backend/modules/services/nftables.py
View File

@ -18,8 +18,8 @@ engine dials the service as the client, two flows look almost identical:
No address distinguishes them. Conntrack does: the first belongs to the intercepted,
redirected connection, the second is a separate entry the engine opened itself. So
`ct status dnat` lets the first out, and everything else from a protected service is
marked home. That is what makes source preservation work even with the service on
`ct status dnat` lets the first out, and everything else from a protected service — and
anything answering a connection the engine opened, from whichever port — is marked home. That is what makes source preservation work even with the service on
this very host — which is also the case that ruled tproxy out, since tproxy leaves no
NAT entry to key off and so could never reach a local service at all.
"""
@ -50,6 +50,16 @@ QUEUE_MARK_OUTPUT = 0x1338
#: growing except that every position costs two chains whether or not anyone uses it.
MAX_CHAIN_POSITIONS = 8
#: Where the NFQUEUE layer meets a connection this host opened to one of its own addresses
#: that somebody rewrote on the way — a port a container runtime publishes. Docker's DNAT
#: for this host's own traffic sits in `nat OUTPUT` at `dstnat` (-100) and sends it out
#: towards the container, so it never comes back in through prerouting where the queue
#: chains are, and neither does the container's answer: it arrives for a local socket
#: and goes to `input`. These run after that rewrite on the way out and before it is
#: undone on the way in (`srcnat`, 100), so both halves show the container's address and
#: a stream follower sees one stream.
LOCAL_QUEUE_PRIORITY = -90
#: Where the proxy layer's redirect sits in the prerouting and output hooks.
#:
#: **Before `dstnat` (-100), and that is the whole point.** Docker and podman publish a
@ -193,6 +203,11 @@ def udp_relay_host(ip: str) -> str:
#: `interface_addresses` — so the rule says which interface it belongs to itself.
IFACE_COMMENT = "iface "
#: Written on a rule that matches its address through conntrack rather than in the packet
#: — `<l4> <port> <address>` — for the same reason: `get()` reads what a rule matches to
#: know whose it is, and there is nothing in these for it to read.
ADDRESS_COMMENT = "address "
def interface_addresses(ip: str) -> list[str]:
"""Every address an interface carries right now, as a rule can match them.
@ -324,6 +339,15 @@ class FiregexTables(NFTableManager):
def queue_output_chain(position: int) -> str:
return f"fgex_queue_out_{position}"
# And the connections this host opens to a rewritten address: see LOCAL_QUEUE_PRIORITY.
@staticmethod
def queue_local_request_chain(position: int) -> str:
return f"fgex_queue_lreq_{position}"
@staticmethod
def queue_local_reply_chain(position: int) -> str:
return f"fgex_queue_lrep_{position}"
# External transport: hand the traffic to a proxy the operator runs
hijack_in_chain = "fgex_hijack_in"
hijack_local_chain = "fgex_hijack_local"
@ -350,6 +374,10 @@ class FiregexTables(NFTableManager):
-307 + position),
_chain(self.queue_output_chain(position), "filter", "postrouting",
107 + position),
_chain(self.queue_local_request_chain(position), "filter", "output",
LOCAL_QUEUE_PRIORITY + position),
_chain(self.queue_local_reply_chain(position), "filter", "input",
LOCAL_QUEUE_PRIORITY + position),
)
],
# Earlier than the queue chains, so traffic destined for somebody else's
@ -388,6 +416,43 @@ class FiregexTables(NFTableManager):
}
}
},
# The same, for a reply the socket lookup cannot place: one to a connection
# the engine opened towards an address somebody rewrote — a port a container
# runtime publishes, above all. The engine dials the published address as
# the client, Docker's DNAT sends it on to the container, and the container
# answers from its own address: `socket transparent` looks for a socket
# talking to *that* and finds none, so the answer was routed on to the real
# client, which reset it. A client on another host reached a Docker service
# through the proxy layer only after the dial timed out and was retried as
# this host — slowly, and with the client's address lost. Conntrack still
# knows whose connection it is, by the mark `fgex_route` copied onto it.
{
"add": {
"rule": {
"family": "inet",
"table": self.table_name,
"chain": self.divert_chain,
"expr": [
{
"match": {
"op": "==",
"left": {"ct": {"key": "direction"}},
"right": "reply",
}
},
{
"match": {
"op": "==",
"left": {"ct": {"key": "mark"}},
"right": PROXY_SELF_MARK,
}
},
{"mangle": {"key": {"meta": {"key": "mark"}}, "value": PROXY_MARK}},
{"accept": None},
],
}
}
},
_chain(self.nat_chain, "nat", "prerouting", NAT_PRIORITY),
# Locally generated traffic never reaches the nat prerouting hook, and
# a client on this host protecting a service on this host is exactly that.
@ -455,9 +520,8 @@ class FiregexTables(NFTableManager):
}
},
# Installed once. A packet on the intercepted (redirected) connection
# is the engine answering a client and has to leave normally; anything
# else from a protected service is a reply to the engine's own dial,
# and the per-service rule below marks it home.
# is the engine answering a client and has to leave normally; a reply to
# the engine's own dial is marked home by the two rules after this one.
{
"add": {
"rule": {
@ -477,6 +541,64 @@ class FiregexTables(NFTableManager):
}
}
},
# **A reply to a connection the engine opened, from whatever port it
# comes — and nothing else.** A per-address rule used to do this, knowing
# the service by the address it is protected on. That is where the engine
# dials it only while the address *is* the service: one published on
# another port is dialled there, and an HTTP/3 edge in front of an
# HTTP/1.1 service is dialled over TCP, and their answers were routed out
# towards the client instead of home. It also matched the answers on
# connections the engine never saw, the ones open before the service
# started, and brought those home to nobody. Conntrack knows who opened a
# connection: the engine's mark is copied onto it on the way out and read
# back off everything that answers it.
{
"add": {
"rule": {
"family": "inet",
"table": self.table_name,
"chain": self.route_chain,
"expr": [
{
"match": {
"op": "==",
"left": {"meta": {"key": "mark"}},
"right": PROXY_SELF_MARK,
}
},
{"mangle": {"key": {"ct": {"key": "mark"}},
"value": PROXY_SELF_MARK}},
],
}
}
},
{
"add": {
"rule": {
"family": "inet",
"table": self.table_name,
"chain": self.route_chain,
"expr": [
{
"match": {
"op": "==",
"left": {"ct": {"key": "direction"}},
"right": "reply",
}
},
{
"match": {
"op": "==",
"left": {"ct": {"key": "mark"}},
"right": PROXY_SELF_MARK,
}
},
{"mangle": {"key": {"meta": {"key": "mark"}},
"value": PROXY_MARK}},
],
}
}
},
],
# The whole table, which is firegex's alone. It used to be flushed chain by
# chain in one batch, and nft refuses a batch outright if any object in it is
@ -517,6 +639,19 @@ class FiregexTables(NFTableManager):
check=False, stderr=subprocess.DEVNULL,
)
def intact(self) -> bool:
"""Whether what `init()` put in the kernel is still there.
One small chain rather than the whole table, because this is asked every few
seconds: `init()` always leaves rules in the route chain, so a table that was
deleted (`nft flush ruleset`, which is how a Debian `/etc/nftables.conf` begins)
and one that was only emptied (`nft flush table`) both fail it.
"""
code, listed, _ = self.raw_cmd({"list": {"chain": {
"family": "inet", "table": self.table_name, "name": self.route_chain,
}}})
return code == 0 and any("rule" in item for item in listed["nftables"])
def reset(self):
super().reset()
# The set went with the table.
@ -709,8 +844,58 @@ class FiregexTables(NFTableManager):
],
insert=True,
))
if not is_ip_parse(ip):
# A connection this host opens is not matched by interface, for the reason
# the proxy layer's output-hook rules give — and what these catch is one
# to an address, rewritten.
continue
# This host dialling one of these addresses when something rewrote it on the
# way out: a port Docker publishes, which is how a CTF service is usually
# reached. Recognised by where the connection was *going*, which conntrack
# keeps, because by now the packet itself says the container's address.
# Without these the request and the answer both went round every queue here,
# and a payload the filter refuses from anywhere else reached the service
# from this host untouched.
original = self._rewritten_to(ip, port, family, l4)
note = f"{ADDRESS_COMMENT}{l4} {int(port)} {ip}"
cmds.append(self._rule(
self.queue_local_request_chain(position),
original + self._ct_direction("original") + [
self.COUNTER,
{"mangle": {"key": {"meta": {"key": "mark"}}, "value": QUEUE_MARK_INPUT}},
queue,
],
insert=True, comment=note,
))
cmds.append(self._rule(
self.queue_local_reply_chain(position),
original + self._ct_direction("reply") + [
{"mangle": {"key": {"meta": {"key": "mark"}}, "value": QUEUE_MARK_OUTPUT}},
queue,
],
insert=True, comment=note,
))
self.cmd(*cmds)
@staticmethod
def _rewritten_to(ip: str, port: int, family: str, l4: str) -> list:
"""A connection that was headed for `ip:port` and had its destination rewritten."""
return [
{"match": {"op": "in", "left": {"ct": {"key": "status"}}, "right": "dnat"}},
{"match": {"op": "==", "left": {"meta": {"key": "l4proto"}}, "right": l4}},
{"match": {"op": "==",
"left": {"ct": {"key": f"{family} daddr", "dir": "original"}},
"right": nftables_int_to_json(ip)}},
{"match": {"op": "==",
"left": {"ct": {"key": "proto-dst", "dir": "original"}},
"right": int(port)}},
]
@staticmethod
def _ct_direction(direction: str) -> list:
return [{"match": {"op": "==", "left": {"ct": {"key": "direction"}},
"right": direction}}]
def _add_external(self, srv: Service, addr: Address, ip: str, port: int, family: str,
l4: str):
"""Put somebody else's proxy in the path, invisibly.
@ -783,14 +968,14 @@ class FiregexTables(NFTableManager):
+ self._match(ip, port, family, l4, "daddr", "dport")
+ [self.COUNTER, {"redirect": {"port": int(proxy_port)}}]
)
# Nothing per service on the way back. The engine's answers are brought home by
# the conntrack mark on the connections it opened (`init()`), whatever address
# and port they come from. A rule here used to mark everything leaving the
# service's address and port as well, which added nothing for the engine and
# caught the connections that were open before the service started: nothing
# redirected those, so their answers belonged to the client, and a client on
# another host stopped hearing back the moment protection began.
cmds = [self._rule(self.nat_chain, redirect)]
cmds.append(
self._rule(
self.route_chain,
self._match(ip, port, family, l4, "saddr", "sport")
+ [{"mangle": {"key": {"meta": {"key": "mark"}}, "value": PROXY_MARK}}],
)
)
if not is_iface:
cmds.append(self._rule(self.nat_output_chain, redirect))
else:
@ -836,6 +1021,8 @@ class FiregexTables(NFTableManager):
chains = [
*[self.queue_input_chain(p) for p in range(MAX_CHAIN_POSITIONS)],
*[self.queue_output_chain(p) for p in range(MAX_CHAIN_POSITIONS)],
*[self.queue_local_request_chain(p) for p in range(MAX_CHAIN_POSITIONS)],
*[self.queue_local_reply_chain(p) for p in range(MAX_CHAIN_POSITIONS)],
self.hijack_in_chain,
self.hijack_local_chain,
self.hijack_out_chain,
@ -844,9 +1031,22 @@ class FiregexTables(NFTableManager):
self.nat_output_chain,
self.route_chain,
]
for rule in self.list_rules(tables=[self.table_name], chains=chains):
for rule in self.list_rules(tables=[self.table_name], chains=chains, family="inet"):
try:
expr = rule["expr"]
note = str(rule.get("comment") or "")
counter = next(
(e["counter"] for e in expr if isinstance(e, dict) and "counter" in e),
{},
)
if note.startswith(ADDRESS_COMMENT):
proto, port, ip_int = note[len(ADDRESS_COMMENT):].split(" ", 2)
res.append(InstalledRule(
chain=rule["chain"], handle=int(rule["handle"]), proto=proto,
port=int(port), ip_int=ip_int,
packets=counter.get("packets", 0), bytes_=counter.get("bytes", 0),
))
continue
# An intercept rule starts with the self-mark exclusion; every other
# per-service rule starts with the address. The two rules init()
# installs match on `socket` or on `ct`, and fall out here.
@ -869,13 +1069,8 @@ class FiregexTables(NFTableManager):
# The comment does, and reading it back here is what lets the rest of
# this file go on treating the rule as the interface's — including
# `delete()`, which would otherwise never find it.
note = str(rule.get("comment") or "")
if note.startswith(IFACE_COMMENT):
ip_int = note[len(IFACE_COMMENT):]
counter = next(
(e["counter"] for e in expr if isinstance(e, dict) and "counter" in e),
{},
)
res.append(
InstalledRule(
chain=rule["chain"],
@ -887,7 +1082,7 @@ class FiregexTables(NFTableManager):
bytes_=counter.get("bytes", 0),
)
)
except (KeyError, TypeError, IndexError):
except (KeyError, TypeError, IndexError, ValueError):
continue # a rule of a shape we did not write; not ours to touch
return res
@ -912,7 +1107,9 @@ class FiregexTables(NFTableManager):
# says. The engine already reports connections properly, so this reports nothing
# there rather than a packet count that is off by the length of every flow.
inbound = {
TRANSPORT.NFQUEUE: {self.queue_input_chain(0)},
# Two chains, but never both for one packet: from elsewhere a request passes
# prerouting, and from this host to a rewritten address it passes only output.
TRANSPORT.NFQUEUE: {self.queue_input_chain(0), self.queue_local_request_chain(0)},
TRANSPORT.EXTERNAL: {self.hijack_in_chain, self.hijack_local_chain},
}.get(srv.transport, set())
targets = addresses if addresses is not None else srv.addresses
@ -923,11 +1120,16 @@ class FiregexTables(NFTableManager):
total_bytes += rule.bytes
return {"packets": packets, "bytes": total_bytes}
def delete(self, srv: Service, addresses: list[Address] | None = None):
def delete(self, srv: Service, addresses: list[Address] | None = None,
keep_guards: bool = False) -> set[tuple[str, int]]:
"""Take back the rules for these addresses, or for all of the service's.
Passing a subset is how one address is removed from a running service: the
others keep their rules, and the datapath keeps their connections.
`keep_guards` leaves the engine's ports guarded and hands them back instead, for
`release_guards` once the engine has gone: one draining the connections it still
carries is listening on them until then.
"""
targets = addresses if addresses is not None else srv.addresses
cmds = [
@ -951,7 +1153,21 @@ class FiregexTables(NFTableManager):
# so its port stays guarded until the engine goes; released earlier, the relay
# would be reachable directly again with nothing steering traffic at it.
if addresses is None:
for l4, port in self._guarded.pop(str(srv.id), set()):
guarded = self._guarded.pop(str(srv.id), set())
if keep_guards:
return guarded
self.release_guards(guarded)
return set()
def release_guards(self, ports: set[tuple[str, int]]) -> None:
"""Stop guarding ports an engine no longer listens on.
Except one some running service is guarding now: an engine that drained for
minutes gave its port back, and the kernel is free to have handed it to another.
"""
owned = set().union(*self._guarded.values()) if self._guarded else set()
for l4, port in ports:
if (l4, port) not in owned:
self._unguard(l4, port)
def _element(self, verb: str, l4: str, port: int) -> dict:

7
backend/modules/services/pyworker.py
View File

@ -271,6 +271,13 @@ class Filters:
if ctx is None:
ctx = {}
exec(self.code, ctx, ctx) # noqa: S102
# No open frame came for it, so this worker is taking over a connection
# already under way — new code, or one restarted after the last died — and
# its first chunk can be the middle of a message. The library then carries
# what it cannot read instead of judging half a message; see
# `_met_mid_message` in the HTTP model.
if connection not in self.endpoints:
ctx["__firegex_joined_late"] = True
self.contexts[connection] = ctx
return ctx

269
backend/modules/services/transports.py
View File

@ -22,11 +22,12 @@ import asyncio
import json
import os
import sys
import time
import traceback
from typing import NamedTuple
from modules.services.models import (KIND, L4, PROTO, TRANSPORT, UPSTREAM, Filter,
Regex, quic_alpn, upstream_refusal)
Regex)
from modules.services.nftables import (MAX_CHAIN_POSITIONS, NoRelayAddress,
interface_addresses, service_at, udp_relay_host,
udp_relay_key, udp_relay_slot)
@ -52,9 +53,40 @@ PYWORKER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pyworker.py
#: both network layers.
PYWORKER_EXCEPTION_MARK = "[exception] [filter]"
#: How long to wait for a binary to acknowledge a configuration push.
#: How long to wait for a binary to acknowledge a control command, or to say it started.
ACK_TIMEOUT = 3
#: How long a binary may take over one filter's worth of a configuration push. Loading a
#: Python filter runs its module body, which the code check lets take up to eight seconds
#: (`CHECK_TIMEOUT` in the router) — so a push held to `ACK_TIMEOUT` gave up on a file the
#: check had just accepted, and the answer that came afterwards was read as the answer to
#: whichever command was sent next.
LOAD_TIMEOUT = 10
#: How long a proxy engine taken out of service may go on carrying the connections it
#: already had. It is taken out of the rules at once — new connections go straight to the
#: service, or to the engine that replaced it — and it exits as soon as it reports none
#: left. Killed instead, which is what a stop used to do, every connection it carried
#: was cut: a download, a websocket, a request half answered. The ceiling is for the ones
#: that never close.
DRAIN_LIMIT = 600
#: The engine reports its connections every two seconds (`STATS_INTERVAL` in `main.rs`);
#: a report sent before it was taken out of the rules cannot say it has none left.
DRAIN_FRESH_AFTER = 2.5
#: How many engines of one service may be draining at once. A service restarted over and
#: over while clients hold connections open would otherwise keep one process per restart.
MAX_DRAINING = 3
#: Every engine taken out of service and still carrying connections, by service id.
_draining: dict[str, list["ProxyTransport"]] = {}
async def stop_draining() -> None:
"""End every drain now: the rules that carry their connections home are going too."""
for engines in list(_draining.values()):
for engine in list(engines):
await engine._end_drain(None)
class UnsupportedChain(Exception):
"""The chosen transport cannot host the chain that was asked of it."""
@ -254,6 +286,17 @@ class Transport:
async def stop(self) -> None:
raise NotImplementedError
async def retire(self, keep_filtering: bool, then=None) -> None:
"""Stop, sparing whatever connections this layer can spare. See `ProxyTransport`.
Only the proxy layer has any to spare: it is the one that terminates them. A
queued service's connections never belonged to its processes, and a handed-off
one's belong to the operator's proxy — both go on when the rules come off.
"""
await self.stop()
if then:
then()
async def _kill(self):
if self.process and self.process.returncode is None:
self.process.kill()
@ -308,6 +351,21 @@ class Transport:
return asyncio.create_task(run())
def _abandon(self, process, what: str, command: str, waited: float) -> None:
"""Give up on a datapath process that stopped answering its control channel.
Killed rather than waited on further, because its answer may still come — and it
would then be read as the answer to the next command, and every command after that
would be told what the one before it got. Killed **without** being marked stopped,
so its watchdog reports it and the service is rebuilt from the saved configuration:
what was applied, if anything, cannot be known from here.
"""
if self.on_engine:
self.on_engine(f"[error] [backend] {what} did not acknowledge {command} within "
f"{waited:.0f}s; restarting it")
if process is not None and process.returncode is None:
process.kill()
async def _died_because(self, fallback: str) -> str:
"""What to tell the operator when the datapath would not come up.
@ -352,6 +410,9 @@ class _QueueStage:
#: did not. Without the distinction every ordinary stop looks like a crash.
self._stopped = False
self._watchdog: asyncio.Task | None = None
#: What this process last acknowledged, so an edit elsewhere in the chain does
#: not send it the same filters again. See `reload`.
self._in_force: bytes | None = None
async def start(self) -> int:
if self.link.kind == KIND.PYFILTER:
@ -453,25 +514,48 @@ class _QueueStage:
self._sock_ready.set()
async def _pump_output(self):
import codecs
# Whole lines only, decoded across reads: a line cut at a read boundary was two
# log entries, and a character cut there was two replacement marks.
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
pending = ""
try:
while True:
data = await self.process.stdout.read(10 * 1024)
if not data:
return
text = data.decode(errors="replace")
# The binary's own diagnostics share this pipe with the user's output, so
# the reason it gives for dying is read here or not at all — and without
# it a stage that would not start was reported as merely "would not
# start", with the sentence that said why sitting in the log.
for line in text.splitlines():
if "[fatal]" in line:
self.owner._last_fatal = line.split("[fatal]", 1)[1].strip()
if self.owner.on_output:
self.owner.on_output(self.srv.id, text)
break
*lines, pending = (pending + decoder.decode(data)).split("\n")
self._forward_output(lines)
except (asyncio.CancelledError, asyncio.IncompleteReadError):
pass
except Exception:
traceback.print_exc()
if pending:
self._forward_output([pending])
def _forward_output(self, lines: list[str]) -> None:
"""The binary's own diagnostics share this pipe with whatever the user's code prints.
So both go through the classifier the other datapaths' stderr goes through, which
reads a line's tag for its level, a traceback as an error, and an untagged line as
the user's own output. They used to go in as the user's output, all of them:
`[error]` lines and the traceback of a filter that raised were grey text among the
prints, where a warning is looked for and not found.
"""
for line in lines:
# The reason it gives for dying is read here or not at all — and without it a
# stage that would not start was reported as merely "would not start", with
# the sentence that said why sitting in the log.
if "[fatal]" in line:
self.owner._last_fatal = line.split("[fatal]", 1)[1].strip()
text = "\n".join(lines)
if not text.strip():
return
if self.owner.on_engine:
self.owner.on_engine(text)
elif self.owner.on_output:
self.owner.on_output(self.srv.id, text)
# --- shared ---------------------------------------------------------------
@ -518,7 +602,16 @@ class _QueueStage:
payload = self._regex_payload(link.regexes)
else:
payload = self._python_payload(link)
# Every stage is reloaded on every edit to the chain, and a new configuration is
# a clean slate for the connections under way: `cppregex` resets its matchers
# and `cpproxy` its filters' state. For the filter that was edited that is the
# point. For the others it cost every open connection its place — a request in
# the middle of an upload was read again from its second half — over a pattern
# changed in a filter they have nothing to do with.
if payload == self._in_force:
return
await self._push(payload)
self._in_force = payload
def _regex_payload(self, regexes: list[Regex]) -> bytes:
"""cppregex takes its patterns as hex codes, one per direction.
@ -586,14 +679,25 @@ class _QueueStage:
else:
raise Exception("the nfqueue binary is not running")
try:
ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT)
ok, detail = await asyncio.wait_for(self._ack, timeout=LOAD_TIMEOUT)
except asyncio.TimeoutError:
await self.stop()
# Whether it took the new filters or is still at it cannot be known now, and
# an answer arriving later would be read as the answer to the next push. So
# it is killed *without* being marked stopped: the watchdog reports it and
# the service is rebuilt from what the database says — which, after an edit
# that was refused, is what it said before. At start the caller stops every
# stage anyway.
self.owner._abandon(self.process, f"the {self.link.filter.name} interceptor",
"the filters", LOAD_TIMEOUT)
raise Exception("the nfqueue binary did not acknowledge the filters")
finally:
self._ack = None
if not ok:
await self.stop()
# Not fatal, and it must not be made so: both binaries answer `ACK FAIL` with
# the filters they had still in force. This used to stop the stage, which
# during an edit left a service reported active with nothing reading its
# queue — unfiltered where it fails open, silent where it does not — and
# stopped on purpose, so the watchdog had nothing to say about it.
raise Exception(f"the nfqueue binary rejected the filters: {detail}")
async def stop(self) -> None:
@ -794,14 +898,6 @@ class ProxyTransport(Transport):
@classmethod
def check(cls, srv, chain: list[ChainLink]) -> None:
super().check(srv, chain)
# Asked here as well as where each was set, because an address added afterwards
# can be one this instance cannot honour.
for addr in srv.addresses:
refusal = upstream_refusal(
srv.proto, addr.upstream, L4.l4_of(addr.edge) == L4.UDP
)
if refusal:
raise UnsupportedChain(refusal)
if srv.carries(L4.UDP):
from utils import get_interface_ips, is_ip_parse
# Only the addresses that need a relay of their own. On an `http` service
@ -831,6 +927,11 @@ class ProxyTransport(Transport):
#: did not — the same distinction the NFQUEUE stages make.
self._stopped = False
self._watchdog: asyncio.Task | None = None
#: When the engine last reported its counters, for a drain to know that `live`
#: was counted after the engine was taken out of the rules.
self._stats_at = 0.0
self._drain: asyncio.Task | None = None
self._after_drain = None
def _published(self) -> list[str]:
"""What this engine is told about its TCP addresses, one entry each.
@ -944,21 +1045,13 @@ class ProxyTransport(Transport):
"FGEX_PROXY_TLS": "1",
"FGEX_PROXY_TLS_OPTIONAL": "1",
"FGEX_PROXY_QUIC": "1",
"FGEX_PROXY_QUIC_ALPN": ",".join(quic_alpn()),
}
if str(self.srv.proto) == L4.QUIC:
return {
**material,
"FGEX_PROXY_QUIC": "1",
# What the engine offers the service, in this order. It cannot be asked
# of the client the way the TLS path asks: a QUIC ClientHello arrives
# inside an encrypted Initial whose processing *is* the handshake, so
# there is nothing to hold it at and nothing to read before answering.
# `h3` is what a QUIC service speaks nine times in ten; an instance in
# front of something else sets this in its own environment, the same way
# the filter deadline beside it is set. Per service it would be a column,
# and nothing yet has wanted one.
"FGEX_PROXY_QUIC_ALPN": ",".join(quic_alpn()),
# No protocol list: the engine offers the service what each client
# offered, read off the client's first packet — see `quic_hello.rs`.
}
return material
@ -1135,13 +1228,19 @@ class ProxyTransport(Transport):
# to an engine that was not there to apply it. The watchdog restarts a dead
# engine from what the database holds.
raise Exception("the proxy engine is not running")
# The engine answers once every Python filter of the chain has loaded, one after
# the other, so the wait grows with them.
python = sum(1 for link in chain
if link.filter.active and link.kind == KIND.PYFILTER and link.code_path)
waited = LOAD_TIMEOUT * (1 + python)
async with self._cmd_lock:
self._ack = asyncio.get_running_loop().create_future()
self.process.stdin.write((self._payload(chain) + "\n").encode())
await self.process.stdin.drain()
try:
ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT)
ok, detail = await asyncio.wait_for(self._ack, timeout=waited)
except asyncio.TimeoutError:
self._abandon(self.process, "the proxy engine", "the chain", waited)
raise Exception("the proxy engine did not acknowledge the chain")
finally:
self._ack = None
@ -1181,6 +1280,7 @@ class ProxyTransport(Transport):
try:
ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT)
except asyncio.TimeoutError:
self._abandon(self.process, "the proxy engine", "ADD_UDP", ACK_TIMEOUT)
raise Exception(f"the proxy engine did not acknowledge ADD_UDP for {target}")
finally:
self._ack = None
@ -1221,6 +1321,7 @@ class ProxyTransport(Transport):
try:
ok, detail = await asyncio.wait_for(self._ack, timeout=ACK_TIMEOUT)
except asyncio.TimeoutError:
self._abandon(self.process, "the proxy engine", line.split()[0], ACK_TIMEOUT)
raise Exception(f"the proxy engine did not acknowledge {line.split()[0]}")
finally:
self._ack = None
@ -1253,6 +1354,7 @@ class ProxyTransport(Transport):
key, _, value = token.partition("=")
if value.isdigit():
self.counters[key] = int(value)
self._stats_at = time.monotonic()
# Only the increase, and only when there is one: the engine counts
# from zero every time it starts, so handing the absolute number up
# would reset the durable trace on every restart of the service.
@ -1281,6 +1383,103 @@ class ProxyTransport(Transport):
# reads it is still running would be a key removed from under a restart.
self._clear_crypto_material()
async def retire(self, keep_filtering: bool, then=None) -> None:
"""Take the engine out of service without cutting the connections it carries.
The caller has already taken it out of the rules, so nothing new reaches it; what
conntrack already sent here keeps coming here, and is carried until it closes.
`keep_filtering` is a restart's answer — the connections go on under the filters
they started with — and a stop's is to carry them unfiltered: stopping is how an
operator takes a filter that is hurting the service out of the way, and a filter
that went on judging the connections already open would not be out of the way.
`then` runs once the engine has gone, which is when its ports can stop being
guarded: until then a relay would be reachable directly with nothing steering
traffic at it.
"""
self._stopped = True
if self._watchdog and self._watchdog is not asyncio.current_task():
self._watchdog.cancel()
self._watchdog = None
self.port = None
self._after_drain = then
# The engine read these when it started and never again, and a restart writes
# its own copies for the engine replacing this one: removed later, they would be
# the new engine's.
self._clear_crypto_material()
if not self.process or self.process.returncode is not None:
await self._end_drain(None)
return
if not keep_filtering:
try:
await self.reload([])
except Exception:
# Not answering is `_abandon`'s to deal with, and it already has.
pass
engines = _draining.setdefault(self.srv.id, [])
engines.append(self)
while len(engines) > MAX_DRAINING:
oldest = engines[0]
await oldest._end_drain(
f"[warn] [drain] {oldest.counters.get('live', 0)} connection(s) carried "
f"from before an earlier restart were closed: {MAX_DRAINING} engines were "
f"already draining")
self._drain = asyncio.create_task(self._wait_out(keep_filtering, time.monotonic()))
async def _wait_out(self, kept_filtering: bool, began: float) -> None:
told = False
while self.process and self.process.returncode is None:
await asyncio.sleep(0.5)
fresh = self._stats_at >= began + DRAIN_FRESH_AFTER
live = self.counters.get("live", 0)
if fresh and live == 0:
if told:
await self._end_drain("[info] [drain] the connections from before "
"have all closed")
else:
await self._end_drain(None)
return
if fresh and not told:
told = True
self._say(
f"[info] [drain] {live} connection(s) were open through the engine; "
f"they carry on "
f"{'under the filters they started with' if kept_filtering else 'unfiltered'}"
f" until they close, for up to {DRAIN_LIMIT // 60} minutes")
if time.monotonic() - began >= DRAIN_LIMIT:
await self._end_drain(
f"[warn] [drain] {live} connection(s) from before were still open after "
f"{DRAIN_LIMIT // 60} minutes and were closed")
return
await self._end_drain(None)
async def _end_drain(self, said: str | None) -> None:
engines = _draining.get(self.srv.id, [])
if self in engines:
engines.remove(self)
if not engines:
_draining.pop(self.srv.id, None)
if self._drain and self._drain is not asyncio.current_task():
self._drain.cancel()
self._drain = None
if said:
self._say(said)
for task in (self._reader_task, self._stderr_pump):
if task:
task.cancel()
self._reader_task = self._stderr_pump = None
await self._kill()
then, self._after_drain = self._after_drain, None
if then:
try:
then()
except Exception:
traceback.print_exc()
def _say(self, line: str) -> None:
if self.on_engine:
self.on_engine(line)
class ExternalTransport(Transport):
"""Hand the traffic to a proxy the operator runs themselves.

14
backend/proxysrc/README.md
View File

@ -227,13 +227,12 @@ terminates (`quic::QuicManager`) instead of something that forwards (`udp::UdpMa
`relays::Relays` is the two-armed enum both the startup list and the control channel go
through.
- **The ALPN is asked of the service**, which is the one thing that could not be kept from
the TLS path: a QUIC ClientHello arrives inside an encrypted Initial whose processing
*is* the handshake, and quinn offers nothing to hold it at. So the order is reversed —
the service is dialled first with the candidate list (`FGEX_PROXY_QUIC_ALPN`, default
`h3`) and the client is advertised exactly the one it chose. The invariant survives; what
is lost is knowing in advance whether the client would take it, which shows up as a
failed handshake with a log line naming the protocol.
- **The ALPN is mirrored, as on the TLS path.** quinn offers nothing to hold a ClientHello
at, but an Initial packet's keys come from the connection ID it carries in the clear, so
`quic_hello.rs` reads the client's list off the socket before the handshake is answered.
The service is dialled first offering exactly that list, and the client is advertised
the one protocol the service chose — nothing to configure, whatever the service speaks.
A hello that cannot be read falls back to offering `h3`, and says so.
- **One stream is one connection to the chain**, with its own sessions and its own module
globals, released when the stream ends. A filter's state follows a stream of bytes from
start to end, which is what a QUIC stream is and what a QUIC connection is not.
@ -486,7 +485,6 @@ outliving its control channel matters.
| `FGEX_PROXY_TLS` / `_TLS_OPTIONAL` | terminate TLS on the TCP listener; `_OPTIONAL` means only for the connections that start a handshake |
| `FGEX_PROXY_TLS_CERT` / `_TLS_KEY` | **paths**, never the material |
| `FGEX_PROXY_QUIC` | bind the relays with a QUIC endpoint instead of a datagram socket |
| `FGEX_PROXY_QUIC_ALPN` | what to offer the service, in order. Default `h3` |
| `FGEX_PROXY_MAX_CONNECTIONS` | 0 is no limit |
| `FGEX_PROXY_OVER_LIMIT_FORWARD` | forward what does not fit, unfiltered, instead of refusing it |
| `FGEX_PROXY_FIRST_BYTE_TIMEOUT` | seconds; 0 is off |

15
backend/proxysrc/src/control.rs
View File

@ -11,16 +11,10 @@ use tokio::io::{AsyncBufReadExt, BufReader};
use crate::filter::{ChainHandle, FilterChain};
use crate::proxy::{Edge, Onward, Published, Targets};
use crate::relays::Relays;
use crate::rules::parse_ruleset;
use crate::rules::parse_ruleset_reusing;
fn ack(ok: bool, detail: &str) {
if ok {
println!("ACK OK");
} else {
println!("ACK FAIL {detail}");
}
use std::io::Write;
let _ = std::io::stdout().flush();
crate::report::reply(if ok { "ACK OK".to_string() } else { format!("ACK FAIL {detail}") });
}
/// Apply rulesets and handle control commands until stdin closes.
@ -80,7 +74,7 @@ pub async fn serve_stdin(
match target_str.parse::<SocketAddr>() {
Ok(upstream) => match relays.add_relay(upstream, onward).await {
Ok(port) => {
println!("UDP {upstream}|{} {port}", onward.word());
crate::report::reply(format!("UDP {upstream}|{} {port}", onward.word()));
ack(true, "");
}
Err(e) => {
@ -165,8 +159,9 @@ pub async fn serve_stdin(
// runtime with every relay: with one worker thread, which is what a one-core
// host is given, the service carried nothing while a reload was loading a file.
let line = trimmed.to_string();
let previous = chain.current().filters();
let prepared = tokio::task::spawn_blocking(move || {
parse_ruleset(&line).and_then(|filters| {
parse_ruleset_reusing(&line, &previous).and_then(|filters| {
for filter in &filters {
filter.prepare()?;
}

117
backend/proxysrc/src/filter.rs
View File

@ -8,7 +8,8 @@
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use std::sync::OnceLock;
use std::time::{Duration, Instant};
/// Which half of the connection a chunk came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -189,6 +190,18 @@ pub trait Filter: Send + Sync + 'static {
/// process of its own — has no other way to learn that it can let go, and would
/// otherwise grow for as long as the service runs.
fn connection_closed(&self, _connection: ConnectionId) {}
/// What makes this filter the same filter in the next ruleset, if anything does.
///
/// A ruleset is rebuilt whole on every edit, and a filter that keeps per-connection
/// state outside its sessions loses all of it when it is rebuilt: the Python worker
/// is a process, and a new one meets every open connection halfway. A filter that
/// answers here, with a key that changes whenever its configuration does, is carried
/// over into the new chain instead when the key matches. `None` — the default — is
/// always rebuilt.
fn reuse_key(&self) -> Option<&str> {
None
}
}
/// One filter's state for one direction of one connection.
@ -250,11 +263,30 @@ pub struct ChainStats {
/// because zero means "no sessions opened yet".
static CHAIN_GENERATION: AtomicU64 = AtomicU64::new(1);
/// How long a filter that kept missing its deadline sits out before it is asked again.
///
/// Slow is not broken. A Python filter misses its deadline when exchanges queue behind one
/// another, which is what a flood does — the one moment its protection is wanted — and
/// disabled for the life of the chain it stayed off until somebody next edited a rule,
/// with one log line to say so. So it is tried again after a pause, and if it is still
/// slow it loses its say again after the same few misses: a filter that genuinely never
/// returns strands a few threads per pause rather than one per chunk. A panic is broken
/// code, and stays out until the chain is replaced.
pub const REARM_AFTER: Duration = Duration::from_secs(30);
/// Milliseconds on a monotonic clock, never zero, so zero can mean "not scheduled".
fn monotonic_ms() -> u64 {
static START: OnceLock<Instant> = OnceLock::new();
START.get_or_init(Instant::now).elapsed().as_millis() as u64 + 1
}
struct Slot {
filter: Arc<dyn Filter>,
/// Set once the filter has misbehaved enough to lose its say.
disabled: AtomicBool,
consecutive_timeouts: AtomicU64,
/// When a filter that timed out is asked again (`monotonic_ms`), or 0 for never.
rearm_at_ms: AtomicU64,
}
/// An immutable set of filters. Swapping the whole chain is how configuration
@ -266,6 +298,10 @@ pub struct FilterChain {
generation: u64,
deadline: Duration,
max_consecutive_timeouts: u64,
/// How long a filter that timed out sits out. `REARM_AFTER` but for tests.
rearm_after: Duration,
/// The earliest `rearm_at_ms` of any slot, or 0: what lets the hot path skip the clock.
next_rearm_ms: AtomicU64,
/// Trips when no filter is left to consult, making the chain a pure relay.
degraded: AtomicBool,
pub stats: Arc<ChainStats>,
@ -276,6 +312,12 @@ impl FilterChain {
Self::with_policy(filters, deadline, 3)
}
/// The filters this chain consults, in order — for the next ruleset to carry over
/// the ones it has not changed (`Filter::reuse_key`).
pub fn filters(&self) -> Vec<Arc<dyn Filter>> {
self.slots.iter().map(|slot| slot.filter.clone()).collect()
}
pub fn with_policy(
filters: Vec<Arc<dyn Filter>>,
deadline: Duration,
@ -299,16 +341,26 @@ impl FilterChain {
filter,
disabled: AtomicBool::new(false),
consecutive_timeouts: AtomicU64::new(0),
rearm_at_ms: AtomicU64::new(0),
})
.collect(),
generation: CHAIN_GENERATION.fetch_add(1, Ordering::Relaxed),
deadline,
max_consecutive_timeouts,
rearm_after: REARM_AFTER,
next_rearm_ms: AtomicU64::new(0),
degraded,
stats: Arc::new(ChainStats::default()),
}
}
/// The same chain, with filters that time out asked again after `after` rather than
/// `REARM_AFTER`.
pub fn rearming_after(mut self, after: Duration) -> Self {
self.rearm_after = after;
self
}
pub fn empty() -> Self {
Self::new(Vec::new(), Duration::from_millis(50))
}
@ -333,11 +385,26 @@ impl FilterChain {
return; // already out of the loop, do not double count
}
self.stats.filters_disabled.fetch_add(1, Ordering::Relaxed);
match reason {
DisableReason::TimedOut => {
let at = monotonic_ms() + self.rearm_after.as_millis() as u64;
slot.rearm_at_ms.store(at, Ordering::Relaxed);
let _ = self.next_rearm_ms.fetch_update(Ordering::Relaxed, Ordering::Relaxed,
|next| (next == 0 || at < next).then_some(at));
eprintln!(
"[warn] [filter] '{}' disabled after {:?}: traffic keeps flowing without it",
"[warn] [filter] '{}' kept missing its {:?} deadline: traffic keeps \
flowing without it, and it is asked again in {}s",
slot.filter.name(),
reason
self.deadline,
self.rearm_after.as_secs(),
);
}
DisableReason::Panicked => eprintln!(
"[warn] [filter] '{}' panicked: traffic keeps flowing without it until the \
filters are next applied — save a change to them, or restart the service",
slot.filter.name(),
),
}
if self
.slots
.iter()
@ -367,6 +434,38 @@ impl FilterChain {
/// Run the chain over one chunk.
///
/// Put back the filters whose pause is over. See `REARM_AFTER`.
fn rearm(&self) {
let now = monotonic_ms();
let mut next = 0;
for slot in &self.slots {
let at = slot.rearm_at_ms.load(Ordering::Relaxed);
if at == 0 {
continue;
}
if now < at {
next = if next == 0 { at } else { next.min(at) };
continue;
}
// Whoever wins this puts the filter back; everybody else sees it done.
if slot
.rearm_at_ms
.compare_exchange(at, 0, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
slot.consecutive_timeouts.store(0, Ordering::Relaxed);
slot.disabled.store(false, Ordering::Relaxed);
self.degraded.store(false, Ordering::Relaxed);
eprintln!(
"[info] [filter] '{}' is back in the chain after sitting out {}s",
slot.filter.name(),
self.rearm_after.as_secs(),
);
}
}
self.next_rearm_ms.store(next, Ordering::Relaxed);
}
/// Any way a filter can fail — panic, or simply never returning — resolves to
/// `Accept` for the chunk in flight. The chunk is never held hostage by the
/// filter's failure.
@ -376,6 +475,10 @@ impl FilterChain {
data: &[u8],
sessions: &mut ChainSessions,
) -> Verdict {
let due = self.next_rearm_ms.load(Ordering::Relaxed);
if due != 0 && monotonic_ms() >= due {
self.rearm();
}
if self.is_bypassed() {
self.stats.bypassed_chunks.fetch_add(1, Ordering::Relaxed);
return Verdict::Accept;
@ -418,9 +521,7 @@ impl FilterChain {
Ok(Verdict::Reject(by)) => {
self.stats.rejected.fetch_add(1, Ordering::Relaxed);
let id = by.clone().unwrap_or_else(|| slot.filter.name().to_string());
println!("BLOCKED {id}");
use std::io::Write;
let _ = std::io::stdout().flush();
crate::report::blocked(&id);
return Verdict::Reject(by);
}
Err(_) => {
@ -456,9 +557,7 @@ impl FilterChain {
// The backend attributes the block to a rule by this id,
// the same way cppregex reports `BLOCKED <id>`.
let id = by.clone().unwrap_or_else(|| slot.filter.name().to_string());
println!("BLOCKED {id}");
use std::io::Write;
let _ = std::io::stdout().flush();
crate::report::blocked(&id);
return Verdict::Reject(by);
}
}

2
backend/proxysrc/src/lib.rs
View File

@ -15,6 +15,7 @@ pub mod http1;
pub mod hyperscan;
pub mod proxy;
pub mod quic;
pub mod quic_hello;
pub mod pyworker;
pub mod rules;
pub mod spec;
@ -22,6 +23,7 @@ pub mod capture;
pub mod tls;
pub mod transparent;
pub mod relays;
pub mod report;
pub mod udp;
pub use filter::{

47
backend/proxysrc/src/main.rs
View File

@ -70,28 +70,6 @@ fn read_material() -> Result<Option<(String, String)>, String> {
}
}
/// The protocols offered to the service, in the operator's order.
///
/// Offered to the *service*, which is what makes the list needed at all: the client's own
/// list arrives inside an encrypted Initial and cannot be read before answering, so this
/// is what firegex has to go on. `h3` alone by default, because that is what a QUIC
/// service is nine times in ten; anything else — a game protocol, a CTF's own — is named
/// here or its handshake fails saying so.
fn quic_alpn() -> Vec<Vec<u8>> {
let raw = std::env::var("FGEX_PROXY_QUIC_ALPN").unwrap_or_default();
let listed: Vec<Vec<u8>> = raw
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(|s| s.as_bytes().to_vec())
.collect();
if listed.is_empty() {
vec![b"h3".to_vec()]
} else {
listed
}
}
fn build_tls(material: Option<&(String, String)>, optional: bool) -> Result<TlsSetup, String> {
let server = match material {
Some((cert, key)) => Some(fgex_proxy::tls::server_config(cert, key)?),
@ -383,9 +361,7 @@ async fn run() {
// pair of numbers whichever half of it the traffic arrived on.
let counters = proxy.stats();
println!("PORT {}", bound.port());
use std::io::Write;
let _ = std::io::stdout().flush();
fgex_proxy::report::reply(format!("PORT {}", bound.port()));
eprintln!(
"[info] [main] listening on {bound} -> {upstream_raw} \
(spoof_source={spoof_source})"
@ -405,8 +381,7 @@ async fn run() {
exit(2);
}
};
let alpn = quic_alpn();
let setup = match QuicSetup::build(cert, key, alpn.clone()) {
let setup = match QuicSetup::build(cert, key) {
Ok(setup) => Arc::new(setup),
Err(e) => {
eprintln!("[fatal] [main] {e}");
@ -414,11 +389,8 @@ async fn run() {
}
};
eprintln!(
"[info] [main] QUIC terminated here; offering the service {}",
alpn.iter()
.map(|p| String::from_utf8_lossy(p).into_owned())
.collect::<Vec<_>>()
.join(", ")
"[info] [main] QUIC terminated here; each client's own protocols are offered \
to the service"
);
Relays::Quic(QuicManager::new(
chain_handle.clone(),
@ -474,9 +446,7 @@ async fn run() {
// With what it speaks onward, because that is half of what names a
// relay: two addresses sending to one service port can want different
// answers, and each gets a relay of its own.
println!("UDP {upstream}|{} {port}", onward.word());
use std::io::Write;
let _ = std::io::stdout().flush();
fgex_proxy::report::reply(format!("UDP {upstream}|{} {port}", onward.word()));
}
Err(e) => {
eprintln!("[fatal] [main] cannot bind a UDP relay for {upstream}: {e}");
@ -507,15 +477,16 @@ async fn run() {
// the backend parses this line as key/value pairs, so a new number costs no
// protocol. `over_limit` is cumulative on purpose — a service that hit the
// wall once an hour ago still says so, which is the whole point of a trace.
println!(
// Dropped rather than waited for when the backend is behind: the next one
// carries the same cumulative numbers two seconds later.
fgex_proxy::report::event(format!(
"STATS seen={} refused={} live={} over_limit={} no_first_byte={}",
counters.accepted.load(std::sync::atomic::Ordering::Relaxed),
counters.closed_by_filter.load(std::sync::atomic::Ordering::Relaxed),
counters.live.load(std::sync::atomic::Ordering::Relaxed),
counters.over_limit.load(std::sync::atomic::Ordering::Relaxed),
counters.no_first_byte.load(std::sync::atomic::Ordering::Relaxed),
);
let _ = std::io::stdout().flush();
));
}
});

36
backend/proxysrc/src/pyworker.rs
View File

@ -114,6 +114,9 @@ pub struct PyWorkerRule {
/// chunk, which runs where blocking is allowed and keeps an open ahead of the data.
pending: Mutex<Vec<u8>>,
pub stats: std::sync::Arc<WorkerStats>,
/// Everything that would make a new worker behave differently from this one; see
/// `Filter::reuse_key`. `None` when the file could not be read to say so.
reuse_key: Option<String>,
}
impl PyWorkerRule {
@ -133,9 +136,39 @@ impl PyWorkerRule {
child: Mutex::new(None),
pending: Mutex::new(Vec::new()),
stats: std::sync::Arc::new(WorkerStats::default()),
reuse_key: None,
}
}
/// What this rule would be built from, as one string: the id, the command, which
/// functions are on, the deadline and the code itself. The same string means a new
/// worker would run exactly what this one runs, so it is kept instead — with the
/// state it holds for every open connection. The code is read rather than trusted to
/// its path, because saving a filter rewrites the same file.
pub fn key_for(
id: &str,
command: &[String],
code_path: &str,
enabled: &Option<Vec<String>>,
deadline: Duration,
) -> Option<String> {
use std::hash::{Hash, Hasher};
let code = std::fs::read(code_path).ok()?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
code.hash(&mut hasher);
Some(format!(
"{id}\0{command:?}\0{code_path}\0{enabled:?}\0{}\0{:016x}",
deadline.as_millis(),
hasher.finish()
))
}
/// Answer to `key` from `Filter::reuse_key`.
pub fn reusable_as(mut self, key: Option<String>) -> Self {
self.reuse_key = key;
self
}
fn spawn(&self) -> Result<Child, String> {
let (program, args) = self
.command
@ -343,6 +376,9 @@ impl Filter for PyWorkerRule {
fn prepare(&self) -> Result<(), String> {
self.warm_up()
}
fn reuse_key(&self) -> Option<&str> {
self.reuse_key.as_deref()
}
fn inspect(&self, ctx: &FilterCtx<'_>) -> Verdict {
match self.exchange(ctx.connection, ctx.direction, ctx.chunk) {
Ok(verdict) => verdict,

200
backend/proxysrc/src/quic.rs
View File

@ -12,16 +12,15 @@
//! per address and for the same reason — `SO_ORIGINAL_DST` is TCP and SCTP only, so a
//! single listener could not recover where a datagram was headed.
//!
//! **The ALPN is asked of the service, not of the client**, which is the one place this
//! differs from the TLS path beside it. There the ClientHello is held open
//! (`LazyConfigAcceptor`), the service is asked, and the client is told what the service
//! picked. In QUIC the ClientHello arrives inside an encrypted Initial whose processing
//! *is* the handshake, and there is nothing to hold it at. So the order is reversed: the
//! service is offered the candidates, and the client is told the one thing the service
//! agreed to. The invariant that matters is unchanged — the client is never told a
//! protocol the service did not choose — and what is lost is knowing in advance whether
//! the client would have accepted it. When it does not, the handshake fails and says so,
//! rather than a protocol being quietly broken.
//! **The ALPN is mirrored, exactly as over TLS.** The client's ClientHello travels inside
//! an Initial packet, but Initial keys come from a connection ID the client sends in the
//! clear, so [`crate::quic_hello`] reads the list the client offered off the socket before
//! the handshake is answered. The service is dialled offering exactly that list, and the
//! client is told the one protocol the service chose. It used to be a candidate list the
//! operator wrote down per service, `h3` unless told otherwise, because the Initial was
//! taken to be unreadable: a QUIC service speaking anything but HTTP/3 was unreachable
//! until somebody found the setting. When a hello cannot be read — a QUIC version this
//! does not know, say — `h3` is offered and the log says so.
use std::collections::HashMap;
use std::io;
@ -34,6 +33,7 @@ use quinn::crypto::rustls::{QuicClientConfig, QuicServerConfig};
use quinn::{Connection, Endpoint, Incoming, VarInt};
use crate::capture::{Capture, Tap};
use crate::quic_hello::{Hellos, SniffingSocket};
use crate::filter::{
next_connection_id, ChainHandle, ChainSessions, ConnectionId, ConnectionMeta, Direction,
FilterChain, Verdict, L4,
@ -63,28 +63,37 @@ const IDLE: Duration = Duration::from_secs(60);
/// that sends faster than the chain judges would be buying memory in this process.
const DATAGRAM_BUFFER: usize = 256 * 1024;
/// How long a client's hello may take to arrive whole, when it spans more than one
/// datagram. Past it, `h3` is offered and the log says the hello could not be read.
const HELLO_WAIT: Duration = Duration::from_millis(1500);
/// What is offered to a service when the client's own list could not be read.
const FALLBACK_ALPN: &[u8] = b"h3";
/// Everything a QUIC service needs decided once, before any client arrives.
///
/// No protocols among it: those are each client's own, and each connection's accepting
/// and dialling configurations are built from them — a clone of a rustls config and a
/// cipher-suite lookup, which is nothing beside the two handshakes they are for, and the
/// same trade the TLS path makes per connection for the same reason.
pub struct QuicSetup {
/// One accept-side configuration per protocol the service might pick, built up front
/// rather than per connection: the candidate list is short and known, and a rustls
/// config clone plus a cipher-suite lookup on the accept path is work done while a
/// client is waiting.
answering: HashMap<Vec<u8>, Arc<quinn::ServerConfig>>,
/// For the case the service agreed to nothing. QUIC requires ALPN, so this is a
/// configuration that will refuse — kept so the refusal is the handshake's, with a
/// reason, rather than a panic here.
/// The service's certificate, answering. Cloned per connection to advertise the one
/// protocol the service chose.
server: Arc<rustls::ServerConfig>,
/// Towards the service. Cloned per connection to offer what the client offered.
client: rustls::ClientConfig,
/// Without datagrams, for HTTP/3; with them, for everything else. See `build`.
streams_only: Arc<quinn::TransportConfig>,
with_datagrams: Arc<quinn::TransportConfig>,
/// What the endpoint answers with before a service has been asked: only ever used
/// for connections this relay turns away, which need a configuration to be turned
/// away *by*. It advertises nothing, so nothing is agreed on it.
silent: Arc<quinn::ServerConfig>,
/// Offered to the service, in the operator's order.
client: quinn::ClientConfig,
candidates: Vec<Vec<u8>>,
}
impl QuicSetup {
/// Build both edges from the service's certificate and the protocols it may speak.
pub fn build(cert_pem: &str, key_pem: &str, candidates: Vec<Vec<u8>>) -> Result<Self, String> {
if candidates.is_empty() {
return Err("a QUIC service needs at least one ALPN protocol".to_string());
}
/// Build both edges from the service's certificate.
pub fn build(cert_pem: &str, key_pem: &str) -> Result<Self, String> {
// Two of them, differing in one advertisement. **HTTP/3 connections do not carry
// datagrams here**, and the honest way to say that is not to advertise the
// extension on them: a peer then knows from the handshake and falls back, instead
@ -95,58 +104,52 @@ impl QuicSetup {
// far side. Everything that is not h3 has no such coupling and carries them.
let streams_only = transport_config(false);
let with_datagrams = transport_config(true);
let base = crate::tls::server_config(cert_pem, key_pem)?;
let mut answering = HashMap::new();
for protocol in &candidates {
let transport = if protocol == b"h3" { &streams_only } else { &with_datagrams };
answering.insert(
protocol.clone(),
Arc::new(server_config(&base, &[protocol.clone()], transport)?),
);
}
let silent = Arc::new(server_config(&base, &[], &streams_only)?);
let mut rustls_client = crate::tls::quic_client_config()?;
rustls_client.alpn_protocols = candidates.clone();
let crypto = QuicClientConfig::try_from(rustls_client)
.map_err(|e| format!("cannot configure QUIC towards the service: {e}"))?;
let mut client = quinn::ClientConfig::new(Arc::new(crypto));
// The service is dialled before it has said which protocol it speaks, so this
// edge is decided by the candidate list rather than by the answer: with h3 the
// only candidate there is nothing a datagram could be for, and advertising it
// upstream would invite what this end would then have to drop.
client.transport_config(if candidates.iter().all(|p| p == b"h3") {
Arc::clone(&streams_only)
} else {
Arc::clone(&with_datagrams)
});
let server = crate::tls::server_config(cert_pem, key_pem)?;
let silent = Arc::new(server_config(&server, &[], &streams_only)?);
Ok(Self {
answering,
server,
client: crate::tls::quic_client_config()?,
streams_only,
with_datagrams,
silent,
client,
candidates,
})
}
/// The configuration that advertises the one protocol the service agreed to.
fn answering_with(&self, agreed: Option<&[u8]>) -> Arc<quinn::ServerConfig> {
agreed
.and_then(|p| self.answering.get(p))
.map(Arc::clone)
.unwrap_or_else(|| Arc::clone(&self.silent))
fn transport_for(&self, protocols: &[Vec<u8>]) -> &Arc<quinn::TransportConfig> {
if protocols.iter().all(|p| p == b"h3") {
&self.streams_only
} else {
&self.with_datagrams
}
}
/// The configuration that advertises the one protocol the service agreed to — or
/// nothing, when it agreed to nothing, so the refusal is the handshake's with a reason.
fn answering_with(&self, agreed: Option<&[u8]>) -> Result<Arc<quinn::ServerConfig>, String> {
let alpn: Vec<Vec<u8>> = agreed.map(|p| vec![p.to_vec()]).unwrap_or_default();
if alpn.is_empty() {
return Ok(Arc::clone(&self.silent));
}
Ok(Arc::new(server_config(&self.server, &alpn, self.transport_for(&alpn))?))
}
/// Towards the service, offering exactly what the client offered. The service is
/// dialled before it has said which protocol it speaks, so the datagram extension is
/// decided by the list rather than by the answer: with nothing but h3 on it there is
/// nothing a datagram could be for, and advertising it upstream would invite what this
/// end would then have to drop.
fn client_for(&self, offered: &[Vec<u8>]) -> Result<quinn::ClientConfig, String> {
let mut rustls_client = self.client.clone();
rustls_client.alpn_protocols = offered.to_vec();
let crypto = QuicClientConfig::try_from(rustls_client)
.map_err(|e| format!("cannot configure QUIC towards the service: {e}"))?;
let mut client = quinn::ClientConfig::new(Arc::new(crypto));
client.transport_config(Arc::clone(self.transport_for(offered)));
Ok(client)
}
/// What the endpoint answers with before a service has been asked. Only ever used
/// for the connections this relay turns away, which need a configuration to be
/// turned away *by*.
fn default_server(&self) -> Arc<quinn::ServerConfig> {
self.candidates
.first()
.and_then(|p| self.answering.get(p))
.map(Arc::clone)
.unwrap_or_else(|| Arc::clone(&self.silent))
Arc::clone(&self.silent)
}
}
@ -212,6 +215,10 @@ pub struct QuicRelay {
chain: ChainHandle,
cfg: QuicConfig,
stats: Arc<ProxyStats>,
/// The hellos read off this endpoint's socket, for each client's own ALPN list.
hellos: Arc<Hellos>,
/// Whether a hello that could not be read has been said once already.
warned_hello: AtomicBool,
}
impl QuicRelay {
@ -228,10 +235,13 @@ impl QuicRelay {
let socket = std::net::UdpSocket::bind(listen)?;
let runtime = quinn::default_runtime()
.ok_or_else(|| io::Error::other("no async runtime for the QUIC endpoint"))?;
let endpoint = Endpoint::new(
// Everything quinn receives passes by the hello reader first.
let hellos = Arc::new(Hellos::new());
let sniffing = SniffingSocket::new(runtime.wrap_udp_socket(socket)?, Arc::clone(&hellos));
let endpoint = Endpoint::new_with_abstract_socket(
quinn::EndpointConfig::default(),
Some((*cfg.setup.default_server()).clone()),
socket,
Arc::new(sniffing),
runtime,
)?;
Ok(Self {
@ -240,6 +250,8 @@ impl QuicRelay {
chain,
cfg,
stats,
hellos,
warned_hello: AtomicBool::new(false),
})
}
@ -338,10 +350,11 @@ impl QuicRelay {
// request to send.
let (endpoint, behind, agreed) = match self.cfg.upstream {
Onward::Same => {
let offered = self.offered(&incoming, client).await;
// The endpoint is held for as long as the connection is: it owns the
// socket the connection speaks through, and dropping it would take the
// connection with it.
let (endpoint, service) = self.dial_upstream(client).await?;
let (endpoint, service) = self.dial_upstream(client, &offered).await?;
let agreed = negotiated_protocol(&service);
(Some(endpoint), Behind::Quic(service), agreed)
}
@ -366,9 +379,12 @@ impl QuicRelay {
),
};
let accepting = incoming
.accept_with(self.cfg.setup.answering_with(agreed.as_deref()))
let answering = self
.cfg
.setup
.answering_with(agreed.as_deref())
.map_err(io::Error::other)?;
let accepting = incoming.accept_with(answering).map_err(io::Error::other)?;
let peer = match tokio::time::timeout(self.cfg.connect_timeout, accepting).await {
Ok(Ok(connection)) => connection,
Ok(Err(e)) => {
@ -420,8 +436,32 @@ impl QuicRelay {
Ok(())
}
/// Open the connection to the service, from the client's own address where we can.
async fn dial_upstream(&self, client: SocketAddr) -> io::Result<(Endpoint, Connection)> {
/// The protocols this client offered, read off its hello; `h3` if it could not be.
async fn offered(&self, incoming: &Incoming, client: SocketAddr) -> Vec<Vec<u8>> {
match self.hellos.offered(incoming.remote_address(), HELLO_WAIT).await {
Some(offered) if !offered.is_empty() => offered,
_ => {
if !self.warned_hello.swap(true, Ordering::Relaxed) {
eprintln!(
"[warn] [quic] could not read which protocols {client} offered; \
offering {} to {} instead. A client speaking something else is \
refused by the handshake.",
String::from_utf8_lossy(FALLBACK_ALPN),
self.upstream,
);
}
vec![FALLBACK_ALPN.to_vec()]
}
}
}
/// Open the connection to the service, from the client's own address where we can,
/// offering what the client offered.
async fn dial_upstream(
&self,
client: SocketAddr,
offered: &[Vec<u8>],
) -> io::Result<(Endpoint, Connection)> {
let endpoint = match self.endpoint_as(client) {
Ok(endpoint) => endpoint,
Err(e) => {
@ -447,7 +487,11 @@ impl QuicRelay {
// authenticated — but rustls needs a name to put in the handshake.
let name = self.upstream.ip().to_string();
let connecting = endpoint
.connect_with(self.cfg.setup.client.clone(), self.upstream, &name)
.connect_with(
self.cfg.setup.client_for(offered).map_err(io::Error::other)?,
self.upstream,
&name,
)
.map_err(io::Error::other)?;
let connection = tokio::time::timeout(self.cfg.connect_timeout, connecting)
.await

552
backend/proxysrc/src/quic_hello.rs Normal file
View File

@ -0,0 +1,552 @@
//! What a QUIC client offers, read off its first packets before anybody answers them.
//!
//! The TLS path mirrors ALPN by holding the ClientHello open, asking the service with the
//! client's own list and telling the client what came back. QUIC was thought to leave
//! nothing to hold: the ClientHello travels inside an Initial packet, encrypted. But the
//! Initial keys are derived from the Destination Connection ID the client put in the
//! clear (RFC 9001 §5.2) — deliberately, so a middlebox can read exactly this — and so
//! the list can be read before the handshake is answered. That is what this does.
//!
//! What it replaced was a list of candidates the operator had to write down per service,
//! `h3` unless told otherwise: a QUIC service speaking anything else was unreachable
//! through firegex until somebody found the setting. With the client's own list the
//! service chooses from what the client actually offered, exactly as over TLS, and there
//! is nothing to configure.
//!
//! It reads; it never answers. The socket the endpoint owns is wrapped
//! ([`SniffingSocket`]) so every datagram quinn is about to process passes by here first,
//! and only long-header Initial packets are looked at — one comparison for everything
//! else. A ClientHello too large for one packet (post-quantum key shares make that
//! ordinary) is reassembled from its CRYPTO frames across datagrams, in whatever order
//! they arrive.
use std::collections::HashMap;
use std::fmt;
use std::io::{self, IoSliceMut};
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use quinn::udp::{RecvMeta, Transmit};
use quinn::{AsyncUdpSocket, UdpPoller};
use rustls::quic::{Suite, Version};
use rustls::Side;
/// How many clients' hellos may be in progress at once. Past it new ones are not read —
/// their connections fall back rather than this growing without end under a flood of
/// forged Initials, which cost the sender nothing: the keys are public, so a valid one can
/// be made from any address. With `MAX_HELLO` this bounds what a flood can make this
/// process hold to 16 MiB.
const MAX_PENDING: usize = 1024;
/// How long an unfinished or unclaimed hello is kept.
const KEPT_FOR: Duration = Duration::from_secs(10);
/// Most CRYPTO bytes kept for one hello. A ClientHello is one or two kilobytes, a few with
/// post-quantum key shares and a resumption ticket; this is a cap on what a peer can make
/// this process hold, not a size anybody meets.
const MAX_HELLO: usize = 16 * 1024;
/// One client's hello as it arrives.
struct Pending {
/// The connection ID its Initials were protected with. A new one means a new attempt —
/// a Retry answered, or another connection from the same port.
dcid: Vec<u8>,
fragments: Vec<(u64, Vec<u8>)>,
held: usize,
alpn: Option<Vec<Vec<u8>>>,
since: Instant,
}
/// The hellos being read, by the address they came from.
pub struct Hellos {
pending: Mutex<HashMap<SocketAddr, Pending>>,
arrived: tokio::sync::Notify,
suite: Suite,
}
impl fmt::Debug for Hellos {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Hellos").finish_non_exhaustive()
}
}
impl Default for Hellos {
fn default() -> Self {
Self::new()
}
}
impl Hellos {
pub fn new() -> Self {
// Every version's Initial packets are protected with this suite, whatever the
// handshake goes on to choose: RFC 9001 §5.2.
let suite = rustls::crypto::ring::cipher_suite::TLS13_AES_128_GCM_SHA256
.tls13()
.and_then(|s| s.quic_suite())
.expect("TLS13_AES_128_GCM_SHA256 carries QUIC");
Self {
pending: Mutex::new(HashMap::new()),
arrived: tokio::sync::Notify::new(),
suite,
}
}
/// Look at one datagram received from `from`. Cheap for anything but an Initial.
pub fn observe(&self, from: SocketAddr, datagram: &[u8]) {
let mut rest = datagram;
// An Initial can have other packets coalesced behind it in one datagram; only
// the Initials matter, and they come first.
while let Some(packet) = decrypt_initial(&self.suite, rest) {
rest = &rest[packet.length..];
self.absorb(from, packet);
}
}
fn absorb(&self, from: SocketAddr, packet: Initial) {
let Ok(mut pending) = self.pending.lock() else { return };
let now = Instant::now();
if pending.len() >= MAX_PENDING {
pending.retain(|_, p| now.duration_since(p.since) < KEPT_FOR);
if pending.len() >= MAX_PENDING && !pending.contains_key(&from) {
return;
}
}
let entry = pending.entry(from).or_insert_with(|| Pending {
dcid: packet.dcid.clone(),
fragments: Vec::new(),
held: 0,
alpn: None,
since: now,
});
if entry.dcid != packet.dcid {
*entry = Pending {
dcid: packet.dcid.clone(),
fragments: Vec::new(),
held: 0,
alpn: None,
since: now,
};
}
if entry.alpn.is_some() {
return;
}
for (offset, data) in packet.crypto {
if entry.held + data.len() > MAX_HELLO {
return;
}
entry.held += data.len();
entry.fragments.push((offset, data));
}
if let Some(alpn) = client_hello_alpn(&entry.fragments) {
entry.alpn = Some(alpn);
entry.fragments.clear();
self.arrived.notify_waiters();
}
}
fn take(&self, from: SocketAddr) -> Option<Vec<Vec<u8>>> {
let mut pending = self.pending.lock().ok()?;
if pending.get(&from)?.alpn.is_some() {
return pending.remove(&from).and_then(|p| p.alpn);
}
None
}
/// The protocols the client at `from` offered, waiting up to `wait` for the rest of a
/// hello split across datagrams. `None` when it could not be read.
pub async fn offered(&self, from: SocketAddr, wait: Duration) -> Option<Vec<Vec<u8>>> {
let deadline = tokio::time::Instant::now() + wait;
loop {
// Registered before looking, so a hello completed in between is not missed.
let notified = self.arrived.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if let Some(alpn) = self.take(from) {
return Some(alpn);
}
if tokio::time::timeout_at(deadline, notified).await.is_err() {
return self.take(from);
}
}
}
}
/// The socket a relay's endpoint receives through, reading hellos on the way past.
#[derive(Debug)]
pub struct SniffingSocket {
inner: Arc<dyn AsyncUdpSocket>,
hellos: Arc<Hellos>,
}
impl SniffingSocket {
pub fn new(inner: Arc<dyn AsyncUdpSocket>, hellos: Arc<Hellos>) -> Self {
Self { inner, hellos }
}
}
impl AsyncUdpSocket for SniffingSocket {
fn create_io_poller(self: Arc<Self>) -> Pin<Box<dyn UdpPoller>> {
Arc::clone(&self.inner).create_io_poller()
}
fn try_send(&self, transmit: &Transmit) -> io::Result<()> {
self.inner.try_send(transmit)
}
fn poll_recv(
&self,
cx: &mut Context,
bufs: &mut [IoSliceMut<'_>],
meta: &mut [RecvMeta],
) -> Poll<io::Result<usize>> {
let polled = self.inner.poll_recv(cx, bufs, meta);
if let Poll::Ready(Ok(count)) = &polled {
for (buf, meta) in bufs.iter().zip(meta.iter()).take(*count) {
let received = &buf[..meta.len.min(buf.len())];
// Several datagrams of `stride` bytes each where the kernel coalesced them.
let stride = if meta.stride == 0 { received.len() } else { meta.stride };
for datagram in received.chunks(stride.max(1)) {
if is_long_header(datagram) {
self.hellos.observe(meta.addr, datagram);
}
}
}
}
polled
}
fn local_addr(&self) -> io::Result<SocketAddr> {
self.inner.local_addr()
}
fn max_transmit_segments(&self) -> usize {
self.inner.max_transmit_segments()
}
fn max_receive_segments(&self) -> usize {
self.inner.max_receive_segments()
}
fn may_fragment(&self) -> bool {
self.inner.may_fragment()
}
}
fn is_long_header(datagram: &[u8]) -> bool {
datagram.first().is_some_and(|b| b & 0x80 != 0)
}
/// One Initial packet, decrypted: what it was protected with, and its CRYPTO frames.
struct Initial {
/// How many bytes of the datagram it took, so a packet coalesced behind it is found.
length: usize,
dcid: Vec<u8>,
crypto: Vec<(u64, Vec<u8>)>,
}
fn varint(buf: &[u8], pos: &mut usize) -> Option<u64> {
let first = *buf.get(*pos)?;
let len = 1usize << (first >> 6);
let bytes = buf.get(*pos..*pos + len)?;
let mut value = u64::from(first & 0x3f);
for byte in &bytes[1..] {
value = (value << 8) | u64::from(*byte);
}
*pos += len;
Some(value)
}
/// Decrypt the Initial packet at the start of `packet`, if that is what it is.
fn decrypt_initial(suite: &Suite, packet: &[u8]) -> Option<Initial> {
let first = *packet.first()?;
if first & 0x80 == 0 {
return None;
}
let version = u32::from_be_bytes(packet.get(1..5)?.try_into().ok()?);
let (version, initial_type) = match version {
0x0000_0001 => (Version::V1, 0),
0x6b33_43cf => (Version::V2, 1),
0xff00_001d..=0xff00_0020 => (Version::V1Draft, 0),
_ => return None,
};
if (first >> 4) & 0x03 != initial_type {
return None;
}
let mut pos = 5;
let dcid_len = usize::from(*packet.get(pos)?);
let dcid = packet.get(pos + 1..pos + 1 + dcid_len)?.to_vec();
pos += 1 + dcid_len;
let scid_len = usize::from(*packet.get(pos)?);
pos += 1 + scid_len;
let token_len = usize::try_from(varint(packet, &mut pos)?).ok()?;
pos = pos.checked_add(token_len)?;
let length = usize::try_from(varint(packet, &mut pos)?).ok()?;
let pn_offset = pos;
let end = pn_offset.checked_add(length)?;
if end > packet.len() {
return None;
}
let keys = suite.keys(&dcid, Side::Server, version);
let sample_len = keys.remote.header.sample_len();
let sample = packet.get(pn_offset + 4..pn_offset + 4 + sample_len)?.to_vec();
let mut buf = packet[..end].to_vec();
let (head, tail) = buf.split_at_mut(1);
let pn_field = tail.get_mut(pn_offset - 1..pn_offset + 3)?;
keys.remote
.header
.decrypt_in_place(&sample, &mut head[0], pn_field)
.ok()?;
let pn_len = usize::from(buf[0] & 0x03) + 1;
let mut number = 0u64;
for byte in &buf[pn_offset..pn_offset + pn_len] {
number = (number << 8) | u64::from(*byte);
}
let (header, payload) = buf.split_at_mut(pn_offset + pn_len);
let plain = keys
.remote
.packet
.decrypt_in_place(number, header, payload)
.ok()?;
Some(Initial {
length: end,
dcid,
crypto: crypto_frames(plain)?,
})
}
/// The CRYPTO frames of a decrypted Initial payload. `None` for a frame an Initial cannot
/// carry, which means this was not what it looked like.
fn crypto_frames(payload: &[u8]) -> Option<Vec<(u64, Vec<u8>)>> {
let mut frames = Vec::new();
let mut pos = 0;
while pos < payload.len() {
match varint(payload, &mut pos)? {
0x00 | 0x01 => {} // PADDING, PING
kind @ (0x02 | 0x03) => {
// ACK: largest, delay, range count, first range, the ranges, and ECN.
varint(payload, &mut pos)?;
varint(payload, &mut pos)?;
let ranges = varint(payload, &mut pos)?;
varint(payload, &mut pos)?;
for _ in 0..ranges {
varint(payload, &mut pos)?;
varint(payload, &mut pos)?;
}
if kind == 0x03 {
for _ in 0..3 {
varint(payload, &mut pos)?;
}
}
}
0x06 => {
let offset = varint(payload, &mut pos)?;
let len = usize::try_from(varint(payload, &mut pos)?).ok()?;
let data = payload.get(pos..pos.checked_add(len)?)?;
frames.push((offset, data.to_vec()));
pos += len;
}
// CONNECTION_CLOSE ends what there is to read.
0x1c => break,
_ => return None,
}
}
Some(frames)
}
/// The ALPN list of a ClientHello, once enough of it has arrived to have one.
///
/// `Some(vec![])` for a complete hello that offers none — which QUIC does not allow, and
/// which the handshake is left to refuse.
fn client_hello_alpn(fragments: &[(u64, Vec<u8>)]) -> Option<Vec<Vec<u8>>> {
// What has arrived contiguously from the start, in whatever order it came.
let mut sorted: Vec<&(u64, Vec<u8>)> = fragments.iter().collect();
sorted.sort_by_key(|(offset, _)| *offset);
let mut hello: Vec<u8> = Vec::new();
for (offset, data) in sorted {
let offset = usize::try_from(*offset).ok()?;
if offset > hello.len() {
break;
}
let overlap = hello.len() - offset;
if overlap < data.len() {
hello.extend_from_slice(&data[overlap..]);
}
}
if hello.len() < 4 || hello[0] != 0x01 {
return None;
}
let len = (usize::from(hello[1]) << 16) | (usize::from(hello[2]) << 8) | usize::from(hello[3]);
let body = hello.get(4..4 + len)?;
Some(alpn_of(body).unwrap_or_default())
}
fn alpn_of(body: &[u8]) -> Option<Vec<Vec<u8>>> {
let u16_at = |pos: usize| -> Option<usize> {
Some(usize::from(u16::from_be_bytes(body.get(pos..pos + 2)?.try_into().ok()?)))
};
// Legacy version and random, then the session id, the suites and the compressions.
let mut pos = 2 + 32;
pos += 1 + usize::from(*body.get(pos)?);
pos += 2 + u16_at(pos)?;
pos += 1 + usize::from(*body.get(pos)?);
let end = pos.checked_add(2 + u16_at(pos)?)?;
pos += 2;
while pos + 4 <= end {
let kind = u16_at(pos)?;
let len = u16_at(pos + 2)?;
let data = body.get(pos + 4..pos + 4 + len)?;
pos += 4 + len;
if kind != 16 {
continue;
}
let mut names = Vec::new();
let mut at = 2;
while at < data.len() {
let name_len = usize::from(data[at]);
names.push(data.get(at + 1..at + 1 + name_len)?.to_vec());
at += 1 + name_len;
}
return Some(names);
}
Some(Vec::new())
}
#[cfg(test)]
mod tests {
use super::*;
fn push_varint(out: &mut Vec<u8>, value: u64) {
if value < 64 {
out.push(value as u8);
} else if value < 16384 {
out.extend_from_slice(&(0x4000 | value as u16).to_be_bytes());
} else {
out.extend_from_slice(&(0x8000_0000 | value as u32).to_be_bytes());
}
}
/// A ClientHello offering `alpn`, and nothing else a parser would need.
fn hello_with(alpn: &[&[u8]]) -> Vec<u8> {
let mut names = Vec::new();
for name in alpn {
names.push(name.len() as u8);
names.extend_from_slice(name);
}
let mut ext = vec![0x00, 0x10];
ext.extend_from_slice(&((names.len() + 2) as u16).to_be_bytes());
ext.extend_from_slice(&(names.len() as u16).to_be_bytes());
ext.extend_from_slice(&names);
let mut body = vec![0x03, 0x03];
body.extend_from_slice(&[0u8; 32]);
body.push(0); // no session id
body.extend_from_slice(&[0x00, 0x02, 0x13, 0x01]); // one suite
body.extend_from_slice(&[0x01, 0x00]); // no compression
body.extend_from_slice(&(ext.len() as u16).to_be_bytes());
body.extend_from_slice(&ext);
let mut hello = vec![0x01];
hello.extend_from_slice(&(body.len() as u32).to_be_bytes()[1..]);
hello.extend_from_slice(&body);
hello
}
/// A client Initial carrying these CRYPTO fragments, protected the way a client
/// protects one — with rustls's own client-side keys, so what is read back is checked
/// against the other half of the library rather than against itself.
fn client_initial(dcid: &[u8], number: u64, crypto: &[(u64, &[u8])]) -> Vec<u8> {
let keys = Hellos::new().suite.keys(dcid, Side::Client, Version::V1);
let mut payload = Vec::new();
for (offset, data) in crypto {
payload.push(0x06);
push_varint(&mut payload, *offset);
push_varint(&mut payload, data.len() as u64);
payload.extend_from_slice(data);
}
payload.push(0x01); // a PING, as Chrome scatters among the fragments
payload.resize(payload.len().max(1100), 0); // PADDING, to the size a client sends
let mut packet = vec![0xc1, 0, 0, 0, 1, dcid.len() as u8];
packet.extend_from_slice(dcid);
packet.extend_from_slice(&[0, 0]); // no source id, no token
packet.extend_from_slice(&(0x4000 | (2 + payload.len() + 16) as u16).to_be_bytes());
let pn_offset = packet.len();
packet.extend_from_slice(&(number as u16).to_be_bytes());
let tag = keys.local.packet.encrypt_in_place(number, &packet, &mut payload).unwrap();
packet.extend_from_slice(&payload);
packet.extend_from_slice(tag.as_ref());
let sample = packet[pn_offset + 4..pn_offset + 20].to_vec();
let (first, rest) = packet.split_at_mut(1);
keys.local
.header
.encrypt_in_place(&sample, &mut first[0], &mut rest[pn_offset - 1..pn_offset + 1])
.unwrap();
packet
}
fn from() -> SocketAddr {
"192.0.2.1:4433".parse().unwrap()
}
#[test]
fn a_protected_initial_is_read_for_what_it_offers() {
let hellos = Hellos::new();
let hello = hello_with(&[b"h3", b"fgex-test"]);
hellos.observe(from(), &client_initial(b"\x83\x94\xc8\xf0\x3e\x51\x57\x08", 0, &[(0, &hello)]));
assert_eq!(hellos.take(from()), Some(vec![b"h3".to_vec(), b"fgex-test".to_vec()]));
}
/// Post-quantum key shares make a ClientHello bigger than one packet, and clients
/// scatter its pieces: two Initials, the second one first.
#[test]
fn a_hello_across_two_packets_arriving_backwards_is_reassembled() {
let hellos = Hellos::new();
let hello = hello_with(&[b"doq"]);
let (a, b) = hello.split_at(30);
let dcid = b"\x01\x02\x03\x04\x05\x06\x07\x08";
hellos.observe(from(), &client_initial(dcid, 1, &[(30, b)]));
assert_eq!(hellos.take(from()), None, "half a hello was read as a whole one");
hellos.observe(from(), &client_initial(dcid, 0, &[(0, a)]));
assert_eq!(hellos.take(from()), Some(vec![b"doq".to_vec()]));
}
/// A Retry answered is a new Initial under a new connection ID, and what was gathered
/// under the old one is not mixed into it.
#[test]
fn a_new_connection_id_starts_again() {
let hellos = Hellos::new();
let old = hello_with(&[b"old"]);
let new = hello_with(&[b"new"]);
hellos.observe(from(), &client_initial(b"\x11\x11\x11\x11\x11\x11\x11\x11", 0, &[(0, &old[..20])]));
hellos.observe(from(), &client_initial(b"\x22\x22\x22\x22\x22\x22\x22\x22", 0, &[(0, &new)]));
assert_eq!(hellos.take(from()), Some(vec![b"new".to_vec()]));
}
#[test]
fn anything_but_an_initial_is_passed_over() {
let hellos = Hellos::new();
hellos.observe(from(), &[0x40, 1, 2, 3, 4, 5, 6, 7]);
hellos.observe(from(), &[0xe0, 0, 0, 0, 1, 0, 0]);
let mut tampered = client_initial(b"\x09\x09\x09\x09\x09\x09\x09\x09", 0, &[(0, &hello_with(&[b"h3"]))]);
let last = tampered.len() - 1;
tampered[last] ^= 0xff;
hellos.observe(from(), &tampered);
assert_eq!(hellos.take(from()), None);
}
#[tokio::test]
async fn a_hello_still_arriving_is_waited_for() {
let hellos = Arc::new(Hellos::new());
let hello = hello_with(&[b"h3"]);
let packet = client_initial(b"\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11", 0, &[(0, &hello)]);
let later = Arc::clone(&hellos);
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(50)).await;
later.observe(from(), &packet);
});
assert_eq!(hellos.offered(from(), Duration::from_secs(2)).await, Some(vec![b"h3".to_vec()]));
assert_eq!(hellos.offered(from(), Duration::from_millis(50)).await, None);
}
}

84
backend/proxysrc/src/report.rs Normal file
View File

@ -0,0 +1,84 @@
//! Every line this engine sends the backend, written by a thread of its own.
//!
//! stdout is a pipe the backend reads, and a pipe fills. Each refused connection used to
//! be a `println!("BLOCKED …")` from the thread that refused it — so when the backend fell
//! behind, which a flood of refused connections makes it do, the pipe filled and those
//! writes blocked: the runtime threads carrying every other connection sat waiting on it.
//! Measured, 5000 refused connections took 74 seconds to get through, and a legitimate
//! client meanwhile waited a median 4.5 seconds and half the time not at all — an attacker
//! slowing a service to a halt by sending it exactly what firegex was there to refuse.
//!
//! So nothing on the datapath writes to stdout any more. Lines go into a bounded queue and
//! a thread of their own writes them out; a report of what happened (`BLOCKED`, `STATS`)
//! is dropped and counted when the queue is full, and a reply the backend is waiting for
//! (`PORT`, `UDP`, `ACK`) waits for room, so it is never lost and never overtakes a line
//! sent before it.
use std::io::Write;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::mpsc::{sync_channel, SyncSender, TrySendError};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
/// How many lines may wait for the backend. At a few dozen bytes each this is a megabyte;
/// what it buys is that a burst of refusals is reported whole rather than cut off.
const QUEUE: usize = 32 * 1024;
/// How often a count of reports that had to be dropped is written out.
const DROPS_SAID_EVERY: Duration = Duration::from_secs(5);
static SENDER: OnceLock<SyncSender<String>> = OnceLock::new();
static DROPPED: AtomicU64 = AtomicU64::new(0);
fn sender() -> &'static SyncSender<String> {
SENDER.get_or_init(|| {
let (tx, rx) = sync_channel::<String>(QUEUE);
std::thread::Builder::new()
.name("fgex-report".to_string())
.spawn(move || {
let stdout = std::io::stdout();
let mut said = Instant::now();
while let Ok(first) = rx.recv() {
let mut out = stdout.lock();
let _ = writeln!(out, "{first}");
// Whatever else is already waiting goes in the same write.
while let Ok(line) = rx.try_recv() {
let _ = writeln!(out, "{line}");
}
let _ = out.flush();
drop(out);
if said.elapsed() >= DROPS_SAID_EVERY {
let dropped = DROPPED.swap(0, Ordering::Relaxed);
if dropped > 0 {
eprintln!(
"[warn] [report] {dropped} refusals were not reported to the \
backend: it was not reading fast enough. They were refused all \
the same; only the counters and the log are short of them."
);
}
said = Instant::now();
}
}
})
.expect("cannot start the thread that talks to the backend");
tx
})
}
/// A reply the backend is waiting for. Waits for room rather than being dropped.
pub fn reply(line: String) {
let _ = sender().send(line);
}
/// A report of something that happened. Never waits: dropped and counted when the backend
/// is too far behind, so the datapath is never held up by it.
pub fn event(line: String) {
if let Err(TrySendError::Full(_)) = sender().try_send(line) {
DROPPED.fetch_add(1, Ordering::Relaxed);
}
}
/// A rule refused a connection.
pub fn blocked(id: &str) {
event(format!("BLOCKED {id}"));
}

37
backend/proxysrc/src/rules.rs
View File

@ -199,6 +199,21 @@ impl FilterSession for HyperscanSession {
/// All or nothing: a single bad rule rejects the update and leaves the running chain
/// untouched, because half a ruleset is worse than the previous one.
pub fn parse_ruleset(line: &str) -> Result<Vec<Arc<dyn Filter>>, String> {
parse_ruleset_reusing(line, &[])
}
/// [`parse_ruleset`], carrying over any filter of `previous` that the new ruleset
/// describes exactly as it was (`Filter::reuse_key`).
///
/// A ruleset is sent whole on every edit, so without this a pattern changed in one
/// filter restarted the Python worker of another, and the new worker met every open
/// connection in the middle: its module globals were gone, its HTTP parser started on
/// the second half of whatever request was in flight, and the addresses it had been told
/// when each connection opened were never told again.
pub fn parse_ruleset_reusing(
line: &str,
previous: &[Arc<dyn Filter>],
) -> Result<Vec<Arc<dyn Filter>>, String> {
let specs: Vec<RuleSpec> =
serde_json::from_str(line).map_err(|e| format!("malformed ruleset: {e}"))?;
@ -306,13 +321,21 @@ pub fn parse_ruleset(line: &str) -> Result<Vec<Arc<dyn Filter>>, String> {
} else {
command
};
out.push(Arc::new(PyWorkerRule::new(
id,
command,
code_path,
enabled,
Duration::from_millis(timeout_ms),
)))
let deadline = Duration::from_millis(timeout_ms);
let key = PyWorkerRule::key_for(&id, &command, &code_path, &enabled, deadline);
let kept = key.as_deref().and_then(|key| {
previous
.iter()
.find(|filter| filter.reuse_key() == Some(key))
.cloned()
});
out.push(match kept {
Some(filter) => filter,
None => Arc::new(
PyWorkerRule::new(id, command, code_path, enabled, deadline)
.reusable_as(key),
),
})
}
}
}

7
backend/proxysrc/src/udp.rs
View File

@ -333,6 +333,13 @@ async fn replies(
loop {
let len = match upstream.recv(&mut buf).await {
Ok(len) => len,
// The service's port answered a datagram with ICMP "unreachable", which a
// connected socket reports once, on its next call: the service was down for a
// moment — being restarted, most often. Nothing about the flow is over. This
// used to end the task, which left the flow in the relay's table with nobody
// reading the service's answers, so a client that went on talking never heard
// back again while its datagrams kept the flow from ever going idle.
Err(e) if e.kind() == io::ErrorKind::ConnectionRefused => continue,
Err(e) => {
eprintln!("[info] [udp] flow for {client} ended: {e}");
return;

36
backend/proxysrc/tests/fail_open.rs
View File

@ -42,11 +42,16 @@ async fn spawn_echo() -> std::net::SocketAddr {
}
async fn spawn_proxy(filters: &str, deadline_ms: u64) -> (std::net::SocketAddr, ChainHandle) {
let upstream = spawn_echo().await;
let chain = ChainHandle::new(FilterChain::new(
spawn_proxy_with(FilterChain::new(
parse_filters(filters).unwrap(),
Duration::from_millis(deadline_ms),
));
))
.await
}
async fn spawn_proxy_with(chain: FilterChain) -> (std::net::SocketAddr, ChainHandle) {
let upstream = spawn_echo().await;
let chain = ChainHandle::new(chain);
let proxy = Proxy::bind(
ProxyConfig::fixed("127.0.0.1:0".parse().unwrap(), upstream),
chain.clone(),
@ -105,7 +110,12 @@ async fn panicking_filter_keeps_traffic_flowing() {
/// process-wide, and the harness runs tests in parallel within one process.
#[tokio::test]
async fn hanging_filter_fails_open_then_loses_its_say() {
let (addr, chain) = spawn_proxy("hang", 50).await;
let rearm = Duration::from_millis(1500);
let (addr, chain) = spawn_proxy_with(
FilterChain::new(parse_filters("hang").unwrap(), Duration::from_millis(50))
.rearming_after(rearm),
)
.await;
// The deadline, not the filter, decides when the chunk moves.
let started = Instant::now();
@ -139,8 +149,24 @@ async fn hanging_filter_fails_open_then_loses_its_say() {
"a disabled filter is still being called, so the thread leak is unbounded"
);
// Let the abandoned threads exit, otherwise dropping the runtime waits on them.
// Slow is not broken: after a pause it is asked again, rather than staying out until
// somebody next edits a rule. It used to stay out for the life of the chain — which
// for a Python filter meant a flood, the one moment it was wanted, switched it off.
tokio::time::sleep(rearm + Duration::from_millis(300)).await;
let answered = roundtrip(addr, b"again").await.unwrap();
let asked = live.stats.timeouts.load(Ordering::Relaxed) - leaked;
let back_in = live.disabled_filters().is_empty();
let bypassed = live.is_bypassed();
// Let the abandoned threads exit, otherwise dropping the runtime waits on them — and
// before asserting, so a failure below fails the test rather than hanging it.
fgex_proxy::spec::release_hangs();
assert_eq!(answered, b"again");
// Once per direction: the echo comes back through the chain as well.
assert_eq!(asked, 2, "the filter was not asked again after its pause");
assert!(back_in, "two misses after the pause took it out again");
assert!(!bypassed);
}
/// Failing open is only worth anything if the filters do something when healthy.

2
backend/proxysrc/tests/h3.rs
View File

@ -207,7 +207,7 @@ async fn spawn_relay_to(
Duration::from_millis(500),
));
let cfg = QuicConfig {
setup: Arc::new(QuicSetup::build(&cert, &key, vec![b"h3".to_vec()]).unwrap()),
setup: Arc::new(QuicSetup::build(&cert, &key).unwrap()),
self_mark: None,
spoof_source: false,
max_connections: 0,

72
backend/proxysrc/tests/pyworker.rs
View File

@ -16,7 +16,7 @@ use std::time::{Duration, Instant};
use fgex_proxy::filter::{
next_connection_id, ChainSessions, ConnectionMeta, Direction, FilterChain, Verdict, L4,
};
use fgex_proxy::rules::parse_ruleset;
use fgex_proxy::rules::{parse_ruleset, parse_ruleset_reusing};
/// Write a filter file and give back its path, kept for the process's lifetime.
fn filter_file(name: &str, body: &str) -> String {
@ -537,3 +537,73 @@ def refuse_b(packet: RawPacket):
assert_eq!(feed(&off, &[b"carrying AAA"]).await, vec![Verdict::Accept]);
assert_eq!(feed(&off, &[b"carrying BBB"]).await, vec![Verdict::Accept]);
}
/// An edit elsewhere in the chain must not restart a Python filter that did not change.
///
/// Every edit sends the whole ruleset, and a new worker meets every open connection
/// halfway: its module globals gone, its HTTP parser starting on the second half of the
/// request in flight. The same filter, described the same way, is carried over with the
/// state it keeps for each connection; new code is a new worker, as it has to be.
#[tokio::test]
async fn an_unchanged_worker_is_carried_into_the_next_ruleset_with_its_state() {
const REMEMBERS: &str = r#"from firegex.pyfilters import pyfilter, ACCEPT, REJECT
from firegex.pyfilters.models import RawPacket
# Module level, so it is per connection: the library gives each stream its own globals.
said_first = False
@pyfilter
def second_after_first(packet: RawPacket):
global said_first
if b"FIRST" in packet.data:
said_first = True
return REJECT if said_first and b"SECOND" in packet.data else ACCEPT
"#;
let path = filter_file("carried", REMEMBERS);
let json = format!(
r#"[{{"kind":"python","id":"py1","code_path":"{path}","timeout_ms":2000,
"command":["python3","{}"]}}]"#,
worker()
);
let deadline = Duration::from_secs(10);
let connection = next_connection_id();
let before = FilterChain::new(parse_ruleset(&json).unwrap(), deadline);
let mut sessions = ChainSessions::new(connection);
assert_eq!(
before.run(Direction::ClientToServer, b"FIRST", &mut sessions).await,
Verdict::Accept
);
// The same ruleset again, as an edit to some other filter would send it.
let after = FilterChain::new(
parse_ruleset_reusing(&json, &before.filters()).unwrap(),
deadline,
);
assert!(
std::sync::Arc::ptr_eq(&before.filters()[0], &after.filters()[0]),
"an unchanged Python filter was rebuilt"
);
let mut sessions = ChainSessions::new(connection);
assert_eq!(
after.run(Direction::ClientToServer, b"SECOND", &mut sessions).await,
Verdict::Reject(Some("py1/second_after_first".to_string())),
"the connection's state did not survive an edit that did not touch this filter"
);
// New code is a new worker, and starts every connection afresh.
filter_file("carried", &format!("{REMEMBERS}\n# edited\n"));
let edited = FilterChain::new(
parse_ruleset_reusing(&json, &after.filters()).unwrap(),
deadline,
);
assert!(
!std::sync::Arc::ptr_eq(&after.filters()[0], &edited.filters()[0]),
"edited code was not given a worker of its own"
);
let mut sessions = ChainSessions::new(connection);
assert_eq!(
edited.run(Direction::ClientToServer, b"SECOND", &mut sessions).await,
Verdict::Accept
);
}

54
backend/proxysrc/tests/quic.rs
View File

@ -79,14 +79,14 @@ async fn spawn_quic_echo(cert: &str, key: &str, speaks: &[&str]) -> SocketAddr {
}
/// The relay, in front of that service.
async fn spawn_relay(upstream: SocketAddr, filters: &str, offers: &[&str]) -> SocketAddr {
async fn spawn_relay(upstream: SocketAddr, filters: &str) -> SocketAddr {
let (cert, key) = self_signed();
let chain = ChainHandle::new(FilterChain::new(
parse_filters(filters).unwrap(),
Duration::from_millis(500),
));
let cfg = QuicConfig {
setup: Arc::new(QuicSetup::build(&cert, &key, alpn(offers)).unwrap()),
setup: Arc::new(QuicSetup::build(&cert, &key).unwrap()),
// No mark and no impersonation: `SO_MARK` and `IP_TRANSPARENT` both need
// CAP_NET_ADMIN, which a test run has no business requiring. What they change is
// which address the service sees, and nothing here is asking about that.
@ -188,7 +188,7 @@ async fn roundtrip(addr: SocketAddr, payload: &[u8], speaks: &[&str]) -> Exchang
async fn carries_a_stream_through() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "", &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "").await;
let exchange = roundtrip(addr, b"hello over quic", &["fgex-raw"]).await;
assert_eq!(exchange.echoed.unwrap(), b"hello over quic");
@ -200,7 +200,7 @@ async fn carries_a_stream_through() {
async fn rules_see_the_plaintext_inside_the_streams() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:FLAG{", &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:FLAG{").await;
let harmless = roundtrip(addr, b"harmless request", &["fgex-raw"]).await;
assert_eq!(harmless.echoed.unwrap(), b"harmless request");
@ -225,7 +225,7 @@ async fn rules_see_the_plaintext_inside_the_streams() {
async fn a_refusal_takes_the_whole_connection() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:FLAG{", &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:FLAG{").await;
let mut config = tls::quic_client_config().unwrap();
config.alpn_protocols = alpn(&["fgex-raw"]);
@ -249,31 +249,47 @@ async fn a_refusal_takes_the_whole_connection() {
);
}
/// The service picks the protocol, and the client is told that and nothing else.
/// The service picks from what the client offered, and the client is told that and
/// nothing else — the TLS path's mirroring, on QUIC.
///
/// This is the ordering that had to change from the TLS path: there the ClientHello is
/// held open and the service is asked what the *client* offered, which QUIC gives no
/// opportunity to do. So the relay offers its candidates to the service and passes the
/// answer on — and a service speaking something unusual is carried without firegex
/// having to guess.
/// The relay used to offer the service a list of its own, `h3` unless the operator named
/// others, because the client's hello was taken to be unreadable. It is read off the
/// Initial instead (`quic_hello`), so a service speaking something nobody configured is
/// carried as it is.
#[tokio::test]
async fn the_service_chooses_the_protocol() {
async fn the_service_chooses_from_what_the_client_offered() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-test"]).await;
let addr = spawn_relay(upstream, "", &["h3", "fgex-test"]).await;
let addr = spawn_relay(upstream, "").await;
let exchange = roundtrip(addr, b"ping", &["h3", "fgex-test"]).await;
assert_eq!(exchange.protocol.as_deref(), Some("fgex-test"));
assert_eq!(exchange.echoed.unwrap(), b"ping");
}
/// A client that cannot speak what the service chose is refused by the handshake, which
/// is the honest failure this ordering costs.
/// A hello too big for one packet — which post-quantum key shares make ordinary — is read
/// across the packets it came in.
#[tokio::test]
async fn a_hello_spanning_several_packets_is_read_whole() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-last"]).await;
let addr = spawn_relay(upstream, "").await;
let filler: Vec<String> = (0..40).map(|i| format!("fgex-unspoken-{i:02}-{}", "x".repeat(40))).collect();
let mut offers: Vec<&str> = filler.iter().map(String::as_str).collect();
offers.push("fgex-last");
let exchange = roundtrip(addr, b"ping", &offers).await;
assert_eq!(exchange.protocol.as_deref(), Some("fgex-last"));
assert_eq!(exchange.echoed.unwrap(), b"ping");
}
/// A client that speaks nothing the service does is refused by the service's handshake,
/// the way it would be with no firegex in the way.
#[tokio::test]
async fn a_client_that_speaks_something_else_is_refused() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-test"]).await;
let addr = spawn_relay(upstream, "", &["fgex-test"]).await;
let addr = spawn_relay(upstream, "").await;
let exchange = roundtrip(addr, b"ping", &["h3"]).await;
assert!(
@ -298,7 +314,7 @@ async fn connect(addr: SocketAddr, speaks: &[&str]) -> quinn::Connection {
async fn datagrams_are_carried_both_ways() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "", &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "").await;
let connection = connect(addr, &["fgex-raw"]).await;
assert!(
@ -324,7 +340,7 @@ async fn datagrams_are_carried_both_ways() {
async fn a_refused_datagram_is_dropped_and_the_connection_lives() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:forbidden", &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:forbidden").await;
let connection = connect(addr, &["fgex-raw"]).await;
connection
@ -361,7 +377,7 @@ async fn a_refused_datagram_is_dropped_and_the_connection_lives() {
async fn a_datagram_flow_keeps_its_filter_state() {
let (cert, key) = self_signed();
let upstream = spawn_quic_echo(&cert, &key, &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:secret", &["fgex-raw"]).await;
let addr = spawn_relay(upstream, "block:secret").await;
let connection = connect(addr, &["fgex-raw"]).await;
connection

55
backend/proxysrc/tests/udp.rs
View File

@ -130,6 +130,61 @@ async fn flows_spend_the_same_budget_as_connections() {
assert_eq!(stats.live.load(Ordering::Relaxed), 1, "a refused flow kept its place");
}
/// A service that goes away for a moment and comes back is answering again, for the
/// client that kept talking through it.
///
/// A datagram sent while it was down is answered with ICMP "port unreachable", which the
/// flow's connected socket reports as an error on its next call. That error ended the task
/// carrying the answers back while the flow stayed in the table — and a client that kept
/// sending kept it from ever going idle, so it never heard from the service again.
#[tokio::test]
async fn a_flow_outlives_its_service_restarting() {
let first = UdpSocket::bind("127.0.0.1:0").await.unwrap();
let upstream = first.local_addr().unwrap();
let echo = |socket: UdpSocket| {
tokio::spawn(async move {
let mut buf = vec![0u8; 2048];
loop {
let Ok((n, from)) = socket.recv_from(&mut buf).await else {
continue;
};
let _ = socket.send_to(&buf[..n], from).await;
}
})
};
let service = echo(first);
let relay = spawn_relay(upstream, "", 0, false, Arc::new(ProxyStats::default())).await;
// One source throughout: this is about a flow that already exists.
let client = UdpSocket::bind("127.0.0.1:0").await.unwrap();
assert_eq!(exchange(&client, relay, b"before").await.as_deref(), Some(&b"before"[..]));
service.abort();
let _ = service.await;
assert_eq!(exchange(&client, relay, b"while down").await, None);
echo(UdpSocket::bind(upstream).await.unwrap());
assert_eq!(
exchange(&client, relay, b"after").await.as_deref(),
Some(&b"after"[..]),
"the service came back and the flow that kept talking was not answered"
);
}
/// One datagram from `client`, and the answer if one comes.
async fn exchange(
client: &UdpSocket,
relay: std::net::SocketAddr,
payload: &[u8],
) -> Option<Vec<u8>> {
client.send_to(payload, relay).await.unwrap();
let mut buf = vec![0u8; 2048];
match tokio::time::timeout(Duration::from_millis(700), client.recv(&mut buf)).await {
Ok(Ok(n)) => Some(buf[..n].to_vec()),
_ => None,
}
}
/// Flows are counted in the unit the backend divides by: flows seen, flows refused.
///
/// UDP counted neither, so a service refusing datagrams all day reported no connections

1
backend/routers/firewall.py
View File

@ -50,6 +50,7 @@ async def startup():
await firewall.init()
async def shutdown():
firewall.stop_watching()
if not firewall.keep_rules:
await firewall.close()
db.disconnect()

77
backend/routers/services.py
View File

@ -21,7 +21,7 @@ import sys
import time
import traceback
from fastapi import APIRouter, HTTPException
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import PlainTextResponse
from pydantic import BaseModel
@ -35,7 +35,7 @@ from modules.services.firewall import (
from modules.services import stats
from modules.services.logs import log_for
from modules.services.models import (KIND, L4, MODE, PROTO, STATUS, Service,
TRANSPORT, UPSTREAM, upstream_refusal)
TRANSPORT, UPSTREAM)
from modules.services.nftables import FiregexTables, hijack_endpoint
from modules.services import transports
from modules.services.transports import PROXY_ENGINE, PYWORKER, UnsupportedChain
@ -50,7 +50,17 @@ from utils.models import ResetRequest, StatusMessageModel
from utils.sqlite import SQLite
app = APIRouter()
def _counters_written() -> None:
"""Every request sees the block counters as they are, and changes them from there.
They are written once a second (`BLOCK_FLUSH` in the manager), not per block — so a
read would otherwise be up to a second behind, and an edit that resets a pattern's
counter would have the old pattern's last blocks written onto the new one after it.
"""
firewall.flush_blocks()
app = APIRouter(dependencies=[Depends(_counters_written)])
db = SQLite(
"db/services.db",
@ -262,6 +272,15 @@ class AddressForm(BaseModel):
proxy_port: PortType | None = None
class ProblemModel(BaseModel):
"""The newest warning or error in a service's log. See `ServiceLog.last_problem`."""
#: Unix milliseconds, like every log line.
at: int
level: str
text: str
class ServiceModel(BaseModel):
service_id: str
name: str
@ -288,6 +307,9 @@ class ServiceModel(BaseModel):
addresses: list[AddressModel] = []
n_filters: int
n_blocked: int
#: The newest warning or error still in its log, so the list can say something is
#: wrong without anybody having the service's page open. Cleared with the log.
problem: ProblemModel | None = None
class ServiceAddForm(BaseModel):
@ -534,6 +556,9 @@ async def shutdown():
async def reset(params: ResetRequest):
if not params.delete:
# Into the copy that is put back, or the last second of counters is written
# after it was taken and then overwritten by it.
firewall.flush_blocks()
db.backup()
await firewall.close()
FiregexTables().reset()
@ -709,6 +734,7 @@ def _with_addresses(row: dict) -> dict:
row["addresses"] = [
_address_row(a) for a in _addresses(row["service_id"])
]
row["problem"] = log_for(row["service_id"]).last_problem()
return row
@ -1052,14 +1078,7 @@ async def add_service(form: ServiceAddForm):
# nobody asked for.
for address in form.addresses:
_address_target(form.transport, address)
edge = _address_edge(form.proto, address)
refusal = upstream_refusal(
form.proto,
_address_upstream(form.proto, edge, address),
L4.l4_of(edge) == L4.UDP,
)
if refusal:
raise HTTPException(status_code=400, detail=refusal)
_address_upstream(form.proto, _address_edge(form.proto, address), address)
# The transport gets a look before the row exists. A chain it could not host is
# caught later, when there is one; what is caught here is what the *service* makes
# impossible on its own — a protocol this layer cannot carry — and it is worth
@ -1254,6 +1273,11 @@ async def edit_service(service_id: str, form: ServiceSettingsForm):
fields["tls_cert"] = form.tls_cert
if form.tls_key is not None:
fields["tls_key"] = form.tls_key
# Only what actually changes. The form sends every setting on every save, and a field
# that already holds the value it is sent is not an edit: counted as one, opening the
# dialog and saving it untouched restarted a running service and dropped every
# connection it was carrying.
fields = {key: value for key, value in fields.items() if row[key] != value}
if not fields:
return {"status": "ok"}
@ -1275,12 +1299,6 @@ async def edit_service(service_id: str, form: ServiceSettingsForm):
+ ", ".join(f"{_shown(a['ip_int'])}:{a['port']}" for a in published)
+ " is published on another port. Take that off the address first.",
)
for address in new_addresses:
refusal = upstream_refusal(proto, address["upstream"],
L4.l4_of(address["edge"]) == L4.UDP)
if refusal:
raise HTTPException(status_code=400, detail=refusal)
# The service it would become, asked what a start would ask of the layer and the
# chain. Without its addresses, the way a service is checked when it is created: what
# an address still lacks on the new layer — the port of the operator's own proxy, say
@ -1329,6 +1347,13 @@ async def edit_service(service_id: str, form: ServiceSettingsForm):
detail=f"Another service already protects one of these addresses on {proto}",
)
if set(fields) == {"name"}:
# A name is the operator's, not the datapath's: nothing running reads it, so it
# is not a reason to take the service down and bring it back.
manager.srv.name = fields["name"]
await refresh_frontend()
return {"status": "ok"}
was_active = manager.active
try:
await manager.refresh(load_service(service_id))
@ -1383,17 +1408,8 @@ async def add_address(service_id: str, form: AddressForm):
detail="The external transport hands traffic to your own proxy and rewrites the "
"source address on return, which requires a concrete IP address rather than an interface.",
)
# Before the row exists, because this is the address's own answer and it can be one
# this instance cannot honour. `ProxyTransport.check` says the same thing at start,
# and would leave the address behind to be undone.
edge = _address_edge(row["proto"], form)
refusal = upstream_refusal(
row["proto"],
_address_upstream(row["proto"], edge, form),
L4.l4_of(edge) == L4.UDP,
)
if refusal:
raise HTTPException(status_code=400, detail=refusal)
# Before the row exists: what the address says has to mean something on this service.
_address_upstream(row["proto"], _address_edge(row["proto"], form), form)
try:
address_id = _insert_address(service_id, row["proto"], row["transport"], form)
except sqlite3.IntegrityError as e:
@ -1469,9 +1485,6 @@ async def edit_address(service_id: str, address_id: str, form: AddressForm):
form.upstream = UPSTREAM.SAME
target_port = _address_target(row["transport"], form)
upstream = _address_upstream(row["proto"], edge, form)
refusal = upstream_refusal(row["proto"], upstream, L4.l4_of(edge) == L4.UDP)
if refusal:
raise HTTPException(status_code=400, detail=refusal)
def write(ip_int, port, proto, edge, target_port, upstream, proxy_ip, proxy_port):
db.query(
@ -2356,6 +2369,8 @@ async def get_logs(service_id: str):
async def clear_logs(service_id: str):
_service_or_404(service_id)
log_for(service_id).clear()
# The list shows the newest problem in the log, and clearing is how it is dismissed.
await refresh_frontend()
return {"status": "ok"}

18
backend/utils/__init__.py
View File

@ -311,15 +311,27 @@ class NFTableManager(Singleton):
def reset(self):
self.raw_cmd(*self.__reset_cmds)
def list_rules(self, tables = None, chains = None):
for filter in [ele["rule"] for ele in self.raw_list() if "rule" in ele ]:
def list_rules(self, tables = None, chains = None, family: str | None = None):
for filter in [ele["rule"] for ele in self.raw_list(tables, family) if "rule" in ele ]:
if tables and filter["table"] not in tables:
continue
if chains and filter["chain"] not in chains:
continue
yield filter
def raw_list(self):
def raw_list(self, tables = None, family: str | None = None):
# One table of ours when that is all that is asked for, rather than the host's
# whole ruleset: on a node running Docker or Kubernetes that is thousands of rules
# of somebody else's, serialised and parsed on every lookup — and a lookup runs
# inside the backend's one event loop, on every start, stop, address change and
# statistics request. A table that is not there is simply no rules.
if tables and family:
out = []
for table in tables:
code, listed, _ = self.raw_cmd({"list": {"table": {"family": family, "name": table}}})
if code == 0:
out.extend(listed["nftables"])
return out
return self.cmd({"list": {"ruleset": None}})["nftables"]
def _json_like(obj: BaseModel|List[BaseModel], unset=False, convert_keys:dict[str, str]=None, exclude:list[str]=None, mode:str="json"):

4
docs/firewall.md
View File

@ -27,7 +27,7 @@ Beyond the rule list and policy, a few toggles affect the whole module:
- **allow_dhcp**: always accept DHCP traffic.
- **multicast_dns**: always accept mDNS (multicast DNS) traffic.
- **allow_upnp**: always accept UPnP traffic.
- **allow_dnat** (on by default): forwarded traffic whose destination another NAT rewrote — a port **Docker or podman published** for a container, a router's port forward — is left to the rules that published it when none of *your* rules matched it, instead of meeting the forward policy. Your `forward` rules still apply to it first, so a rule that drops it drops it.
- **allow_dnat** (on by default, "Leave container traffic to the container runtime"): forwarded traffic whose destination another NAT rewrote — a port **Docker or podman published** for a container, a router's port forward — and traffic **the containers send**, to each other or out (anything arriving from `docker0`, `br-*`, `docker_gwbridge`, `podman*` or `cni-podman*`), is left to the container runtime's own rules when none of *yours* matched it, instead of meeting the forward policy. Without it, a drop policy cuts a web container off from its database on the same bridge. Your `forward` rules still apply to it first, so a rule that drops it drops it.
Each of the others, when enabled, inserts a small accept rule ahead of your own rules — they're conveniences for common cases you'd otherwise have to write by hand. **allow_dnat** is the exception: it sits after your rules, because it only decides what the *policy* does with what they did not match.
@ -48,6 +48,8 @@ Two things follow from Firegex owning those tables rather than writing into the
- **`iptables` keeps working.** Earlier versions put their chains into the tables named `filter` and `mangle`, which is where `iptables-nft` — the default `iptables` on most current distributions — keeps its own. `iptables` rejects a whole table as soon as it contains a rule it cannot express in its own format, and the connection-tracking rules Firegex writes for **allow_established** and **drop_invalid** are exactly that. The result was `iptables -L` and `iptables-save` answering ``table `filter' is incompatible, use 'nft' tool`` on a host where Docker, ufw, fail2ban or the organisers' own scripts were the ones asking. That cannot happen now: `iptables` never looks at a table Firegex owns.
- **Turning the firewall off no longer touches the host's own.** The default policy is a property of a base chain, and it used to be *your* `INPUT` chain that Firegex set to `drop` — and set back to `accept` on shutdown, silently undoing a default-deny an administrator had configured elsewhere. Firegex now sets the policy on its own base chain, so a reset is just deleting two tables.
**If something else removes those tables, Firegex puts them back within a few seconds.** `nft -f /etc/nftables.conf` starts with `flush ruleset` on Debian, and so does `systemctl restart nftables`; either one used to switch the Firegex firewall off with nothing to say so. The same watch covers the services' table: each running service is steered at its filters again without restarting, and its log records the gap as an error, since traffic was not filtered while it lasted. The lasting fix is still to make whatever manages your nftables leave the `fgex` tables alone. With `flush ruleset`, for example, you can delete only your own tables instead.
If you are upgrading from a version that used the shared tables, whatever it left behind is still there — Firegex does not go looking in tables it no longer owns. Clear it by hand once, and check the policy while you are there, since an older Firegex was the one driving it:
```bash

2
docs/pyfilter.md
View File

@ -404,7 +404,7 @@ compressed WebSocket message is bounded the same way.
from firegex.pyfilters import ExceptionAction
```
- `FGEX_INVALID_ENCODING_ACTION: ExceptionAction` — action taken when parsing hits an invalid/unsupported encoding (a parser-level failure). Default: `ExceptionAction.REJECT`. Values: `ACCEPT` (accept the packet that caused the error), `DROP` (drop the connection), `REJECT` (reject the connection), `NOACTION` (do nothing — the error is signaled and the stream is accepted without calling any more pyfilters on it).
- `FGEX_INVALID_ENCODING_ACTION: ExceptionAction` — action taken when parsing hits an invalid/unsupported encoding (a parser-level failure). Default: `ExceptionAction.ACCEPT` — the traffic is carried without the HTTP filters seeing it, and the service log says so once per connection; `REJECT` is the stricter choice against request smuggling, and yours to make. Values: `ACCEPT` (accept the packet that caused the error), `DROP` (drop the connection), `REJECT` (reject the connection), `NOACTION` (do nothing — the error is signaled and the stream is accepted without calling any more pyfilters on it). One case never reaches this setting: a connection the filter took over already open — it started, was rebuilt or got new code while the connection was running — whose first bytes it saw were the middle of a message, such as the rest of an upload. That is not malformed traffic, so that direction of the connection is carried without its HTTP filters until it closes, and the log says so once, as information. A connection taken over between two messages is filtered as usual.
- `FGEX_MAX_HISTORY_SIZE: int` — max number of requests/responses kept per stream by [`HttpHistory`](#httphistory-alias-httpstreamhistory--http-only). Default: `100`. The entries of one direction together are also held under `FGEX_STREAM_MAX_SIZE` bytes.
## Testing a filter locally

105
docs/services.md
View File

@ -162,9 +162,9 @@ allowed to do to the traffic. Pick by those.
| | **Proxy** | **NFQUEUE** |
|---|---|---|
| Unit of work | a connection | a packet |
| Transparency | full — dials your service **from the client's own address** | full — the real packets, on their way to your service |
| Transparency | dials your service **from the client's own address** (not its source port) | full — the real packets, on their way to your service |
| Reassembly | the kernel's, on two real sockets | rebuilt in userspace with libtins |
| Rewriting | patterns: no. Python: exact, any length | patterns: no. Python: unstable on TCP, exact on UDP |
| Rewriting | none — a filter answers with a verdict | none — a filter answers with a verdict |
| If a filter dies | the engine rebuilds fail-open by hand | **the kernel keeps forwarding**, by itself |
| Cost | terminating a connection, once | **a userspace round trip per packet**, per filter |
| Measured ([how](../tests/bench/README.md#performance)) | **4035 MB/s** at 1 thread, **13 984** at 8 | 1820 at 1 thread, 2956 at 8 |
@ -207,14 +207,28 @@ halves.
- **The kernel's fail-open backstop is gone**, and the engine has to rebuild it by hand:
`catch_unwind`, a deadline per filter, a filter that misbehaves losing its say rather
than the traffic being held. It works — and it is code, where the other layer has a
kernel guarantee.
than the traffic being held. One that keeps missing its deadline — a Python filter
under a flood, usually — is asked again after half a minute rather than staying out;
one that crashed stays out until you next save a change to the filters. It works —
and it is code, where the other layer has a kernel guarantee.
It stays invisible: it always dials your service **from the client's own address** on both
TCP and UDP, so anything that logs, rate-limits or bans by IP keeps working. That is not
a setting, because a service that suddenly saw one address for the whole internet would be
a regression nobody would attribute to us.
What it cannot keep is the client's **source port**: the connection to your service is a
new one, from the client's address and a port the kernel picks. And a reset from either
end reaches the other as an ordinary close.
**A service in a container is the ordinary case.** Protect the port the container is
published on (`docker run -p 8080:80` → this host's address, port `8080`), on either layer.
Firegex's rules run before the container runtime rewrites the destination, so it sees the
traffic first; the service is reached, filtered and shown the real client address whether
the client is on another host or on this one. Both were measured with a container behind
Docker's own rewrite, and `tests/integration/test_remote_clients.py` asks it of every
release with a stand-in that needs no Docker.
### NFQUEUE
Packets are lifted to userspace, inspected, and a verdict is handed back to the kernel.
@ -228,6 +242,11 @@ Nothing is terminated.
timestamps intact.
- **No connection termination.** Suitable for protocols that cannot be proxied or where
terminating the transport is prohibited.
- **Connections already open are protected too.** Starting a service, adding a filter or
rebuilding the chain inspects the connections that were open at that moment from their
next packet on. The proxy layer cannot do this: a connection made before its redirect
existed was never sent to it, and goes on straight to the service, unfiltered but
uninterrupted, until it closes.
**What it costs**
@ -427,10 +446,10 @@ its own connection to the service — dialled from the client's own address like
dial firegex makes.
What cannot be done, and is a limit rather than a missing setting: a QUIC edge carrying
anything **other than HTTP/3**. Opaque bytes on a QUIC stream have no HTTP/1.1 form, so
there is nothing to send a service that speaks one. In practice that means an instance
whose `FGEX_PROXY_QUIC_ALPN` is not `h3`: the choice is refused there, on `QUIC` and
`HTTPS` services alike, with the ALPN named.
anything **other than HTTP/3** to a service that speaks HTTP/1.1. Opaque bytes on a QUIC
stream have no HTTP/1.1 form, so there is nothing to send it. On such an address firegex
answers the client itself, in HTTP/3, and a client offering only something else is
refused by that handshake.
#### What is carried through, and what cannot be
@ -613,31 +632,16 @@ bytes are real and the framing is reconstructed.** Concretely —
#### What is carried through, and what cannot be
**ALPN is the service's answer, carried — asked in the other order.** On TLS over TCP the
client's hello is held open, the service is asked with exactly the list the client
offered, and the client is told what came back. In QUIC the hello arrives inside an
encrypted Initial packet whose processing *is* the handshake, so there is nothing to hold
it at. The order is therefore reversed: firegex opens its connection to the service
first, offering the protocols it was told to offer, and tells the client the one thing
the service agreed to. The invariant that matters is unchanged — the client is never told
a protocol the service did not choose — and what is lost is knowing in advance whether
the client would have accepted it. When it would not, its handshake fails and the log
says which protocol the service picked.
That list is `h3`, which is what a QUIC service speaks nine times in ten. An instance in
front of something else sets `FGEX_PROXY_QUIC_ALPN` as a comma-separated list in
preference order:
```bash
python3 run.py restart --env FGEX_PROXY_QUIC_ALPN=h3,doq
```
`run.py` stores it and puts it back on every later start — editing the generated compose
file by hand does not survive one, because `run.py` rewrites that file each time. The
list is **per instance, not per service**, and it is a superset rather than a choice:
every QUIC service is offered all of it and each one picks what it actually speaks, so
listing every protocol the instance carries is the way to run more than one kind of QUIC
service at once.
**ALPN is the service's answer, carried — and there is nothing to configure.** On TLS
over TCP the client's hello is held open, the service is asked with exactly the list the
client offered, and the client is told what came back. QUIC does the same. Its hello
travels inside an Initial packet, but Initial packets are protected with keys anyone can
derive from the packet itself — on purpose, so something in the path can read exactly
this — and firegex reads the client's list off it before answering. Your service is then
offered precisely what the client offered, and the client is told the one protocol your
service chose. HTTP/3, DNS over QUIC, a game's protocol or a CTF's own all work the same
way, with no list to keep in step. If a hello cannot be read (a QUIC version firegex does
not know), `h3` is offered instead and the service log says so.
**0-RTT is not offered.** Early data is replayable by anyone who watched it go past, and
a filter that refused a request has no way to un-deliver the copy your service already
@ -959,6 +963,12 @@ as it happens rather than polled:
missing its deadline, a fallback, a ruleset that was refused. These used to go to the
backend's stderr, where nobody looks during a competition.
Warnings and errors do not wait for you to open the service: each one also pops up as a
notification wherever you are in the interface (one per service every twenty seconds,
the rest counted into it — except that an error always gets through a warning), and the service's row in the list carries a **WARNING** or
**ERROR** badge for an hour, with the line itself on hover. Clearing the service's log
dismisses it.
It is bounded at both ends. A service under attack refuses thousands of connections a
minute; the tail keeps the last few hundred lines and the rest fall off, and lines are
coalesced before being sent so a burst arrives as one message instead of thousands.
@ -1003,15 +1013,34 @@ would.
Adding, removing, reordering, enabling and disabling filters, patterns and individual
`@pyfilter` functions all take effect on live connections **without dropping any of
them**. Both layers swap their configuration
underneath the traffic.
them**. Both layers swap their configuration underneath the traffic. The exception is
adding, removing or reordering a whole filter on the NFQUEUE layer, where each filter
is a process of its own: there the chain of processes is rebuilt, traffic passes
unfiltered for the moment that takes, and the new processes take over the connections
already open from their next packet.
An edit reaches only the filter it was made to. A Python filter keeps what it knows about
each open connection — its module globals, and how far into a request it has read —
through edits to the other filters; its own new code starts each connection afresh. A
filter that takes over a connection in the middle of a message (a request half uploaded)
does not judge the half it sees: HTTP filters leave that direction alone until the
connection closes, rather than refusing a valid request or calling it malformed.
Adding an address does not drop anything either, and removing or moving one costs only
the connections on that address.
Changing the service's own definition — its protocol, network layer, or TLS — is
different: that changes what the datapath *is*, so the service is stopped and started
around it, and the connections it was carrying are dropped. The UI says so before you do it.
Changing the service's own definition — its protocol, network layer, TLS or limits —
changes what the datapath *is*, so the service is restarted around it. The connections it
was carrying are not cut: new ones get the new settings, and the ones already open go on.
Stopping a service does not cut them either. On the NFQUEUE layer the kernel simply stops
asking the filters. The proxy layer terminates the connections it carries, so its engine
is not killed: it is taken out of the rules at once, and new connections go straight to
the service. It carries the ones it already had, unfiltered, until they close, and then
exits. After a restart the old engine does the same, and keeps filtering those connections
the way it did before. Either way it gives up after ten minutes, closing whatever is still
open, and the service's log says so. Only shutting firegex itself down cuts them
straight away.
## See also

7
fgex-lib/firegex/pyfilters/internals/__init__.py
View File

@ -182,11 +182,14 @@ def get_code_proto(code:str) -> str:
#: What a model raises while being built, and the verdict it means. `NotReadyToRun` is
#: not here: it is the ordinary "there is nothing to give this filter yet", handled by
#: skipping the call rather than by ending the packet.
#: Each is credited to what actually decided it, never to the filter function: those two
#: are raised only when the parser could not read the traffic, and a function the
#: operator wrote was listed as having refused a connection it was never shown.
_RAISED_VERDICT = {
StreamFullDrop: (Action.DROP, "@MAX_STREAM_SIZE_REACHED"),
StreamFullReject: (Action.REJECT, "@MAX_STREAM_SIZE_REACHED"),
DropPacket: (Action.DROP, None),
RejectConnection: (Action.REJECT, None),
DropPacket: (Action.DROP, "@INVALID_ENCODING"),
RejectConnection: (Action.REJECT, "@INVALID_ENCODING"),
}

8
fgex-lib/firegex/pyfilters/internals/data.py
View File

@ -210,7 +210,13 @@ class DataStreamCtx:
filter_call_info = _Kept(list)
stream_max_size = _Kept(DEFAULT_STREAM_MAX_SIZE, int)
full_stream_action = _Kept(FullStreamAction.FLUSH, FullStreamAction)
invalid_encoding_action = _Kept(ExceptionAction.REJECT, ExceptionAction)
#: Traffic the HTTP parser cannot read is carried, and said so, unless the filter file
#: asks for it to be refused. It was refused by default — so a request the service
#: would have taken and llhttp would not (a quirk of one client, a protocol that only
#: resembles HTTP) closed the connection with nothing chosen by anybody, and the block
#: was credited to the filter function as if it had decided it. Refusing is the
#: stricter answer to request smuggling and stays one line away; it is the operator's.
invalid_encoding_action = _Kept(ExceptionAction.ACCEPT, ExceptionAction)
data_handler_context = _Kept(dict)
def __init__(self, glob: dict, init_pkt: bool = True):

71
fgex-lib/firegex/pyfilters/models/http.py
View File

@ -21,6 +21,26 @@ from websockets.extensions.permessage_deflate import PerMessageDeflate
from firegex._llhttp import PAUSED_H2_UPGRADE, PAUSED_UPGRADE
#: Whether this process has said that it met connections in the middle of a message.
#: Once per process rather than per connection: a filter taking over from another meets
#: every connection that was open at that moment, and a line each would bury the log.
_told_met_mid_message = False
def _met_mid_message() -> None:
global _told_met_mid_message
if _told_met_mid_message:
return
_told_met_mid_message = True
print(
"[info] [http] connections already open when this filter took over were met in "
"the middle of a message; whatever could not be read from where it came in is "
"carried without its HTTP filters until that connection closes. Connections "
"opened from now on are filtered from their first byte.",
flush=True,
)
@dataclass
class InternalHTTPMessage:
"""Internal class to handle HTTP messages"""
@ -754,6 +774,11 @@ class InternalBasicHttpMetaClass:
)
parser_key = f"{cls._parser_class()}_{'in' if internal_data.current_pkt.is_input else 'out'}"
# A connection this filter met already open, and in the middle of a message: it
# cannot be read from where it came in, and it did nothing wrong. See below.
if internal_data.data_handler_context.get(f"{parser_key}_met_mid_message"):
raise NotReadyToRun()
parser = internal_data.data_handler_context.get(parser_key, None)
if parser is None or parser.raised_error:
parser: InternalHttpRequest | InternalHttpResponse = ParserType()
@ -816,8 +841,47 @@ class InternalBasicHttpMetaClass:
try:
parser.parse_data(internal_data.current_pkt.data)
except Exception as e:
traceback.print_exc()
match internal_data.invalid_encoding_action:
# **Met already open, and not yet read a single thing cleanly.** The
# datapath says when this filter's state began after its connection did:
# a queued filter that started, or was rebuilt, with the connection open,
# or new code taking over one. Its first bytes here can be the middle of a
# request — the rest of an upload — and llhttp reads that as a malformed
# message. It is not one, and judging it as one refused valid uploads in
# flight whenever a filter changed (with REJECT), or reported a working
# client as "not HTTP" (without). That direction is carried unfiltered
# until the connection closes instead. A connection met at the start of
# a message parses, is in step from then on, and is judged as usual.
if (
internal_data.filter_glob.get("__firegex_joined_late")
and not internal_data.data_handler_context.get(f"{parser_key}_in_step")
):
internal_data.data_handler_context[f"{parser_key}_met_mid_message"] = True
_met_mid_message()
raise NotReadyToRun()
action = internal_data.invalid_encoding_action
# Said once per connection and direction, and in one line: the parser is
# started afresh on the next packet, so traffic it cannot read fails on
# every packet — and a traceback each was the whole log, several hundred
# times, saying less than this does.
told = f"{parser_key}_unreadable_told"
if not internal_data.data_handler_context.get(told):
internal_data.data_handler_context[told] = True
print(
f"[warn] [http] this connection's "
f"{'requests' if internal_data.current_pkt.is_input else 'responses'}"
f" are not HTTP this filter can read ({e}): "
+ {
ExceptionAction.ACCEPT: "they are carried without its HTTP "
"filters seeing them. FGEX_INVALID_ENCODING_ACTION = "
"ExceptionAction.REJECT refuses such connections instead.",
ExceptionAction.REJECT: "the connection is refused "
"(FGEX_INVALID_ENCODING_ACTION).",
ExceptionAction.DROP: "the packet is dropped "
"(FGEX_INVALID_ENCODING_ACTION).",
}.get(action, "reported as the filter raising."),
flush=True,
)
match action:
case ExceptionAction.REJECT:
raise RejectConnection()
case ExceptionAction.DROP:
@ -827,6 +891,9 @@ class InternalBasicHttpMetaClass:
case ExceptionAction.ACCEPT:
raise NotReadyToRun()
# Read cleanly once: from here on this direction is a stream like any other.
internal_data.data_handler_context[f"{parser_key}_in_step"] = True
if any(msg.decoded_too_large for msg in parser.messages):
# Too large once decoded is too large: the operator's answer to a stream
# past the cap is the answer here too, rather than a second policy nobody

3
fgex-lib/requirements.txt
View File

@ -1,4 +1,7 @@
typer
# Imported directly by the CLI and both simulators for their output. `typer` happens to
# bring it today; `typer-slim` does not, and nothing here should rest on which one it is.
rich
pydantic>=2
typing-extensions>=4.7.1
pycryptodome

2
frontend/src/components/MainLayout.tsx
View File

@ -7,6 +7,7 @@ import { getMainPath, HOME_SECTIONS } from '../js/utils';
import { useLocation } from 'react-router';
import { useNavbarStore, useSessionStore } from '../js/store';
import { HiMenu } from "react-icons/hi";
import ProblemNotifier from './Services/ProblemNotifier';
function MainLayout({ children }:{ children:any }) {
@ -26,6 +27,7 @@ function MainLayout({ children }:{ children:any }) {
>
<HeaderPage />
<NavBar />
<ProblemNotifier />
<AppShell.Main>
<Container size="lg">
{children}

4
frontend/src/components/Services/AddEditService.tsx
View File

@ -440,8 +440,8 @@ export default function AddEditService({ opened, onClose, edit }: {
{edit ?
<Alert color="yellow" mt="md">
Changing any of this restarts the service, which drops the connections it
is carrying. Adding an address, or adding and reordering filters, does not.
Changing any of this restarts the service. The connections it is carrying
are not cut; new ones get the new settings.
</Alert> : null}
<Space h="md" />
</Box>

68
frontend/src/components/Services/ProblemNotifier.tsx Normal file
View File

@ -0,0 +1,68 @@
import { showNotification } from '@mantine/notifications';
import { useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef } from 'react';
import { useNavigate } from 'react-router';
import { socketio } from '../../js/utils';
import { LogEntry, serviceQueryKey, services } from './utils';
/** How long one service's warnings are folded into the one already on screen. */
const QUIET_MS = 20_000
/**
* Says so, wherever the operator is, when a service reports a problem.
*
* The live log is on the service's own page, and it was the only place a warning went:
* an engine that died and came back, a filter raising on every packet, a fallback that
* cost the service its clients' addresses — each of them waited for somebody to open that
* page. Warnings and errors now also arrive as a notification, one per service per
* `QUIET_MS` with the rest counted into the next, because a flood arriving twice — once
* in the log and once as a stack of toasts — informs nobody. Blocks are not problems and
* never notify.
*/
export default function ProblemNotifier() {
const queryClient = useQueryClient()
const navigate = useNavigate()
const told = useRef(new Map<string, { at: number, held: number, worst: string }>())
useEffect(() => {
const onLog = (payload: { service_id: string, entries: LogEntry[] }) => {
const problems = payload.entries.filter(e => e.level === "warn" || e.level === "error")
if (problems.length === 0) return
const now = Date.now()
const worst = problems.some(e => e.level === "error") ? "error" : "warn"
const was = told.current.get(payload.service_id) ?? { at: 0, held: 0, worst }
// An error is not folded into a warning already on screen: "restarting it"
// followed by "could not restart it" is the second one that matters, and
// folded it waited for a third that need never come.
const escalates = worst === "error" && was.worst !== "error"
if (now - was.at < QUIET_MS && !escalates) {
told.current.set(payload.service_id, { ...was, held: was.held + problems.length })
return
}
told.current.set(payload.service_id, { at: now, held: 0, worst })
const last = problems[problems.length - 1]
const more = was.held + problems.length - 1
// By name, which on a page that never listed the services means asking for
// them first: an id is not something an operator recognises mid-round.
queryClient.ensureQueryData({ queryKey: serviceQueryKey, queryFn: services.list })
.catch(() => [])
.then(listed => show(
listed.find(s => s.service_id === payload.service_id)?.name
?? payload.service_id,
worst, last.text, more, payload.service_id))
}
const show = (name: string, worst: string, text: string, more: number, id: string) =>
showNotification({
autoClose: 8000,
color: worst === "error" ? "red" : "orange",
title: `${name}: ${worst === "error" ? "something went wrong" : "a warning"}`,
message: text + (more > 0 ? ` (and ${more} more — see its log)` : ""),
style: { cursor: "pointer" },
onClick: () => navigate(`/services/${id}`),
})
socketio.on("log", onLog)
return () => { socketio.off("log", onLog) }
}, [queryClient, navigate])
return null
}

26
frontend/src/components/Services/ServiceRow.tsx
View File

@ -4,7 +4,7 @@ import { useState } from 'react';
import { FaPlay, FaStop, FaTrash } from 'react-icons/fa';
import { IoSettingsSharp } from 'react-icons/io5';
import { MdChevronRight, MdMoreHoriz } from 'react-icons/md';
import { TbHexagon, TbShieldLock } from 'react-icons/tb';
import { TbAlertTriangle, TbHexagon, TbShieldLock } from 'react-icons/tb';
import { bareAddress, errorNotify, isMediumScreen, okNotify } from '../../js/utils';
import YesNoModal from '../YesNoModal';
import AddEditService from './AddEditService';
@ -38,6 +38,29 @@ export const transportSummary = (transport: string, proto: string) => {
: "The connection is terminated and reopened, so the kernel reassembles and the chain has no length limit. It also carries bulk traffic several times faster than NFQUEUE, which pays a userspace round trip per packet; what it costs is fail-open being rebuilt in userspace rather than guaranteed by the kernel. The service still sees the real client address."
}
/** How long a warning stays on the row after it happened, unless the log is cleared first. */
const PROBLEM_SHOWN_MS = 60 * 60 * 1000
/**
* The newest warning or error in the service's log, on the row, for an hour.
*
* The log is on the service's own page, and a problem there waits for somebody to open
* it — the notification says it once, this keeps saying it for whoever looks at the list
* afterwards. Clearing the log is how it is dismissed.
*/
export function ProblemBadge({ service }: { service: Service }) {
const problem = service.problem
if (!problem || Date.now() - problem.at > PROBLEM_SHOWN_MS) return null
const error = problem.level === "error"
return <Tooltip position="bottom" multiline w={380}
label={`${new Date(problem.at).toLocaleTimeString(undefined, { hour12: false })} — ${problem.text}. Clear the service's log to dismiss this.`}>
<Badge color={error ? "red" : "orange"} variant="light" size="xs" radius="sm"
leftSection={<TbAlertTriangle size={10} />}>
{error ? "ERROR" : "WARNING"}
</Badge>
</Tooltip>
}
/** Where a service is reachable, short enough to sit on one line. */
export const addressSummary = (addresses: Address[]) => {
if (!addresses || addresses.length === 0) return "no address yet"
@ -135,6 +158,7 @@ export default function ServiceRow({ service, onClick }: { service: Service, onC
<Badge color={statusColor} variant="light" size="xs" radius="sm">
{service.status.toUpperCase()}
</Badge>
<ProblemBadge service={service} />
<Tooltip position="bottom" multiline w={340}
label={transportSummary(service.transport, service.proto)}>
<Badge color="indigo" variant="light" size="xs" radius="sm">

14
frontend/src/components/Services/utils.ts
View File

@ -218,6 +218,20 @@ export type Service = {
addresses: Address[],
n_filters: number,
n_blocked: number,
/**
* The newest warning or error still in the service's log. The log lives on the
* service's own page, and this is what lets the list say something went wrong without
* that page being open. Clearing the log dismisses it.
*/
problem?: Problem | null,
}
export type Problem = {
/** Unix milliseconds. */
at: number,
/** warn | error */
level: string,
text: string,
}
export type ServiceAddForm = {

16
frontend/src/js/demo.ts
View File

@ -234,6 +234,10 @@ const addLog = (service_id: string, level: string, text: string) => {
logs[service_id] = [...(logs[service_id] ?? []), entry].slice(-500)
for (const cb of listeners["log"] ?? []) cb({ service_id, entries: [entry] })
}
// One warning already in the tail, so the list shows what a problem looks like: the line a
// service at its connection limit really writes.
addLog(svcShop, "warn", "the 512 connection limit turned away 37 more: they were refused. "
+ "Raise the limit, or find out who is holding connections open.")
let ticker: ReturnType<typeof setInterval> | null = null
const emit = (...tags: string[][]) => {
@ -261,6 +265,13 @@ const startTicker = () => {
}
if (changed) emit(["services"])
}, 4000)
// And one warning a little after the start, the way a real one arrives: as a
// notification wherever you are, and as the badge on the service's row.
setTimeout(() => {
addLog(svcShop, "warn", "the 512 connection limit turned away 12 more: they were refused. "
+ "Raise the limit, or find out who is holding connections open.")
emit(["services"])
}, 10_000)
}
let welcomed = false
@ -325,6 +336,9 @@ const decorate = (s: Json) => ({
addresses: addressesOf(s.service_id),
n_filters: state.filters.filter(f => f.service_id === s.service_id).length,
n_blocked: state.filters.filter(f => f.service_id === s.service_id).reduce((acc, f) => acc + f.blocked, 0),
// The newest warning or error in its tail, as the backend reports it.
problem: [...(logs[s.service_id] ?? [])].reverse()
.find(e => e.level === "warn" || e.level === "error") ?? null,
})
/**
@ -589,7 +603,7 @@ const routes: [string, RegExp, Handler][] = [
}
}],
["GET", /^services\/([^/]+)\/logs$/, m => { service(m[1]); return logs[m[1]] ?? [] }],
["DELETE", /^services\/([^/]+)\/logs$/, m => { service(m[1]); logs[m[1]] = []; return ok }],
["DELETE", /^services\/([^/]+)\/logs$/, m => { service(m[1]); logs[m[1]] = []; emit(["services"]); return ok }],
// ---- services: the filter chain
["GET", /^services\/([^/]+)\/filters$/, m => {

4
frontend/src/pages/Firewall/SettingsModal.tsx
View File

@ -43,8 +43,8 @@ export function SettingsModal({ opened, onClose }:{ opened:boolean, onClose:()=>
<Switch label="Allow UPnP protocol" checked={settings.allow_upnp} onChange={v => setSettings({...settings, allow_upnp:v.target.checked})}/>
<Switch label="Drop invalid packet" checked={settings.drop_invalid} onChange={v => setSettings({...settings, drop_invalid:v.target.checked})}/>
<Switch label="Allow DHCP" checked={settings.allow_dhcp} onChange={v => setSettings({...settings, allow_dhcp:v.target.checked})}/>
<Switch label="Leave port-forwarded traffic to the rules that published it"
description="Docker and podman published ports, and any other destination NAT: when no rule here matches the traffic, it does not meet the forward policy. Rules in forward mode still apply to it first."
<Switch label="Leave container traffic to the container runtime"
description="Published ports (and any other destination NAT), and what containers send to each other or out: when no rule here matches it, it does not meet the forward policy. Rules in forward mode still apply to it first."
checked={settings.allow_dnat} onChange={v => setSettings({...settings, allow_dnat:v.target.checked})}/>
</Stack>
</Card>

28
run.py
View File

@ -149,10 +149,9 @@ def load_config():
"proxy_ip_header": None,
"unsafe_disable_auth": False,
# Anything the engine or the backend reads straight out of the environment and
# run.py has no opinion about: the filter deadline, the QUIC ALPN list, the
# stream size cap. Kept here because run.py **rewrites** the compose file on
# run.py has no opinion about: the filter deadline, the stream size cap. Kept here because run.py **rewrites** the compose file on
# every start, so a variable hand-edited into it is gone by the next one — which
# is what the documentation telling an operator to set FGEX_PROXY_QUIC_ALPN
# is what the documentation telling an operator to set an engine variable
# "in the container's environment" silently ran into.
"env": {},
}
@ -216,7 +215,7 @@ def gen_args(args_to_parse: list[str]|None = None):
parser_start.add_argument('--port', "-p", type=int, required=False, help=f'Port where open the web service of the firewall (default from config: {config["port"]})', default=config["port"])
parser_start.add_argument('--host', required=False, help=f'Host IP address to bind the service to (default from config: {config["host"]})', default=config["host"])
parser_start.add_argument('--socket-dir', required=False, type=str, help=f'Listen on socket_dir/firegex.sock instead of TCP (default from config: {config["socket_dir"]})', default=config["socket_dir"])
parser_start.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_QUIC_ALPN. Stored, so later runs keep it.')
parser_start.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_FILTER_TIMEOUT_MS. Stored, so later runs keep it.')
parser_start.add_argument('--allowed-ips', required=False, type=str, help=f'Comma-separated list of CIDR addresses allowed to contact firegex (default from config: {config.get("allowed_ips")})', default=config.get("allowed_ips"))
parser_start.add_argument('--proxy-ip-header', required=False, type=str, help=f'Header name to read the client IP from (default from config: {config.get("proxy_ip_header")})', default=config.get("proxy_ip_header"))
parser_start.add_argument('--unsafe-disable-auth', action=argparse.BooleanOptionalAction, default=config.get("unsafe_disable_auth", False), help='UNSAFE: disable Firegex password/JWT authentication, making every request that reaches firegex a full administrator (for a trusted reverse proxy only)')
@ -234,7 +233,7 @@ def gen_args(args_to_parse: list[str]|None = None):
parser_restart.add_argument('--port', "-p", type=int, required=False, help=f'Port where open the web service of the firewall (default from config: {config["port"]})', default=config["port"])
parser_restart.add_argument('--host', required=False, help=f'Host IP address to bind the service to (default from config: {config["host"]})', default=config["host"])
parser_restart.add_argument('--socket-dir', required=False, type=str, help=f'Listen on socket_dir/firegex.sock instead of TCP (default from config: {config["socket_dir"]})', default=config["socket_dir"])
parser_restart.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_QUIC_ALPN. Stored, so later runs keep it.')
parser_restart.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_FILTER_TIMEOUT_MS. Stored, so later runs keep it.')
parser_restart.add_argument('--allowed-ips', required=False, type=str, help=f'Comma-separated list of CIDR addresses allowed to contact firegex (default from config: {config.get("allowed_ips")})', default=config.get("allowed_ips"))
parser_restart.add_argument('--proxy-ip-header', required=False, type=str, help=f'Header name to read the client IP from (default from config: {config.get("proxy_ip_header")})', default=config.get("proxy_ip_header"))
parser_restart.add_argument('--unsafe-disable-auth', action=argparse.BooleanOptionalAction, default=config.get("unsafe_disable_auth", False), help='UNSAFE: disable Firegex password/JWT authentication, making every request that reaches firegex a full administrator (for a trusted reverse proxy only)')
@ -254,7 +253,7 @@ def gen_args(args_to_parse: list[str]|None = None):
parser_config.add_argument('--host', required=False, help='Set default host IP address to bind the service to')
parser_config.add_argument('--socket-dir', required=False, type=str, help=f'Listen on socket_dir/firegex.sock instead of TCP (default from config: {config["socket_dir"]})', default=config["socket_dir"])
parser_config.add_argument('--password', required=False, type=str, nargs='?', const='', help='Change the password of the firewall (omit the value to be prompted for it interactively)')
parser_config.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_QUIC_ALPN. Stored, so later runs keep it.')
parser_config.add_argument('--env', '-e', required=False, action='append', metavar='KEY=VALUE', default=None, help='Extra environment variable for the firegex container, as KEY=VALUE (repeatable; KEY= removes one). For what the engine reads straight out of its environment, such as FGEX_PROXY_FILTER_TIMEOUT_MS. Stored, so later runs keep it.')
parser_config.add_argument('--allowed-ips', required=False, type=str, help=f'Comma-separated list of CIDR addresses allowed to contact firegex (default from config: {config.get("allowed_ips")})', default=config.get("allowed_ips"))
parser_config.add_argument('--proxy-ip-header', required=False, type=str, help=f'Header name to read the client IP from (default from config: {config.get("proxy_ip_header")})', default=config.get("proxy_ip_header"))
parser_config.add_argument('--unsafe-disable-auth', action=argparse.BooleanOptionalAction, default=None, help='UNSAFE: turn Firegex password/JWT authentication off or back on, on the running instance and for the next start')
@ -410,6 +409,19 @@ def get_web_interface_url():
display_host = "localhost" if args.host == "0.0.0.0" else args.host
return f"http://{display_host}:{args.port}"
def compose_env(key: str, value) -> str:
"""One operator-supplied `KEY=VALUE` for the compose file, as compose will read it back.
Quoted, because `dict_to_yaml` writes scalars as they come and this value is the
operator's: a `: ` in it made the entry a mapping compose refuses, and a ` #` cut it
short as a comment. And every `$` doubled, because compose interpolates variables in its
own file — a value naming `$HOME` would have arrived as a path. The standalone start
passes the same entries through `shlex.quote` for the neighbouring reason.
"""
import json
return json.dumps(f"{key}={value}".replace("$", "$$"))
def write_compose(skip_password = True):
psw_set = get_password() if not skip_password else None
with open(g.composefile,"wt") as compose:
@ -434,7 +446,7 @@ def write_compose(skip_password = True):
*(["UNSAFE_DISABLE_AUTH=1"] if getattr(args, 'unsafe_disable_auth', False) else []),
# Whatever the operator asked to be here and run.py has no
# opinion about. Last, so it reads as the addition it is.
*(f"{k}={v}" for k, v in sorted(getattr(args, 'env_vars', {}).items())),
*(compose_env(k, v) for k, v in sorted(getattr(args, 'env_vars', {}).items())),
],
"volumes": [
"firegex_data:/execute/db",
@ -497,7 +509,7 @@ def write_compose(skip_password = True):
*([f"ALLOWED_IPS={args.allowed_ips}"] if getattr(args, 'allowed_ips', None) else []),
*([f"PROXY_IP_HEADER={args.proxy_ip_header}"] if getattr(args, 'proxy_ip_header', None) else []),
*(["UNSAFE_DISABLE_AUTH=1"] if getattr(args, 'unsafe_disable_auth', False) else []),
*(f"{k}={v}" for k, v in sorted(getattr(args, 'env_vars', {}).items())),
*(compose_env(k, v) for k, v in sorted(getattr(args, 'env_vars', {}).items())),
],
"volumes": [
"firegex_data:/execute/db"

51
tests/helpers/quicserver.py
View File

@ -25,7 +25,7 @@ try:
from aioquic.h3.connection import H3_ALPN, H3Connection
from aioquic.h3.events import DataReceived, HeadersReceived
from aioquic.quic.configuration import QuicConfiguration
from aioquic.quic.events import ConnectionTerminated
from aioquic.quic.events import ConnectionTerminated, StreamDataReceived
HAVE_AIOQUIC = True
except ImportError: # pragma: no cover - depends on what is installed
HAVE_AIOQUIC = False
@ -49,11 +49,16 @@ class QuicEcho:
the suite is not. Stopping it closes the loop, which is what ends the server.
"""
def __init__(self, port: int, cert: str, key: str, ipv6: bool = False):
def __init__(self, port: int, cert: str, key: str, ipv6: bool = False,
alpn: list[str] | None = None):
self.port = port
self.cert = cert
self.key = key
self.ipv6 = ipv6
#: What it negotiates. HTTP/3 unless told otherwise; anything else is a service
#: that echoes each stream back as it is — QUIC carrying a protocol of its own,
#: which is what a CTF's QUIC service usually is when it is not a web one.
self.alpn = alpn
#: Every address a connection arrived from. What the *service* saw, which is the
#: only way to check that the client's own address survived the relay.
self.seen_peers: list[tuple] = []
@ -89,11 +94,12 @@ class QuicEcho:
handle.write(material)
os.chmod(path, 0o600)
paths.append(path)
config = QuicConfiguration(is_client=False, alpn_protocols=H3_ALPN)
config = QuicConfiguration(is_client=False, alpn_protocols=self.alpn or H3_ALPN)
config.load_cert_chain(paths[0], paths[1])
seen = self.seen_peers
protocol = _EchoProtocol if not self.alpn else _RawEchoProtocol
await serve(self.host, self.port, configuration=config,
create_protocol=lambda *a, **kw: _EchoProtocol(seen, *a, **kw))
create_protocol=lambda *a, **kw: protocol(seen, *a, **kw))
for path in paths:
os.unlink(path)
self._ready.set()
@ -144,6 +150,22 @@ if HAVE_AIOQUIC:
self._http.send_data(stream_id, answer, end_stream=True)
self.transmit()
class _RawEchoProtocol(QuicConnectionProtocol):
"""Each stream back as it came, once the client has finished sending it."""
def __init__(self, seen_peers, *args, **kwargs):
super().__init__(*args, **kwargs)
self._streams: dict[int, bytes] = {}
def quic_event_received(self, event):
if isinstance(event, StreamDataReceived):
self._streams[event.stream_id] = \
self._streams.get(event.stream_id, b"") + event.data
if event.end_stream:
self._quic.send_stream_data(
event.stream_id, self._streams.pop(event.stream_id), end_stream=True)
self.transmit()
class _RequestingProtocol(QuicConnectionProtocol):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -208,6 +230,27 @@ async def _ask(host: str, port: int, method: str, path: str,
)
async def _raw(host: str, port: int, alpn: str, payload: bytes, timeout: float) -> bytes:
config = QuicConfiguration(is_client=True, alpn_protocols=[alpn])
config.verify_mode = ssl.CERT_NONE
async with connect(host, port, configuration=config) as client:
reader, writer = await client.create_stream()
writer.write(payload)
writer.write_eof()
return await asyncio.wait_for(reader.read(), timeout)
def quic_exchange(host: str, port: int, alpn: str, payload: bytes,
timeout: float = 8.0) -> bytes | None:
"""One stream over a QUIC connection negotiating `alpn`: what came back, or `None`
when no connection could be made or nothing came back in time."""
try:
return asyncio.run(asyncio.wait_for(_raw(host, port, alpn, payload, timeout),
timeout + 2))
except (ConnectionError, OSError, asyncio.TimeoutError, TimeoutError):
return None
def authority(host: str, port: int) -> str:
"""What goes in `:authority`, bracketed where the host needs it.

5
tests/integration/conftest.py
View File

@ -301,11 +301,12 @@ def quic_stand_in(certificate):
from helpers.quicserver import QuicEcho
started = []
def _serve(ipv6: bool = False, port: int | None = None) -> "QuicEcho":
def _serve(ipv6: bool = False, port: int | None = None,
alpn: list[str] | None = None) -> "QuicEcho":
cert, key = certificate("::1" if ipv6 else "127.0.0.1")
# A UDP probe: this one is going to bind a QUIC endpoint, and a port free for TCP
# says nothing about whether it is free for UDP.
echo = QuicEcho(port or free_port(ipv6, udp=True), cert, key, ipv6)
echo = QuicEcho(port or free_port(ipv6, udp=True), cert, key, ipv6, alpn=alpn)
echo.start()
echo.material = (cert, key)
started.append(echo)

189
tests/integration/test_filters_python.py
View File

@ -7,6 +7,7 @@ taking `(data, direction)`, so a filter written from the docs worked on one tran
failed on the other.
"""
import socket
import time
import pytest
@ -201,6 +202,67 @@ def test_switching_one_function_off_stops_it_deciding_without_touching_the_code(
assert "refuse_traversal" in stored, "the code was edited to switch a function off"
def test_new_code_reaches_a_connection_already_open(api, protected, inspecting_layer):
"""An edit is for the attack in progress, which is on a connection already open.
The proxy layer hands a connection's next chunk to the new chain, and `cppregex`
resets its matchers on a new configuration. `cpproxy` kept a version number for the
same purpose and never read it, so a stream went on running the code it had started
with for as long as it stayed open: a check added against one connection did not reach
that connection.
"""
service_id, server, port = protected(inspecting_layer, name="py-live")
filter_id = add_python_filter(api, service_id, filter_code.BLOCK_MARKER)
start_and_settle(api, service_id)
server.connect_client(timeout=3)
try:
server.send_packet(b"carrying NEWRULE")
assert server.recv_packet() == b"carrying NEWRULE", \
"the connection did not work to begin with"
assert api.services_set_code(
service_id, filter_id, filter_code.BLOCK_MARKER.replace("PYBLOCK", "NEWRULE"))
time.sleep(RELOAD)
server.send_packet(b"carrying NEWRULE")
assert server.recv_packet() != b"carrying NEWRULE", \
"the connection that was already open went on running the old code"
finally:
server.close_client()
def test_a_python_filter_added_later_reaches_a_connection_already_open(
api, protected, inspecting_layer):
"""The same as for patterns, for the binary that runs code.
`cpproxy` read each stream's direction from which end it had seen open it, so besides
dropping a stream it met halfway it would have had nothing to go on. It takes the
direction from the packet now, which the kernel knows either way.
"""
service_id, server, port = protected(inspecting_layer, name="py-late")
add_regex_filter(api, service_id, "UNRELATED")
start_and_settle(api, service_id)
server.connect_client(timeout=3)
try:
server.send_packet(b"carrying PYBLOCK")
assert server.recv_packet() == b"carrying PYBLOCK", \
"the connection did not work to begin with"
add_python_filter(api, service_id, filter_code.BLOCK_MARKER)
time.sleep(RELOAD + 1)
server.send_packet(b"still fine")
assert server.recv_packet() == b"still fine", \
"the connection stopped working when the new filter picked it up"
try:
server.send_packet(b"carrying PYBLOCK")
got = server.recv_packet()
except OSError:
got = False
assert got != b"carrying PYBLOCK", \
"a connection that was already open was never inspected by the new filter"
finally:
server.close_client()
def test_saving_different_code_reconciles_the_list_of_functions(api, http_filter):
"""The code is what decides which functions exist.
@ -273,3 +335,130 @@ def test_removing_a_filter_leaves_the_rest_of_the_chain_running(api, protected,
time.sleep(1.0)
assert channel.gets_through(b"carrying PYBLOCK"), "the removed filter still decides"
assert channel.is_blocked(b"carrying BLOCKME"), "the rest of the chain stopped"
#: Strict about what it cannot read, as an operator guarding against request smuggling
#: would be — the setting under which judging half a request refused it.
STRICT_HTTP = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT, ExceptionAction
from firegex.pyfilters.models import HttpRequest
FGEX_INVALID_ENCODING_ACTION = ExceptionAction.REJECT
@pyfilter
def refuse_marked_url(req: HttpRequest):
return REJECT if "PYBLOCK" in (req.url or "") else ACCEPT
"""
def _answer(sock: socket.socket) -> bytes:
"""One HTTP response off `sock`, or what arrived before it closed or went quiet."""
got = b""
try:
while b"\r\n\r\n" not in got:
chunk = sock.recv(65536)
if not chunk:
return got
got += chunk
head, _, body = got.partition(b"\r\n\r\n")
length = next((int(line.split(b":")[1]) for line in head.split(b"\r\n")
if line.lower().startswith(b"content-length:")), 0)
while len(body) < length:
chunk = sock.recv(65536)
if not chunk:
break
body += chunk
return head + b"\r\n\r\n" + body
except OSError:
return got
def test_a_request_in_flight_when_the_filter_changes_is_not_judged_by_its_second_half(
api, service, http_stand_in, inspecting_layer):
"""A filter meeting a connection halfway leaves what it cannot read alone.
Half an upload is sent, the filter changes — a queued one is added, which rebuilds the
chain, or a proxied one gets new code — and the other half follows. Whatever runs the
filter now meets that connection in the middle of a body. It used to hand the body to
the HTTP parser as if it were the start of a request, which it is not: with the
strict setting on, a valid upload was refused in flight, and without it the log said
a working client was not speaking HTTP. Connections opened afterwards are filtered
from their first byte, as always.
"""
http = http_stand_in(inspecting_layer.ipv6)
service_id = service(f"py-inflight-{http.port}", inspecting_layer.ip, http.port,
inspecting_layer.transport)
queued = inspecting_layer.transport == "nfqueue"
if queued:
add_regex_filter(api, service_id, "UNRELATED")
else:
filter_id = add_python_filter(api, service_id, STRICT_HTTP)
start_and_settle(api, service_id)
body = b"x" * 3000
with socket.create_connection((inspecting_layer.ip, http.port), timeout=3) as sock:
sock.sendall(b"POST /upload HTTP/1.1\r\nHost: svc\r\nContent-Length: 3000\r\n\r\n"
+ body[:1500])
time.sleep(0.3)
if queued:
add_python_filter(api, service_id, STRICT_HTTP)
else:
assert api.services_set_code(service_id, filter_id, STRICT_HTTP + "\n# edited\n")
time.sleep(RELOAD + 1)
sock.sendall(body[1500:])
assert b"sent 3000 bytes" in _answer(sock), \
"a valid upload in flight was refused when the filter changed"
said = [e["text"] for e in api.services_logs(service_id)
if e["level"] in ("warn", "error") and "not HTTP" in e["text"]]
assert not said, f"a working client was reported as not speaking HTTP: {said}"
with socket.create_connection((inspecting_layer.ip, http.port), timeout=3) as sock:
sock.sendall(b"GET /PYBLOCK HTTP/1.1\r\nHost: svc\r\n\r\n")
assert b"200" not in _answer(sock), "a new connection was not filtered"
#: Remembers, per connection, whether the client has said FIRST — so a verdict on
#: SECOND says whether the filter's state for the connection survived.
REMEMBERS = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT
from firegex.pyfilters.models import RawPacket
said_first = False
@pyfilter
def second_after_first(packet: RawPacket):
global said_first
if packet.is_input and b"FIRST" in packet.data:
said_first = True
return REJECT if said_first and b"SECOND" in packet.data else ACCEPT
"""
def test_an_edit_to_another_filter_leaves_a_python_filters_connections_alone(
api, protected, inspecting_layer):
"""A Python filter keeps its per-connection state through edits that are not its own.
Every edit reloads the whole chain. On both layers that restarted the Python filters
too — a new worker on the proxy, a clean slate in `cpproxy` — so changing a pattern
in one filter wiped what another had kept about every open connection: its module
globals, and the place its HTTP parser had reached in a request.
"""
service_id, server, port = protected(inspecting_layer, name="py-kept")
add_python_filter(api, service_id, REMEMBERS)
regex_filter, _ = add_regex_filter(api, service_id, "UNRELATED")
start_and_settle(api, service_id)
server.connect_client(timeout=3)
try:
server.send_packet(b"FIRST")
assert server.recv_packet() == b"FIRST", "the connection did not work to begin with"
assert api.services_add_regex(service_id, regex_filter, "ALSOUNRELATED", mode="C")
time.sleep(RELOAD + 1)
try:
server.send_packet(b"SECOND")
got = server.recv_packet()
except OSError:
got = False
assert got != b"SECOND", \
"a pattern added to another filter wiped this filter's state for the connection"
finally:
server.close_client()

63
tests/integration/test_filters_regex.py
View File

@ -173,3 +173,66 @@ def test_switching_a_pattern_off_stops_it_deciding(api, blocking):
assert api.services_edit_regex(service_id, filter_id, pattern_id, active=True)
time.sleep(RELOAD)
assert channel.is_blocked(b"carrying BLOCKME"), "it did not start deciding again"
def _held_open(server, payload: bytes) -> bytes | bool:
"""Send `payload` on the connection the test is holding and return the answer."""
try:
server.send_packet(payload)
except OSError:
return False
return server.recv_packet()
def test_a_filter_added_to_a_running_service_reaches_a_connection_already_open(
api, protected, inspecting_layer):
"""A new filter is for the attack in progress, which is on a connection already open.
On the NFQUEUE layer a new filter is a new process, and it meets that connection
halfway through. The stream follower it used threw away any stream whose handshake it
had not seen, so every connection open when a filter was added — or when the chain was
rebuilt for any other reason — went uninspected until it closed.
"""
service_id, server, port = protected(inspecting_layer, name="rx-late")
add_regex_filter(api, service_id, "UNRELATED", name="first")
start_and_settle(api, service_id)
server.connect_client(timeout=3)
try:
assert _held_open(server, b"hello") == b"hello", \
"the connection did not work to begin with"
add_regex_filter(api, service_id, "BLOCKME", name="second")
time.sleep(RELOAD + 1)
# Picked up, not broken: the connection has to go on working for everything else.
assert _held_open(server, b"still fine") == b"still fine", \
"the connection stopped working when the new filter picked it up"
assert _held_open(server, b"carrying BLOCKME") != b"carrying BLOCKME", \
"a connection that was already open was never inspected by the new filter"
finally:
server.close_client()
def test_a_connection_open_before_the_service_started_is_inspected(
api, protected, inspecting_layer):
"""Starting a service protects the connections it finds, not only the ones after.
Only the queued layer can: it filters packets, so a connection that predates it is as
visible as any other. The proxy layer takes connections as they are made, and one
already made goes on straight to the service.
"""
if inspecting_layer.transport != "nfqueue":
pytest.skip("this is the queued layer's guarantee")
service_id, server, port = protected(inspecting_layer, name="rx-before")
add_regex_filter(api, service_id, "BLOCKME")
server.connect_client(timeout=3)
try:
assert _held_open(server, b"hello") == b"hello", \
"the connection did not work to begin with"
start_and_settle(api, service_id)
assert _held_open(server, b"still fine") == b"still fine", \
"the connection stopped working when the service started"
assert _held_open(server, b"carrying BLOCKME") != b"carrying BLOCKME", \
"a connection open before the service started was never inspected"
finally:
server.close_client()

33
tests/integration/test_quic.py
View File

@ -276,3 +276,36 @@ def test_what_the_service_speaks_can_be_changed_while_it_runs(api, service, quic
answer = h3_request(quic.host, quic.port, "/which")
assert answer.endswith("bytes"), \
f"the edit was saved and the relay went on speaking HTTP/3: {answer!r}"
# --- what a QUIC service negotiates --------------------------------------------------
@needs_quic
def test_a_quic_service_that_is_not_http3_is_reached_with_nothing_to_configure(
api, service, quic_stand_in):
"""QUIC carries whatever its two ends agree on, and the engine has to say it to both.
It offered the service a list of its own — one environment variable for the whole
instance, `h3` unless firegex was restarted with another — because the client's hello
was taken to be unreadable. So a QUIC service speaking anything but HTTP/3 was not
reachable through firegex until somebody found that. The hello is read off the
client's first packet now and offered to the service as it is, the way the TLS path
has always mirrored it; there is nothing to set.
"""
from helpers.quicserver import quic_exchange
server = quic_stand_in(alpn=["fgx-echo"])
cert, key = server.material
service_id = service(f"quic-alpn-{server.port}", "127.0.0.1", server.port, "proxy",
proto="quic", tls_cert=cert, tls_key=key)
add_regex_filter(api, service_id, "BLOCKME")
start_and_settle(api, service_id)
assert quic_exchange(server.host, server.port, "fgx-echo", b"hello") == b"hello", \
"a QUIC service that is not HTTP/3 was not reached"
assert quic_exchange(server.host, server.port, "fgx-echo", b"x BLOCKME", timeout=3) \
!= b"x BLOCKME", "a stream carrying a blocked pattern reached the service"
# And one speaking something the service does not is refused by the service, as it
# would be with nothing in the way.
assert quic_exchange(server.host, server.port, "fgx-other", b"hello", timeout=3) is None

419
tests/integration/test_remote_clients.py Normal file
View File

@ -0,0 +1,419 @@
"""A client on another host.
Every other test here dials from the host firegex runs on, and a reply to such a client is
delivered locally whether or not the rules brought it home — so a return path that only
works for a client on this host passes all of them, and fails for every client a CTF
service actually has. Here the client is in a network namespace behind a veth pair: its
packets arrive on an interface, its address is none of this host's, and a reply that is
not brought home is routed out to it, past the engine waiting for it.
"""
import os
import socket
import subprocess
import threading
import time
import pytest
from integration.conftest import add_regex_filter, start_and_settle
from helpers.host import as_root
from helpers.net import free_port
pytestmark = [pytest.mark.instance, pytest.mark.root]
NETNS = "fgextest"
HOST_SIDE, CLIENT_SIDE = "fgxt0", "fgxt1"
HOST_IP, CLIENT_IP = "10.199.77.1", "10.199.77.2"
def _take_down() -> None:
as_root("ip", "netns", "del", NETNS)
as_root("ip", "link", "del", HOST_SIDE)
@pytest.fixture
def elsewhere(datapath_in_our_path):
"""Run a snippet of Python on a client that is not this host, and read what it printed."""
if not datapath_in_our_path:
pytest.skip("firegex's datapath is not in front of this host")
_take_down()
for step in (
("ip", "netns", "add", NETNS),
("ip", "link", "add", HOST_SIDE, "type", "veth", "peer", "name", CLIENT_SIDE),
("ip", "link", "set", CLIENT_SIDE, "netns", NETNS),
("ip", "addr", "add", f"{HOST_IP}/24", "dev", HOST_SIDE),
("ip", "link", "set", HOST_SIDE, "up"),
("ip", "netns", "exec", NETNS, "ip", "addr", "add", f"{CLIENT_IP}/24",
"dev", CLIENT_SIDE),
("ip", "netns", "exec", NETNS, "ip", "link", "set", CLIENT_SIDE, "up"),
("ip", "netns", "exec", NETNS, "ip", "link", "set", "lo", "up"),
):
if as_root(*step) is None:
_take_down()
pytest.skip("cannot stage a second host here: no root, or no network namespaces")
def run(code: str) -> str:
return (as_root("ip", "netns", "exec", NETNS, "python3", "-c", code) or "").strip()
yield run
_take_down()
SERVICE_NETNS = "fgextest-svc"
SERVICE_HOST_SIDE, SERVICE_SIDE = "fgxs0", "fgxs1"
CONTAINER_IP = "10.199.78.2"
CONTAINER_TABLE = "fgextest_dnat"
#: What the "container" runs: TCP and UDP on 8080, echoing — except `WHO`, answered with
#: the address the service saw, which is how transparency is asked about.
CONTAINER_SERVICE = r"""
import socket, threading
def tcp():
s = socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(("0.0.0.0", 8080)); s.listen(64)
def serve(c, peer):
with c:
while True:
d = c.recv(4096)
if not d: return
c.sendall(peer[0].encode() if d == b"WHO" else d)
while True:
c, peer = s.accept()
threading.Thread(target=serve, args=(c, peer), daemon=True).start()
def udp():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(("0.0.0.0", 8080))
while True:
d, peer = s.recvfrom(4096)
s.sendto(peer[0].encode() if d == b"WHO" else d, peer)
threading.Thread(target=udp, daemon=True).start()
tcp()
"""
#: Forwarding between the client's link and the container's, both ways. A host running
#: Docker drops forwarded traffic by policy, and this is how a runtime lets its own
#: published ports through it.
FORWARD_RULES = [("-i", HOST_SIDE, "-o", SERVICE_HOST_SIDE),
("-i", SERVICE_HOST_SIDE, "-o", HOST_SIDE)]
def _root_command(*argv: str) -> list[str]:
return list(argv) if os.geteuid() == 0 else ["sudo", "-n", *argv]
def _rewrites(published: int) -> list[tuple[str, ...]]:
"""Docker's own rule for a published port, in its own tool: `iptables` in `nat`."""
return [("-t", "nat", chain, "-d", HOST_IP, "-p", proto, "--dport", str(published),
"-j", "DNAT", "--to-destination", f"{CONTAINER_IP}:8080")
for chain in ("PREROUTING", "OUTPUT") for proto in ("tcp", "udp")]
_staged: list[tuple[str, ...]] = []
def _undo_rewrites() -> None:
while _staged:
rule = _staged.pop()
as_root("iptables", *rule[:2], "-D", *rule[2:])
def _take_container_down() -> None:
as_root("pkill", "-f", "fgextest-container")
as_root("nft", "delete", "table", "ip", CONTAINER_TABLE)
_undo_rewrites()
for rule in FORWARD_RULES:
as_root("iptables", "-D", "FORWARD", *rule, "-j", "ACCEPT")
as_root("ip", "netns", "del", SERVICE_NETNS)
as_root("ip", "link", "del", SERVICE_HOST_SIDE)
@pytest.fixture
def container(elsewhere):
"""A service the way a container runtime publishes one, on the far side of a DNAT.
The service lives in a namespace of its own, and a port of this host's is rewritten to
it at `dstnat` in `nat PREROUTING` and `nat OUTPUT` — the priority, the hooks and the
shape of Docker's own rule — without needing Docker. It is the deployment a CTF
service almost always has, and the rewrite is what the rest of the suite never puts
in the way: every other service here listens on this host, where a reply is local.
"""
_take_container_down()
for step in (
("ip", "netns", "add", SERVICE_NETNS),
("ip", "link", "add", SERVICE_HOST_SIDE, "type", "veth", "peer", "name", SERVICE_SIDE),
("ip", "link", "set", SERVICE_SIDE, "netns", SERVICE_NETNS),
("ip", "addr", "add", "10.199.78.1/24", "dev", SERVICE_HOST_SIDE),
("ip", "link", "set", SERVICE_HOST_SIDE, "up"),
("ip", "netns", "exec", SERVICE_NETNS, "ip", "addr", "add", f"{CONTAINER_IP}/24",
"dev", SERVICE_SIDE),
("ip", "netns", "exec", SERVICE_NETNS, "ip", "link", "set", SERVICE_SIDE, "up"),
("ip", "netns", "exec", SERVICE_NETNS, "ip", "link", "set", "lo", "up"),
("ip", "netns", "exec", SERVICE_NETNS, "ip", "route", "add", "default",
"via", "10.199.78.1"),
):
if as_root(*step) is None:
_take_container_down()
pytest.skip("cannot stage a container's network here")
# Best effort: without them the client elsewhere cannot reach the container even with
# no firegex in the way, and the test says so rather than blaming firegex.
for rule in FORWARD_RULES:
as_root("iptables", "-I", "FORWARD", "1", *rule, "-j", "ACCEPT")
published = free_port()
# `iptables` first — it is what Docker writes with, and it is on every machine Docker
# is; `nft` for a host with no iptables at all.
for rule in _rewrites(published):
if as_root("iptables", *rule[:2], "-A", *rule[2:]) is None:
_undo_rewrites()
break
_staged.append(rule)
ruleset = (
f"table ip {CONTAINER_TABLE} {{\n"
+ "".join(
f" chain {hook} {{\n"
f" type nat hook {hook} priority dstnat; policy accept;\n"
f" ip daddr {HOST_IP} meta l4proto {{ tcp, udp }} th dport {published} "
f"dnat to {CONTAINER_IP}:8080\n"
f" }}\n"
for hook in ("prerouting", "output")
)
+ "}\n"
)
if not _staged and subprocess.run(
_root_command("nft", "-f", "-"), input=ruleset, text=True,
capture_output=True).returncode != 0:
_take_container_down()
pytest.skip("cannot stage a container runtime's rewrite here: neither iptables "
"nor nft took it")
serving = subprocess.Popen(
_root_command("ip", "netns", "exec", SERVICE_NETNS, "python3", "-c",
"# fgextest-container\n" + CONTAINER_SERVICE),
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(0.5)
yield published
serving.terminate()
_take_container_down()
def _local(port: int, payload: str, udp: bool = False) -> str:
"""The same question as `_ask`, from this host."""
kind = socket.SOCK_DGRAM if udp else socket.SOCK_STREAM
with socket.socket(socket.AF_INET, kind) as s:
s.settimeout(4)
try:
if udp:
s.sendto(payload.encode(), (HOST_IP, port))
return repr(s.recvfrom(64)[0])
s.connect((HOST_IP, port))
s.sendall(payload.encode())
return repr(s.recv(64))
except OSError as e:
return type(e).__name__
@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"])
@pytest.mark.parametrize("transport", ["proxy", "nfqueue"])
def test_a_service_behind_a_container_runtimes_rewrite_is_protected_from_anywhere(
api, service, elsewhere, container, transport, udp):
"""Answered, filtered and transparent, from another host and from this one.
Two ways this failed, one per layer, and neither was visible from this host alone.
**Proxy, a client elsewhere:** the engine dials the published address as the client,
the runtime's DNAT sends it on to the container, and the container answers from its
own address. The rule that brings the engine's answers home looked the socket up by
that address, found none, and the answer went on to the real client, which reset it:
every connection waited out the dial and was retried as this host — slowly, and with
the client's address lost.
**NFQUEUE, a client on this host:** the runtime rewrites this host's own connection in
`nat OUTPUT`, so it leaves towards the container without passing prerouting, where the
queues are, and the container's answer arrives for a local socket without passing
postrouting. Neither half was inspected, and a payload refused from anywhere else
reached the service from here.
"""
clients = {
"another host": (lambda payload: elsewhere(_ask(container, udp, payload)), CLIENT_IP),
"this host": (lambda payload: _local(container, payload, udp), HOST_IP),
}
# Each path has to work with nothing of firegex in it before firegex can be judged on
# it: a host that does not forward to the container's network is not a firegex bug.
for where, (ask, _) in clients.items():
if ask("hello") != "b'hello'":
pytest.skip(f"the stand-in container is not reachable from {where} even "
f"without firegex: this host does not forward to it")
service_id = service(f"container-{transport}-{container}", HOST_IP, container, transport,
proto="udp" if udp else "tcp")
add_regex_filter(api, service_id, "BLOCKME")
start_and_settle(api, service_id, wait=1.5)
# A datagram refused is one that is not answered; a connection refused is closed.
refused = ("TimeoutError",) if udp else ("b''", "ConnectionResetError", "TimeoutError")
for where, (ask, client_ip) in clients.items():
assert ask("hello") == "b'hello'", f"the service did not answer {where}"
assert ask("WHO") == repr(client_ip.encode()), \
f"the service did not see the address of a client on {where}"
assert ask("x BLOCKME") in refused, f"a blocked payload from {where} reached the service"
def _echo_on(host: str, udp: bool) -> int:
"""A service answering with what it is sent, on this host's side of the link."""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM if udp else socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, 0))
def serve():
if udp:
while True:
data, peer = sock.recvfrom(2048)
sock.sendto(data, peer)
sock.listen(8)
while True:
conn, _ = sock.accept()
with conn:
conn.sendall(conn.recv(2048))
threading.Thread(target=serve, daemon=True).start()
return sock.getsockname()[1]
def _ask(port: int, udp: bool, payload: str = "hello") -> str:
kind = "SOCK_DGRAM" if udp else "SOCK_STREAM"
exchange = (
f"s.sendto({payload.encode()!r}, ('{HOST_IP}', {port})); print(s.recvfrom(64)[0])"
if udp else
f"s.connect(('{HOST_IP}', {port})); s.sendall({payload.encode()!r}); print(s.recv(64))"
)
return (
"import socket\n"
f"s = socket.socket(socket.AF_INET, socket.{kind}); s.settimeout(4)\n"
f"try:\n {exchange}\n"
"except OSError as e:\n print(type(e).__name__)\n"
)
@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"])
def test_a_published_address_answers_a_client_on_another_host(api, service, elsewhere, udp):
"""Published: the engine dials the service on a port the address does not name.
The rule that brings the service's answers home knew the service only by the address
it is protected on, which is where the engine dials it while the address *is* the
service. Published elsewhere, the answers came from a port nothing matched and were
routed straight to the client — who was expecting them from the engine and dropped
them. A client on this host never noticed, the reply being local either way.
The service's own port is not protected here, which is the case that broke: protected
too, its rule happened to cover the answers. And it keeps answering its own clients
directly, which is what a fix matching every reply from that port would have broken.
"""
served = _echo_on(HOST_IP, udp)
extra = free_port(udp=udp)
service_id = service(
f"remote-pub-{served}", HOST_IP, extra, "proxy", proto="udp" if udp else "tcp",
addresses=[{"ip_int": HOST_IP, "port": extra, "target_port": served}],
)
start_and_settle(api, service_id, wait=1.2)
assert elsewhere(_ask(extra, udp)) == "b'hello'", \
"a client on another host was not answered through the published address"
assert elsewhere(_ask(served, udp)) == "b'hello'", \
"the service stopped answering its own port directly"
def _echo_all_on(host: str, udp: bool) -> int:
"""`_echo_on`, answering every message of a connection rather than the first."""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM if udp else socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, 0))
def answer(conn):
with conn:
try:
while data := conn.recv(2048):
conn.sendall(data)
except OSError:
pass
def serve():
if udp:
while True:
data, peer = sock.recvfrom(2048)
sock.sendto(data, peer)
sock.listen(8)
while True:
conn, _ = sock.accept()
threading.Thread(target=answer, args=(conn,), daemon=True).start()
threading.Thread(target=serve, daemon=True).start()
return sock.getsockname()[1]
def _held_across(elsewhere, udp: bool, port: int, between) -> list[str]:
"""Ask once from the other host, run `between`, and ask again on the same socket."""
kind = "SOCK_DGRAM" if udp else "SOCK_STREAM"
exchange = (f"s.sendto(p, ('{HOST_IP}', {port})); print(s.recvfrom(64)[0])" if udp
else "s.sendall(p); print(s.recv(64))")
held = (
"import socket, time\n"
f"s = socket.socket(socket.AF_INET, socket.{kind}); s.settimeout(4)\n"
+ ("" if udp else f"s.connect(('{HOST_IP}', {port}))\n")
+ "def ask(p):\n"
+ " try:\n"
+ " " + exchange + "\n"
+ " except OSError as e:\n"
+ " print(type(e).__name__)\n"
+ "ask(b'before')\n"
+ "time.sleep(4)\n"
+ "ask(b'after')\n"
)
said: list[str] = []
client = threading.Thread(target=lambda: said.append(elsewhere(held)))
client.start()
time.sleep(1.5)
between()
client.join(timeout=20)
return said[0].splitlines() if said else []
@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"])
def test_a_connection_through_the_engine_survives_the_service_stopping(
api, service, elsewhere, udp):
"""The engine is taken out of the rules and carries what it has; see `retire`.
From another host because that is where the return path can fail: the answers to the
engine's own dial have to keep coming home after the service's rules are gone.
"""
served = _echo_all_on(HOST_IP, udp)
service_id = service(f"remote-stop-{served}", HOST_IP, served, "proxy",
proto="udp" if udp else "tcp")
start_and_settle(api, service_id, wait=1.2)
lines = _held_across(elsewhere, udp, served, lambda: api.services_stop(service_id))
assert lines[:1] == ["b'before'"], f"the connection did not work to begin with: {lines}"
assert lines[1:2] == ["b'after'"], \
f"stopping the service cut a connection the engine was carrying: {lines}"
@pytest.mark.parametrize("udp", [False, True], ids=["tcp", "udp"])
@pytest.mark.parametrize("transport", ["proxy", "nfqueue"])
def test_a_connection_already_open_survives_the_service_starting(
api, service, elsewhere, transport, udp):
"""Starting protection must not cut the clients a service already has.
A connection made before the service started is not the datapath's — nothing
redirected it, and nothing will: conntrack translates a connection once, at its first
packet. It goes on straight to the service. What it must not do is stop working, and on
the proxy layer it did for a client on another host: the rule that brings the service's
answers home to the engine matched every answer from the service's port, these
included, and delivered them here instead of sending them to the client.
"""
served = _echo_all_on(HOST_IP, udp)
service_id = service(f"remote-open-{served}", HOST_IP, served, transport,
proto="udp" if udp else "tcp")
lines = _held_across(elsewhere, udp, served,
lambda: start_and_settle(api, service_id, wait=0.5))
assert lines[:1] == ["b'before'"], f"the connection did not work to begin with: {lines}"
assert lines[1:2] == ["b'after'"], \
f"a connection already open stopped working when the service started: {lines}"

31
tests/integration/test_resilience.py
View File

@ -16,6 +16,7 @@ import pytest
from integration import filter_code
from integration.conftest import add_python_filter, add_regex_filter, start_and_settle
from helpers.host import as_root, ruleset
from helpers.traffic import Channel
pytestmark = pytest.mark.instance
@ -185,3 +186,33 @@ def test_the_kernel_keeps_forwarding_when_a_queued_filter_dies(api, protected,
assert channel.gets_through(b"traffic after the binary was killed"), \
"the kernel stopped forwarding when the filter died"
@pytest.mark.root
def test_rules_removed_from_under_firegex_are_put_back_and_said(api, protected,
inspecting_layer):
"""Something else managing the host's nftables must not quietly switch firegex off.
`nft -f /etc/nftables.conf` begins with `flush ruleset` on Debian, and so does
restarting its unit. Every service went on reading ACTIVE with nothing reaching a
filter, and nothing said so. They are put back within a few seconds now — the
datapath never stopped, so nothing restarts — and the service's log says what
happened, as an error, because traffic went through unfiltered in between.
"""
service_id, server, port = protected(inspecting_layer, name="flushed")
add_regex_filter(api, service_id, "BLOCKME")
start_and_settle(api, service_id)
channel = Channel(server, port, inspecting_layer.ipv6)
assert channel.is_blocked(b"carrying BLOCKME"), "it was not filtering to begin with"
if as_root("nft", "delete", "table", "inet", "fgex") is None:
pytest.skip("cannot remove the table from here")
deadline = time.time() + 10
while f"dport {port}" not in (ruleset() or "") and time.time() < deadline:
time.sleep(0.5)
assert channel.gets_through(b"harmless"), "the service stopped answering"
assert channel.is_blocked(b"carrying BLOCKME"), "the rules were not put back"
said = [e for e in api.services_logs(service_id)
if e["level"] == "error" and "removed from the kernel" in e["text"]]
assert said, "the rules were put back without a word about the gap"

31
tests/integration/test_service_edit.py
View File

@ -5,9 +5,11 @@ and written all at once. Each test here is a way it used to leave the service so
between the old definition and the new one.
"""
import time
import pytest
from integration.conftest import start_and_settle
from integration.conftest import RELOAD, start_and_settle
from helpers.certs import UNPARSEABLE_CERT, UNPARSEABLE_KEY
from helpers.traffic import Channel
@ -49,6 +51,33 @@ def test_an_edit_that_collides_leaves_the_service_as_it_was(api, service, stand_
assert [a["proto"] for a in api.services_addresses(udp)] == ["udp"]
def test_saving_a_service_as_it_was_or_renaming_it_drops_nothing(api, protected, proxy_layer):
"""The form sends every setting on every save, so most saves change nothing.
Each of them was taken for a change of definition anyway: a running service was
stopped and started around it, and every connection it carried was reset — for opening
the dialog and pressing save, or for a new name, which nothing running reads.
"""
service_id, server, _ = protected(proxy_layer, name="edit-same")
start_and_settle(api, service_id)
was = api.services_get(service_id)
server.connect_client(timeout=3)
try:
server.send_packet(b"before")
assert server.recv_packet() == b"before", "the connection did not work to begin with"
assert api.services_edit(service_id, **{key: was[key] for key in (
"name", "proto", "transport", "fail_open", "max_connections",
"over_limit_forwards", "first_byte_timeout")})
assert api.services_edit(service_id, name=f"{was['name']}-renamed")
time.sleep(RELOAD)
server.send_packet(b"after")
assert server.recv_packet() == b"after", "saving the service dropped its connections"
finally:
server.close_client()
assert api.services_get(service_id)["name"] == f"{was['name']}-renamed"
def test_a_running_service_whose_edit_will_not_start_is_put_back(api, protected,
proxy_layer):
"""Refused by the engine, not by the form: nothing short of starting it can tell.

81
tests/integration/test_service_lifecycle.py
View File

@ -4,9 +4,11 @@ Parametrised over every layer, because a service is a service whichever one it i
the layer is a choice inside it, not a different kind of object.
"""
import time
import pytest
from integration.conftest import start_and_settle
from integration.conftest import RELOAD, add_regex_filter, start_and_settle
from helpers.traffic import Channel
pytestmark = pytest.mark.instance
@ -66,3 +68,80 @@ def test_a_tls_service_occupies_no_port_of_its_own(api, protected, tls_layer):
start_and_settle(api, service_id)
address = api.services_addresses(service_id)[0]
assert "ssl_port" not in address and "clear_port" not in address, str(address)
def _held_open(server, payload: bytes) -> bytes | bool:
try:
server.send_packet(payload)
except OSError:
return False
return server.recv_packet()
def _log_says(api, service_id: str, words: str, wait: float = 10) -> bool:
deadline = time.monotonic() + wait
while time.monotonic() < deadline:
if any(words in e["text"] for e in api.services_logs(service_id)):
return True
time.sleep(0.3)
return False
def test_stopping_a_proxied_service_does_not_cut_its_connections(api, protected, proxy_layer):
"""Stopping takes the filters away, not the service.
The proxy terminates every connection it carries, so killing it — which a stop did —
cut them all: a download, a websocket, a request half answered. It is taken out of the
rules instead, and carries what it already has until it closes. Unfiltered: a stop is
how a filter that is hurting the service is taken out of the way, and it would not be
if it went on judging the connections already open. New ones go straight to the service.
"""
service_id, server, port = protected(proxy_layer, name="drain-stop")
add_regex_filter(api, service_id, "BLOCKME")
start_and_settle(api, service_id)
server.connect_client(timeout=3)
try:
assert _held_open(server, b"hello") == b"hello", "the connection did not work to begin with"
assert api.services_stop(service_id)
assert _held_open(server, b"still here") == b"still here", \
"stopping the service cut a connection it was carrying"
assert _held_open(server, b"carrying BLOCKME") == b"carrying BLOCKME", \
"a stopped service went on filtering the connections it had"
time.sleep(3)
assert _held_open(server, b"a while later") == b"a while later"
assert _log_says(api, service_id, "carry on unfiltered"), \
"the log did not say connections were still being carried"
finally:
server.close_client()
assert _log_says(api, service_id, "have all closed"), \
"the stopped engine did not notice its last connection had closed"
channel = Channel(server, port, proxy_layer.ipv6)
assert channel.gets_through(b"carrying BLOCKME"), "a new connection did not reach the service"
def test_a_restart_hands_over_without_cutting_connections(api, protected, proxy_layer):
"""A change that rebuilds the engine starts a new one beside the old.
New connections go to the new engine; the ones the old engine carries stay with it,
under the filters they started with, until they close. Killing it — which a restart
did — cut them all for an edit as small as the connection limit.
"""
service_id, server, port = protected(proxy_layer, name="drain-restart")
add_regex_filter(api, service_id, "BLOCKME")
start_and_settle(api, service_id)
server.connect_client(timeout=3)
try:
assert _held_open(server, b"hello") == b"hello", "the connection did not work to begin with"
assert api.services_edit(service_id, max_connections=100)
time.sleep(RELOAD)
assert _held_open(server, b"still here") == b"still here", \
"a restart cut a connection the old engine was carrying"
assert _held_open(server, b"carrying BLOCKME") != b"carrying BLOCKME", \
"a connection carried through a restart lost its filters"
finally:
server.close_client()
channel = Channel(server, port, proxy_layer.ipv6)
assert channel.gets_through(b"harmless"), "the new engine did not take new connections"
assert channel.is_blocked(b"carrying BLOCKME"), "the new engine came up without the filters"

7
tests/integration/test_udp.py
View File

@ -238,9 +238,12 @@ def test_a_queued_udp_service_keeps_flows_up_to_its_own_limit(api, service, udp_
assert _from(first, echo, b"MARK") == b"MARK"
assert _from(first, echo, b"again") is None, "the flow did not keep its state at all"
# Enough of them that some land on the first flow's thread: flows are shared out
# between the queue threads by a hash, and so is the limit.
# between the queue threads by a hash, and so is the limit. Each from a port handed
# out once, rather than one the kernel picks: that can be the first flow's own, now
# that nothing holds it, and a datagram sent from it is the first flow — the one the
# filter was told to refuse — which failed this one run in a few hundred.
for _ in range(200):
assert echo.exchange(b"another flow") == b"another flow"
assert _from(free_port(udp=True), echo, b"another flow") == b"another flow"
answered = _from(first, echo, b"once more")
if kept:

79
tests/unit/test_datapath_acks.py Normal file
View File

@ -0,0 +1,79 @@
"""What a datapath process's answer to a configuration push leaves running.
Both NFQUEUE binaries answer `ACK FAIL` with the filters they had still in force, so a
refusal is an error for the operator and nothing more. A push that is not answered at all
is different: whatever the process ends up doing is unknown, and its answer, if it comes,
would be read as the answer to the next command.
"""
import asyncio
import pytest
from modules.services import transports
from modules.services.models import KIND, Filter, Service, TRANSPORT
from modules.services.transports import ChainLink, NfqueueTransport, _QueueStage
class _Stdin:
def write(self, data: bytes) -> None:
self.data = data
async def drain(self) -> None:
pass
class _Process:
def __init__(self):
self.returncode = None
self.stdin = _Stdin()
self.killed = False
def kill(self) -> None:
self.killed = True
self.returncode = -9
def _stage() -> tuple[_QueueStage, _Process, list[str]]:
said: list[str] = []
srv = Service(service_id="s", name="n", status="active", proto="tcp",
transport=TRANSPORT.NFQUEUE)
owner = NfqueueTransport(srv, on_engine=said.append)
flt = Filter(filter_id="f1", service_id="s", position=0, kind=KIND.REGEX, name="p",
active=True)
stage = _QueueStage(srv, ChainLink(flt), owner)
stage.process = _Process()
return stage, stage.process, said
def test_a_refused_push_leaves_the_binary_running():
"""It used to be stopped, and stopped *on purpose*, so its watchdog said nothing: the
service went on reading active with no process behind its queue."""
stage, process, _ = _stage()
async def run():
pushing = asyncio.create_task(stage._push(b"payload\n"))
await asyncio.sleep(0)
stage._ack.set_result((False, "Failed to compile hyperscan db"))
with pytest.raises(Exception, match="rejected the filters"):
await pushing
asyncio.run(run())
assert not process.killed
assert stage.process is process and not stage._stopped
def test_an_unanswered_push_hands_the_binary_to_its_watchdog(monkeypatch):
"""Killed, and **not** marked stopped: its watchdog is what rebuilds the service from
the saved configuration, and what the process had applied cannot be known."""
monkeypatch.setattr(transports, "LOAD_TIMEOUT", 0.05)
stage, process, said = _stage()
async def run():
with pytest.raises(Exception, match="did not acknowledge"):
await stage._push(b"payload\n")
asyncio.run(run())
assert process.killed
assert not stage._stopped
assert any("restarting it" in line for line in said), said

66
tests/unit/test_firewall_rules.py Normal file
View File

@ -0,0 +1,66 @@
"""What the plain firewall writes into nftables, for the rules an operator lists.
Nothing here talks to the kernel: the commands are built and read back as data. The two
things pinned are both about traffic a rule was meant to let through and did not.
"""
from modules.firewall.models import Action, FirewallSettings, Mode, Protocol, Rule, Table
from modules.firewall.nftables import CONTAINER_BRIDGES, FiregexTables
def rule(**kw) -> Rule:
fields = dict(proto=Protocol.TCP, src="", dst="", port_src_from=1, port_dst_from=80,
port_src_to=65535, port_dst_to=80, action=Action.ACCEPT, mode=Mode.IN,
table=Table.FILTER)
fields.update(kw)
return Rule(**fields)
def families_of(commands: list[dict]) -> list[str]:
return [c["add"]["rule"]["family"] for c in commands]
def test_a_rule_naming_no_address_is_installed_for_both_families_wherever_it_is():
"""The family was narrowed once and never widened again, so every rule after the first
one naming an IPv4 address was IPv4 only — an "accept port 80" below it did not
exist for IPv6, and under a drop policy every IPv6 client of that port was refused."""
commands = FiregexTables().get_rules(
rule(src="10.0.0.0/8", port_dst_from=22, port_dst_to=22),
rule(),
)
assert families_of(commands) == ["ip", "ip", "ip6"], families_of(commands)
def test_a_rule_for_both_protocols_is_not_rewritten_underneath_its_owner():
both = rule(proto=Protocol.BOTH)
commands = FiregexTables().get_rules(both)
assert both.proto == Protocol.BOTH, "the caller's rule was turned into a TCP one"
protocols = {c["add"]["rule"]["expr"][0]["match"]["left"]["payload"]["protocol"]
for c in commands}
assert protocols == {"tcp", "udp"}, protocols
def settings(**kw) -> FirewallSettings:
fields = dict(keep_rules=False, allow_loopback=True, allow_established=True,
allow_icmp=True, multicast_dns=True, allow_upnp=True, drop_invalid=True,
allow_dhcp=True)
fields.update(kw)
return FirewallSettings(**fields)
def test_container_traffic_is_left_to_the_container_runtime():
"""Under a drop policy, a chain of firegex's own is evaluated on its own — and while the
rules lived in iptables' `FORWARD`, Docker's accepts beside them let containers talk to
each other and out. Only published ports were given back at first, which cut a web
container off from its database on the same bridge."""
commands = FiregexTables().dnat_rules(settings())
matched = [c["add"]["rule"]["expr"][0]["match"] for c in commands]
bridges = {m["right"] for m in matched if m["left"] == {"meta": {"key": "iifname"}}}
assert set(CONTAINER_BRIDGES) <= bridges, bridges
assert {"docker0", "br-*"} <= bridges
assert any(m["left"] == {"ct": {"key": "status"}} for m in matched), \
"published ports are no longer left to the runtime"
assert set(families_of(commands)) == {"ip", "ip6"}
assert all(c["add"]["rule"]["chain"] == "fgex_forward" for c in commands)
assert FiregexTables().dnat_rules(settings(allow_dnat=False)) == []

107
tests/unit/test_http_parsing.py
View File

@ -83,20 +83,25 @@ def run(*chunks: tuple[bytes, bool]) -> list[dict]:
return _feed(*chunks)[0]
def run_with_verdict(*chunks: tuple[bytes, bool]) -> tuple[list[dict], int]:
def run_with_verdict(*chunks: tuple[bytes, bool], settings: str = "") -> tuple[list[dict], int]:
"""The same, plus what the chain answered — which is the whole story when the
parser refuses the traffic and the filter is therefore never called."""
return _feed(*chunks)
return _feed(*chunks, settings=settings)[:2]
def _feed(*chunks: tuple[bytes, bool]) -> tuple[list[dict], int]:
def _feed(*chunks: tuple[bytes, bool], settings: str = "",
joined_late: bool = False) -> tuple[list[dict], int, str]:
glob = {"__firegex_pyfilter_enabled": ["watch_request", "watch_response"]}
exec(FILTER_CODE, glob, glob)
exec(FILTER_CODE + settings, glob, glob)
compile(glob)
if joined_late:
# What the datapath sets on a context that began after its connection did.
glob["__firegex_joined_late"] = True
for payload, is_input in chunks:
glob["__firegex_packet_info"] = packet(payload, is_input)
handle_packet(glob)
return glob["seen"], glob.get("__firegex_pyfilter_result", {}).get("action")
result = glob.get("__firegex_pyfilter_result") or {}
return glob["seen"], result.get("action"), result.get("matched_by")
def final(seen: list[dict], kind: str) -> dict:
@ -348,21 +353,101 @@ def test_an_identity_encoding_is_left_alone():
# answers changed between 9.2.1 and 9.4.3, and both are decisions about what a filter is
# shown rather than details of how it is parsed.
EMPTY_TRANSFER_ENCODING = b"POST / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: \r\n\r\n"
def test_an_empty_transfer_encoding_never_reaches_a_filter():
"""`Transfer-Encoding:` with nothing after it is refused, and the connection with it.
"""`Transfer-Encoding:` with nothing after it is not parsed into a request.
This is the shape request smuggling is built out of: a header that one parser reads
as "chunked follows" and another as "no framing here" is two readings of where the
next request begins. llhttp accepted it until 9.4.3 — which means firegex parsed it,
showed a filter an ordinary-looking request, and forwarded whatever the service then
made of it. Now the parse fails, `invalid_encoding_action` decides, and its default
is to reject: the filter is never called, because there is no one message here to
call it with.
made of it. Now the parse fails and `invalid_encoding_action` decides. The filter is
never called, because there is no one message here to call it with.
**By default the traffic is carried**, and said so: refusing whatever the parser cannot
read broke a service over any quirk of a client it would have accepted, with nobody
having chosen that. Refusing is one setting away.
"""
seen, action = run_with_verdict(
(b"POST / HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: \r\n\r\n", True))
seen, action = run_with_verdict((EMPTY_TRANSFER_ENCODING, True))
assert seen == []
assert action == 0 # ACCEPT
def test_traffic_the_parser_cannot_read_is_refused_when_asked_and_not_blamed_on_a_filter():
"""The stricter answer, chosen in the file — and credited to the parser.
It used to be credited to the filter function, which had never been shown the
request: the operator was told their own code refused a connection it did not see.
"""
seen, action, matched_by = _feed(
(EMPTY_TRANSFER_ENCODING, True),
settings="\nfrom firegex.pyfilters import ExceptionAction\n"
"FGEX_INVALID_ENCODING_ACTION = ExceptionAction.REJECT\n")
assert seen == []
assert action == 2 # REJECT
assert matched_by == "@INVALID_ENCODING"
def test_unreadable_traffic_is_reported_once_per_connection(capsys):
"""The parser starts afresh on every packet, so every packet fails the same way.
A traceback each was the whole log, hundreds of times; one line says it and what was
done about it.
"""
run_with_verdict(*[(EMPTY_TRANSFER_ENCODING, True)] * 4)
said = [line for line in capsys.readouterr().out.splitlines() if "[warn] [http]" in line]
assert len(said) == 1, said
assert "carried" in said[0], said
STRICT = ("\nfrom firegex.pyfilters import ExceptionAction\n"
"FGEX_INVALID_ENCODING_ACTION = ExceptionAction.REJECT\n")
def test_a_connection_met_in_the_middle_of_a_body_is_carried_not_refused(capsys, monkeypatch):
"""What a filter taking over an open connection meets first can be half a request.
The rest of an upload is not a malformed request, and judging it as one refused valid
uploads in flight whenever a filter changed — under the strict setting, which is
exactly the one an operator worried about smuggling chooses. That direction of the
connection is carried, and said once, as information rather than as a problem.
"""
from firegex.pyfilters.models import http
monkeypatch.setattr(http, "_told_met_mid_message", False)
seen, action, _ = _feed(
(b"x" * 200, True),
(b"GET /next HTTP/1.1\r\nHost: a\r\n\r\n", True),
settings=STRICT, joined_late=True)
assert action == 0, "a request met halfway was refused"
assert seen == [], "a direction met halfway was judged after all"
out = capsys.readouterr().out
assert "[warn] [http]" not in out, "a working client was reported as not speaking HTTP"
assert out.count("[info] [http]") == 1, out
def test_a_connection_met_at_the_start_of_a_message_is_filtered_as_usual():
"""Met between two requests — the ordinary idle keep-alive — nothing is lost."""
seen = _feed((b"GET /first HTTP/1.1\r\nHost: a\r\n\r\n", True),
settings=STRICT, joined_late=True)[0]
assert final(seen, "request")["url"] == "/first"
def test_once_in_step_a_connection_met_late_answers_to_the_usual_rules():
"""Only the first thing met is given the benefit of the doubt; garbage after a clean
request is garbage, and the strict setting refuses it as it would anywhere."""
_, action, matched_by = _feed(
(b"GET /first HTTP/1.1\r\nHost: a\r\n\r\n", True),
(EMPTY_TRANSFER_ENCODING, True),
settings=STRICT, joined_late=True)
assert action == 2 and matched_by == "@INVALID_ENCODING"
def test_a_connection_seen_from_its_start_gets_no_such_benefit():
"""Unreadable from the first byte is not a connection met halfway."""
_, action, _ = _feed((b"x" * 200, True), settings=STRICT)
assert action == 2
def test_tabs_around_a_content_length_are_tolerated():

24
tests/unit/test_nftables_addresses.py
View File

@ -206,7 +206,7 @@ def test_installed_rule_parsing():
# Simulate get() parsing logic
table = detached_table()
table.list_rules = lambda tables, chains: [raw_rule]
table.list_rules = lambda tables, chains, family=None: [raw_rule]
rules = table.get()
assert len(rules) == 1
r = rules[0]
@ -244,7 +244,7 @@ def test_an_interfaces_output_rule_is_read_back_as_the_interfaces():
],
}
table = detached_table()
table.list_rules = lambda tables, chains: [raw_rule]
table.list_rules = lambda tables, chains, family=None: [raw_rule]
rule = table.get()[0]
assert rule.ip_int == "wg0", "the rule was read back under the address it matches"
@ -292,9 +292,12 @@ def test_proxy_rule_generation_interface_vs_ip(monkeypatch):
# each, which is what makes a service protected on `lo` filter a local client.
chains = [c["add"]["rule"]["chain"] for c in commands]
assert chains.count("fgex_nat") == 1
assert chains.count("fgex_route") == 1
assert chains.count("fgex_nat_out") == 2, "one per address the interface carries"
assert len(commands) == 4
# Nothing on the way back: the engine's answers come home by the conntrack mark on
# the connections it opened. A per-service rule there caught the answers on
# connections open before the service started, and a remote client lost them.
assert "fgex_route" not in chains
assert len(commands) == 3
# Inbound stays the interface and nothing narrower. Pinning it to the addresses the
# interface carries was tried and reverted: a name is what an operator reaches for
@ -328,22 +331,13 @@ def test_proxy_rule_generation_interface_vs_ip(monkeypatch):
for e in nat_rule["expr"]
)
# Check expressions in route rule
route_rule = [c["add"]["rule"] for c in commands if c["add"]["rule"]["chain"] == "fgex_route"][0]
assert any(
e.get("match", {}).get("left", {}).get("meta", {}).get("key") == "oifname"
and e["match"]["right"] == "eth0"
for e in route_rule["expr"]
)
# 2. IP target
commands.clear()
table._add_proxy(srv, "127.0.0.1/32", 80, "ip", "tcp", 38000)
# Should write to nat_chain, route_chain, AND nat_output_chain
assert len(commands) == 3
# Both hooks of the redirect, and nothing on the way back.
assert len(commands) == 2
chains_ip = [c["add"]["rule"]["chain"] for c in commands]
assert "fgex_nat" in chains_ip
assert "fgex_route" in chains_ip
assert "fgex_nat_out" in chains_ip

137
tests/unit/test_proxy_drain.py Normal file
View File

@ -0,0 +1,137 @@
"""A proxy engine taken out of service carries its connections until they close.
The proxy terminates every connection it carries, so a stop or a restart that killed it
cut them all. `ProxyTransport.retire` lets it go on instead, and ends it as soon as it
reports none left — or at `DRAIN_LIMIT`, or when too many engines of one service are
draining at once. Driven here against a stand-in process, because the interesting part
is the bookkeeping, and the integration suite cannot wait ten minutes.
"""
import asyncio
import time
import pytest
from modules.services import transports
from modules.services.models import TRANSPORT, Service
class _Process:
def __init__(self):
self.returncode = None
def kill(self):
self.returncode = -9
async def wait(self):
return self.returncode
@pytest.fixture
def engine(monkeypatch):
monkeypatch.setattr(transports, "DRAIN_FRESH_AFTER", 0)
monkeypatch.setattr(transports, "_draining", {})
said: list[str] = []
def _build(live: int):
srv = Service(service_id="drain", name="n", status="active", proto="tcp",
transport=TRANSPORT.PROXY)
t = transports.ProxyTransport(srv, on_engine=said.append)
t.process = _Process()
t.counters["live"] = live
return t
return _build, said
def _report(t, live: int):
"""What a `STATS` line arriving now does."""
t.counters["live"] = live
t._stats_at = time.monotonic()
def test_it_goes_as_soon_as_its_last_connection_does(engine):
build, said = engine
gone = []
async def run():
t = build(live=2)
process = t.process
await t.retire(keep_filtering=True, then=lambda: gone.append(True))
_report(t, 2)
await asyncio.sleep(1.2)
assert process.returncode is None, "killed while it still carried connections"
_report(t, 0)
await asyncio.sleep(1.2)
assert process.returncode is not None, "left running with nothing to carry"
asyncio.run(run())
assert gone == [True], "its ports were never released"
assert any("under the filters they started with" in line for line in said), said
assert any("have all closed" in line for line in said), said
def test_a_count_from_before_it_was_taken_out_does_not_end_it(engine, monkeypatch):
"""`live=0` said before the rules came off says nothing about what arrived since."""
build, _ = engine
monkeypatch.setattr(transports, "DRAIN_FRESH_AFTER", 60)
async def run():
t = build(live=0)
_report(t, 0)
await t.retire(keep_filtering=False)
await asyncio.sleep(1.2)
assert t.process.returncode is None
await t._end_drain(None)
asyncio.run(run())
def test_connections_that_never_close_are_closed_at_the_limit(engine, monkeypatch):
build, said = engine
monkeypatch.setattr(transports, "DRAIN_LIMIT", 1)
async def run():
t = build(live=3)
process = t.process
await t.retire(keep_filtering=True)
_report(t, 3)
await asyncio.sleep(2)
assert process.returncode is not None
asyncio.run(run())
assert any(line.startswith("[warn]") and "still open" in line for line in said), said
def test_only_so_many_engines_of_a_service_drain_at_once(engine):
build, said = engine
async def run():
engines = [build(live=1) for _ in range(transports.MAX_DRAINING + 1)]
processes = [t.process for t in engines]
for t in engines:
await t.retire(keep_filtering=True)
_report(t, 1)
assert processes[0].returncode is not None, "the oldest was kept"
assert all(p.returncode is None for p in processes[1:])
assert len(transports._draining["drain"]) == transports.MAX_DRAINING
await transports.stop_draining()
assert all(p.returncode is not None for p in processes)
assert not transports._draining
asyncio.run(run())
assert any("were closed" in line for line in said), said
def test_an_engine_already_dead_is_simply_cleaned_up(engine):
build, _ = engine
gone = []
async def run():
t = build(live=0)
t.process.returncode = 1
await t.retire(keep_filtering=True, then=lambda: gone.append(True))
assert not transports._draining
asyncio.run(run())
assert gone == [True]

137
tests/unit/test_proxysim.py Normal file
View File

@ -0,0 +1,137 @@
"""`fgex pyfilters`: the way an operator tries a filter before it goes near a service.
Run as they would run it — the `fgex` script, as a process of its own, in front of a
real service — because every part of that path has broken on its own at some point: the
file watcher that starts the proxy, the child process it runs in (Python 3.14 starts
those with `forkserver`, which imports the script again), and the filtering itself.
Nothing else in the suite runs it, so a release could ship a simulator that no longer
simulates and nobody would find out until they needed it.
What it has to agree with is production. A filter that refuses here refuses there, clean
traffic comes back, and traffic the HTTP parser cannot read is carried — the library's
default — rather than refused as it used to be.
"""
import os
import signal
import socket
import subprocess
import sys
import threading
import time
import pytest
from helpers.net import free_port
FGEX = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "fgex-lib", "fgex"))
RAW = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT
from firegex.pyfilters.models import RawPacket
@pyfilter
def refuse(packet: RawPacket):
return REJECT if b"BLOCKME" in packet.data else ACCEPT
"""
HTTP = """from firegex.pyfilters import pyfilter, ACCEPT, REJECT
from firegex.pyfilters.models import HttpRequest
@pyfilter
def refuse(req: HttpRequest):
return REJECT if "BLOCKME" in (req.url or "") else ACCEPT
"""
@pytest.fixture
def echo():
"""A service that answers with whatever it was sent. Returns its port."""
server = socket.socket()
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", free_port()))
server.listen(16)
def answer(conn):
try:
while data := conn.recv(4096):
conn.sendall(data)
except OSError:
pass
finally:
conn.close()
def serve():
while True:
try:
conn, _ = server.accept()
except OSError:
return
threading.Thread(target=answer, args=(conn,), daemon=True).start()
threading.Thread(target=serve, daemon=True).start()
yield server.getsockname()[1]
server.close()
@pytest.fixture
def simulator(echo, tmp_path):
"""Start `fgex pyfilters` on a filter file in front of `echo`. Returns an asker."""
started = []
def _start(code: str):
path = tmp_path / "filter.py"
path.write_text(code)
port = free_port()
proc = subprocess.Popen(
[sys.executable, FGEX, "pyfilters", str(path), "127.0.0.1", str(echo),
"--from-port", str(port)],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
started.append(proc)
deadline = time.time() + 20
while time.time() < deadline:
if proc.poll() is not None:
pytest.fail(f"fgex pyfilters exited: {proc.stdout.read()}")
try:
socket.create_connection(("127.0.0.1", port), timeout=0.2).close()
break
except OSError:
time.sleep(0.1)
else:
pytest.fail("fgex pyfilters never started listening")
def ask(payload: bytes) -> bytes:
try:
with socket.create_connection(("127.0.0.1", port), timeout=3) as conn:
conn.sendall(payload)
return conn.recv(4096)
except OSError:
return b""
return ask
yield _start
for proc in started:
# Interrupted rather than killed: that is how an operator stops it, and it is the
# path that takes the proxy's own process down with it.
proc.send_signal(signal.SIGINT)
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
def test_a_raw_filter_refuses_what_it_should_and_carries_the_rest(simulator):
ask = simulator(RAW)
assert ask(b"hello") == b"hello"
assert ask(b"carrying BLOCKME") == b""
def test_an_http_filter_reads_requests_and_carries_what_is_not_http(simulator):
ask = simulator(HTTP)
clean = b"GET /fine HTTP/1.1\r\nHost: x\r\n\r\n"
assert ask(clean) == clean
assert ask(b"GET /BLOCKME HTTP/1.1\r\nHost: x\r\n\r\n") == b""
# Not a refusal: `invalid_encoding_action` is ACCEPT unless the file says otherwise.
garbage = b"\x16\x03\x01 not a request\r\n\r\n"
assert ask(garbage) == garbage

2
tests/unit/test_pyfilter_inference.py
View File

@ -198,7 +198,7 @@ def test_a_size_that_is_not_a_size_is_ignored_too():
def test_the_other_two_knobs_want_their_own_enum():
assert settings_of("").full_stream_action is FullStreamAction.FLUSH
assert settings_of("").invalid_encoding_action is ExceptionAction.REJECT
assert settings_of("").invalid_encoding_action is ExceptionAction.ACCEPT
chosen = settings_of("from firegex.pyfilters import FullStreamAction\n"
"FGEX_FULL_STREAM_ACTION = FullStreamAction.REJECT\n")

5
tests/unit/test_rule_round_trip.py
View File

@ -121,7 +121,10 @@ def test_the_nfqueue_layers_rules_are_all_taken_back(tables):
srv = service(TRANSPORT.NFQUEUE)
handle.add(srv, queue_nums=[1000, 1001])
assert len(fake.rules) == 4, "two positions, one pair of rules each"
# Two for traffic arriving, and two for this host's own to the address after a
# container runtime rewrote it — the ones matched through conntrack, which `get()`
# only finds by their comment.
assert len(fake.rules) == 8, "two positions, two pairs of rules each"
handle.delete(srv)
assert fake.rules == [], str(fake.rules)

64
tests/unit/test_service_lifecycle_lock.py
View File

@ -27,6 +27,11 @@ class _Transport:
self.seen["stop"] += 1
await asyncio.sleep(0.05)
async def retire(self, keep_filtering, then=None):
await self.stop()
if then:
then()
class _Quiet:
"""Stands in for the rules, the log and the database: none of them is the question."""
@ -35,6 +40,9 @@ class _Quiet:
pass
def delete(self, *args, **kwargs):
return set()
def release_guards(self, *args, **kwargs):
pass
def query(self, *args, **kwargs):
@ -73,3 +81,59 @@ def test_two_stops_at_once_take_it_down_once(manager):
asyncio.run(run())
assert seen["stop"] == 1, "the second stop tore down a datapath that was already gone"
assert not service.active
class _Rules(_Quiet):
"""The rules, counted: how many times each service was steered, and un-steered."""
def __init__(self):
self.added = 0
self.deleted = 0
def add(self, *args, **kwargs):
self.added += 1
def delete(self, *args, **kwargs):
self.deleted += 1
return set()
def test_rules_are_not_put_back_behind_a_stop(manager, monkeypatch):
"""The table watchdog and a stop, at once.
A stop takes the rules away before the datapath, and is suspended in between. Put
back in that gap, they would outlive the stop — steering traffic at an engine that is
gone — so putting them back waits for the same lock and finds nothing running.
"""
service, _ = manager
rules = _Rules()
monkeypatch.setattr(firewall, "nft", rules)
async def run():
await service.enable()
stopping = asyncio.create_task(service.disable())
await asyncio.sleep(0.01) # inside `disable`, waiting on the datapath
await service.put_rules_back()
await stopping
asyncio.run(run())
assert not service.active
assert rules.added == 1, "the rules were put back behind the stop"
def test_rules_put_back_on_a_running_service_say_so(manager, monkeypatch):
service, _ = manager
rules = _Rules()
monkeypatch.setattr(firewall, "nft", rules)
said = []
service.log = type("Log", (), {"add": lambda self, level, text: said.append((level, text))})()
async def run():
await service.enable()
await service.put_rules_back()
await service.put_rules_back()
asyncio.run(run())
assert rules.added == 3
errors = [text for level, text in said if level == firewall.LEVEL.ERROR]
assert len(errors) == 1, f"said once per quiet period, not once per time: {errors}"

1
tests/unit/test_service_validation.py
View File

@ -308,3 +308,4 @@ def test_the_index_name_is_still_understood_if_it_is_ever_reported():
assert "proxy endpoint" in _address_taken(sqlite3.IntegrityError(
"UNIQUE constraint failed: index 'unique_hijack_target'"))