ad/firegex
1
0
Fork 0
mirror of https://github.com/Av0idD4rk/firegex.git synced 2026-10-23 16:31:00 +00:00
Code Issues Packages Projects Releases Wiki Activity
5319d2cc83
firegex/backend/routers/services.py
Domingo Dirutigliano 896eb2dfb4
backup managment fixes
2026-10-01 12:32:12 +02:00

2570 lines
112 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Services: one network layer, one ordered chain of filters on top of it.
This is the module that replaced `nfregex`, `nfproxy` and `tls`. Those were three ways
of saying the same two things — how to intercept, and what to do — with the two welded
together, so choosing regexes also chose NFQUEUE, and choosing TLS meant creating a
second object and keeping its address in step with the first.
Here a service says only where the traffic is and how to reach it. Filters are attached
to it, in an order the operator chooses, and can be added, reordered, switched off and
switched on without the service being recreated or a single connection being dropped.
"""
import asyncio
import base64
import binascii
import json
import os
import secrets
import sqlite3
import sys
import time
import traceback
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import PlainTextResponse
from pydantic import BaseModel
from modules.services.firewall import (
CODE_DIR,
FirewallManager,
clear_code,
read_code,
write_code,
)
from modules.services import stats
from modules.services.logs import log_for
from modules.services.models import (KIND, L4, MODE, PROTO, STATUS, Service,
TRANSPORT, UPSTREAM)
from modules.services.nftables import FiregexTables, hijack_endpoint, one_address
from modules.services import transports
from modules.services.transports import PROXY_ENGINE, PYWORKER, UnsupportedChain
from utils import (
PortType,
ip_family,
ip_parse,
is_ip_parse,
parse_ip_or_int,
refactor_name,
socketio_emit,
)
from utils.models import ResetRequest, StatusMessageModel
from utils.sqlite import SQLite
def _counters_written() -> None:
"""Every request sees the block counters as they are, and changes them from there.
They are written once a second (`BLOCK_FLUSH` in the manager), not per block — so a
read would otherwise be up to a second behind, and an edit that resets a pattern's
counter would have the old pattern's last blocks written onto the new one after it.
"""
firewall.flush_blocks()
app = APIRouter(dependencies=[Depends(_counters_written)])
db = SQLite(
"db/services.db",
{
"services": {
"service_id": "VARCHAR(100) PRIMARY KEY",
"name": "VARCHAR(100) NOT NULL UNIQUE",
"status": 'VARCHAR(100) NOT NULL CHECK (status IN ("active", "stop"))',
# What the service speaks, which is not always what the kernel matches on:
# `tls` is TCP on the wire and `quic` is UDP. The address carries the second
# answer.
"proto": 'VARCHAR(4) NOT NULL CHECK (proto IN ("tcp", "udp", "tls", "quic", "http"))',
"transport": 'VARCHAR(20) NOT NULL CHECK (transport IN ("nfqueue", "proxy", "external"))',
"fail_open": "BOOLEAN NOT NULL CHECK (fail_open IN (0, 1)) DEFAULT 1",
# How many connections (or UDP flows) this service may carry at once, and
# what happens to the next one. `0` is no limit, which is what every service
# had before there was one — kept as a value rather than as an absence,
# because "unlimited" is a choice an operator can make and should have to.
"max_connections": "INTEGER NOT NULL DEFAULT 0",
# 0: refuse what does not fit. 1: forward it with no filter in front of it.
"over_limit_forwards": "BOOLEAN NOT NULL CHECK (over_limit_forwards IN (0, 1)) DEFAULT 0",
# Seconds a connection may carry nothing in either direction before it is
# closed. 0 is off. Only until the *first* byte: a session that has spoken
# and gone quiet is a session, and sessions are allowed to think.
"first_byte_timeout": "INTEGER NOT NULL DEFAULT 0",
# The trace. A counter in the log would be gone with the next thousand lines,
# and the ring the live log keeps is bounded on purpose; this survives a
# restart, so "did we ever hit the wall" has an answer tomorrow morning.
"over_limit_hits": "INTEGER NOT NULL DEFAULT 0",
#: When it first happened and when it last did, unix seconds. Two instants
#: rather than one, because "it started an hour ago and is still going" and
#: "it happened once at 3am" are different situations.
"over_limit_first": "INT",
"over_limit_last": "INT",
"tls_cert": "TEXT",
"tls_key": "TEXT",
# The first instant this service could refuse anything: running, with a
# filter attached. Statistics start there rather than at the beginning of
# retention, because the hours before a service existed are not quiet hours
# — they are hours the question was not being asked, and drawing them as an
# empty chart says the opposite. Never cleared once set: a service stopped
# and started again has a real gap in the middle, and that is worth seeing.
"filtering_since": "INT",
},
# Where a service is reachable. A list, because one service routinely answers on
# more than one address — a v4 and a v6 one, a public and an internal one, two
# ports of the same daemon — and making the operator create one service per
# address meant keeping their filter chains in step by hand.
"service_addresses": {
"address_id": "VARCHAR(100) PRIMARY KEY",
"service_id": "VARCHAR(100) NOT NULL",
"ip_int": "VARCHAR(100) NOT NULL",
"port": "INT NOT NULL CHECK(port > 0 and port < 65536)",
# What clients speak at this address, in the same words a service speaks
# them: `tls` is TCP with TLS on it, `quic` is UDP with QUIC. For every
# protocol but `http` it is the service's own and nobody is asked; `http` is
# the one that means "each address says", which is what it has always meant.
"edge": 'VARCHAR(4) NOT NULL CHECK (edge IN ("tcp", "udp", "tls", "quic")) '
"DEFAULT 'tcp'",
# Where this address's traffic actually goes, when that is not the port it
# arrived on. NULL is transparent — the address is the service — which is
# what firegex did and only did until an address could be a *publication*:
# a service answering in the clear on :80 reached over TLS on :443 without
# moving. Only the port: the same host, because what is being published is a
# service this instance is already in front of.
"target_port": "INT CHECK(target_port IS NULL OR "
"(target_port > 0 AND target_port < 65536))",
# What the service behind *this address* speaks, which used to be assumed to
# be whatever the client spoke. `same` is that assumption and the default;
# `tcp` and `tls` say the service speaks HTTP/1.1 — in the clear or under its
# own TLS — whatever arrived at the front, which is what puts an ordinary web
# service behind a TLS or HTTP/3 edge with firegex terminating on its behalf.
#
# On the address rather than on the service because that is where it has an
# answer: one daemon reached over TLS on one port and in the clear on another
# is re-encrypted for the first and handed the plaintext for the second.
"upstream": 'VARCHAR(8) NOT NULL CHECK (upstream IN ("same", "tcp", "tls")) '
"DEFAULT 'same'",
# The transport the *kernel* sees, derived from the service's own: `tls` is
# TCP on the wire. Kept here so `(ip, port, proto)` is a usable uniqueness
# key — a TCP service and a UDP one may share an address exactly as the
# kernel allows, while a TCP service and a TLS one may not, because that is
# the same port twice.
"proto": 'VARCHAR(3) NOT NULL CHECK (proto IN ("tcp", "udp"))',
# Where the operator's own proxy is listening for *this* address. Only for
# `external`; loopback when they do not say otherwise, which is where such a
# proxy normally is.
"proxy_ip": "VARCHAR(100)",
"proxy_port": "INT CHECK(proxy_port IS NULL OR (proxy_port > 0 AND proxy_port < 65536))",
"FOREIGN KEY (service_id)": "REFERENCES services (service_id)",
},
# The chain. `position` is what makes it ordered, and the operator sets it.
"filters": {
"filter_id": "VARCHAR(100) PRIMARY KEY",
"service_id": "VARCHAR(100) NOT NULL",
"position": "INT NOT NULL",
"kind": 'VARCHAR(20) NOT NULL CHECK (kind IN ("regex", "pyfilter"))',
# Only meaningful for a pyfilter, and never chosen: read off the code when
# it is saved, and stored so the interface can show it without executing
# anything. The code stays the only thing that decides it.
"proto": 'VARCHAR(10) NOT NULL CHECK (proto IN ("tcp", "http")) DEFAULT "tcp"',
"name": "VARCHAR(100) NOT NULL",
"active": "BOOLEAN NOT NULL CHECK (active IN (0, 1)) DEFAULT 1",
"blocked": "INTEGER UNSIGNED NOT NULL DEFAULT 0",
"FOREIGN KEY (service_id)": "REFERENCES services (service_id)",
},
# One row per `@pyfilter` function a filter's code defines. Derived from the
# code and reconciled with it every time the code is saved — the function list
# is the file's to decide — but `active` and `blocked` are the operator's, and
# survive an edit that leaves the function in place. Turning a function off is
# how you stop consulting it without deleting the code that defines it.
"pyfilters": {
"filter_id": "VARCHAR(100) NOT NULL",
"name": "VARCHAR(100) NOT NULL",
"active": "BOOLEAN NOT NULL CHECK (active IN (0, 1)) DEFAULT 1",
"blocked": "INTEGER UNSIGNED NOT NULL DEFAULT 0",
# Where the function sits in its file. The library runs them in definition
# order, so listing them in any other order — this was `ORDER BY name`, and
# before that the run order itself was a set's — shows the operator a
# sequence that is not the one deciding their traffic.
"position": "INTEGER NOT NULL DEFAULT 0",
"FOREIGN KEY (filter_id)": "REFERENCES filters (filter_id)",
},
# The patterns inside one regex filter. They are compiled together into a
# single hyperscan database, which is why they belong to a filter rather than
# each being one.
"regexes": {
"regex_id": "VARCHAR(100) PRIMARY KEY",
"filter_id": "VARCHAR(100) NOT NULL",
"regex": "TEXT NOT NULL",
"mode": 'VARCHAR(1) NOT NULL CHECK (mode IN ("C", "S", "B"))',
"case_sensitive": "BOOLEAN NOT NULL CHECK (case_sensitive IN (0, 1)) DEFAULT 1",
"active": "BOOLEAN NOT NULL CHECK (active IN (0, 1)) DEFAULT 1",
"blocked": "INTEGER UNSIGNED NOT NULL DEFAULT 0",
"FOREIGN KEY (filter_id)": "REFERENCES filters (filter_id)",
},
# How much each filter refused, in fixed time buckets. Bounded and coalesced —
# see `modules/services/stats.py`, where both properties are the point.
"block_history": {
"service_id": "VARCHAR(100) NOT NULL",
"filter_id": "VARCHAR(100) NOT NULL",
# Whatever the datapath reported: a pattern id, `<filter>/<function>`, or the
# filter itself. Kept at this resolution so that asking for a time range
# answers the same question everywhere on the page — the chart, the totals
# and the shares — instead of the chart honouring it and the table beside it
# quietly reporting all of time.
"rule_id": "VARCHAR(200) NOT NULL",
"bucket": "INTEGER NOT NULL",
"blocked": "INTEGER UNSIGNED NOT NULL DEFAULT 0",
},
"QUERY": [
"CREATE UNIQUE INDEX IF NOT EXISTS unique_block_bucket "
"ON block_history (rule_id, bucket);",
"CREATE INDEX IF NOT EXISTS block_history_by_service "
"ON block_history (service_id, bucket);",
"CREATE UNIQUE INDEX IF NOT EXISTS unique_service_address "
"ON service_addresses (ip_int, port, proto);",
# Two addresses handed to the same proxy endpoint could not be told apart on
# the way back: the return rule recognises the operator's proxy by address
# and port, and puts the original port back. Refusing the collision here is
# cheaper than an intermittently wrong source port in production.
"CREATE UNIQUE INDEX IF NOT EXISTS unique_hijack_target "
"ON service_addresses (proxy_ip, proxy_port) WHERE proxy_port IS NOT NULL;",
"CREATE UNIQUE INDEX IF NOT EXISTS unique_filter_position ON filters (service_id, position);",
"CREATE UNIQUE INDEX IF NOT EXISTS unique_pyfilter ON pyfilters (filter_id, name);",
"CREATE UNIQUE INDEX IF NOT EXISTS unique_regex ON regexes (filter_id, regex, mode, case_sensitive);",
],
},
)
firewall = FirewallManager(db)
class AddressModel(BaseModel):
address_id: str
service_id: str
ip_int: str
port: PortType
proto: str
#: What clients speak here, which is what the operator chose. `proto` is what the
#: kernel matches, and is this with the hat taken off.
edge: str = L4.TCP
#: Where the service is, when this address is a publication rather than the service.
target_port: int | None = None
#: What the service behind this address speaks: `same`, `tcp` or `tls`.
upstream: str = UPSTREAM.SAME
proxy_ip: str | None = None
proxy_port: int | None = None
class AddressForm(BaseModel):
ip_int: str
port: PortType
#: What clients speak here — `tcp`, `tls`, `udp` or `quic`. Only ever set on an
#: `http` service, which is the one whose addresses are not all the same; elsewhere
#: the service's own protocol decides and a value here is refused rather than
#: allowed to disagree with it.
edge: str | None = None
#: Where the service actually is, when it is not on this port. Absent is the
#: transparent case: the address *is* the service.
target_port: int | None = None
#: What the service behind this address speaks. Absent is `same` — whatever arrived
#: at the front goes back out the same way, which is what every address did before
#: there was a choice.
upstream: str | None = None
#: `external` only: where your own proxy listens for this address. Two addresses
#: cannot share one — the return rule tells them apart by it.
proxy_ip: str | None = None
proxy_port: PortType | None = None
class ProblemModel(BaseModel):
"""The newest warning or error in a service's log. See `ServiceLog.last_problem`."""
#: Unix milliseconds, like every log line.
at: int
level: str
text: str
class ServiceModel(BaseModel):
service_id: str
name: str
status: str
proto: str
transport: str
fail_open: bool
#: How many connections or UDP flows may be in flight at once; 0 means no limit.
max_connections: int = 0
#: Whether what does not fit is forwarded unfiltered rather than refused.
over_limit_forwards: bool = False
#: Seconds a connection may say nothing at all before it is closed; 0 means never.
first_byte_timeout: int = 0
#: How many times the limit turned something away, ever. Survives a restart, which
#: is what makes it a trace rather than a reading.
over_limit_hits: int = 0
over_limit_first: int | None = None
over_limit_last: int | None = None
#: Whether a certificate *and* a key are stored for it. The material itself never
#: comes back — the key deliberately, the certificate for symmetry — so this is what
#: lets the form know whether an empty field means "unchanged" or "never set", and
#: ask for one before the service is started rather than after the engine has refused it.
has_tls_material: bool = False
addresses: list[AddressModel] = []
n_filters: int
n_blocked: int
#: The newest warning or error still in its log, so the list can say something is
#: wrong without anybody having the service's page open. Cleared with the log.
problem: ProblemModel | None = None
class ServiceAddForm(BaseModel):
name: str
proto: str = L4.TCP
#: Every address to protect. At least one, and they may mix address families —
#: one service, one chain, however many places it answers.
addresses: list[AddressForm]
transport: str = TRANSPORT.PROXY
fail_open: bool = True
max_connections: int = 0
over_limit_forwards: bool = False
first_byte_timeout: int = 0
tls_cert: str | None = None
tls_key: str | None = None
class ServiceSettingsForm(BaseModel):
name: str | None = None
proto: str | None = None
transport: str | None = None
fail_open: bool | None = None
max_connections: int | None = None
over_limit_forwards: bool | None = None
first_byte_timeout: int | None = None
tls_cert: str | None = None
tls_key: str | None = None
class ServiceAddResponse(BaseModel):
status: str
service_id: str | None = None
class FilterModel(BaseModel):
filter_id: str
service_id: str
position: int
kind: str
proto: str
name: str
active: bool
blocked: int
n_regexes: int
#: How many `@pyfilter` functions the code defines, and how many are switched on.
n_functions: int = 0
n_functions_active: int = 0
class FilterAddForm(BaseModel):
kind: str
name: str | None = None
active: bool = True
class FilterEditForm(BaseModel):
name: str | None = None
active: bool | None = None
class PyFunctionModel(BaseModel):
filter_id: str
#: The `@pyfilter` function's own name, as written in the code.
name: str
active: bool
blocked: int
class PyFunctionEditForm(BaseModel):
active: bool
class ReorderForm(BaseModel):
#: Every filter of the service, in the order they should run.
filters: list[str]
class RegexModel(BaseModel):
regex_id: str
filter_id: str
regex: str
mode: str
case_sensitive: bool
active: bool
blocked: int
class RegexAddForm(BaseModel):
regex: str
mode: str = MODE.BOTH
case_sensitive: bool = True
active: bool = True
class RegexEditForm(BaseModel):
active: bool | None = None
#: base64, like the one on the add form: a pattern is bytes.
regex: str | None = None
mode: str | None = None
case_sensitive: bool | None = None
class CodeForm(BaseModel):
code: str
class StatsEntryModel(BaseModel):
id: str
name: str
kind: str
filter_id: str | None = None
#: True for the one entry per filter that stands for blocks no rule still in it
#: accounts for — a pattern deleted, or one whose text was edited and so gave its
#: history back to the filter. `name` describes it rather than naming a rule, which
#: is why the interface prints it as prose instead of as a pattern.
residual: bool = False
#: Refused in the selected range.
blocked: int
#: Refused since it was created. Only reported for filters, which are the only
#: things carrying a lifetime counter of their own.
all_time: int | None = None
#: What share of everything this service refused is this one's doing, 0–100.
#: Computed against the total rather than left to the browser, so every view of it
#: agrees and a chart cannot disagree with a table beside it.
share: float = 0.0
class TrafficModel(BaseModel):
"""How much arrived, in the units each layer can honestly report.
Packets come from the kernel's own counters on the intercept rules, and are
available on every network layer. Connections come from the proxy engine, which is
the only layer that works in them — and is therefore the only one where the share
of refused traffic can be computed without dividing two different things.
"""
packets: int = 0
bytes: int = 0
connections: int | None = None
connections_refused: int | None = None
#: Percentage of connections refused, 0–100. Null where the layer counts packets.
refused_share: float | None = None
class StatsSeriesModel(BaseModel):
id: str
name: str
counts: list[int]
class StatsModel(BaseModel):
filters: list[StatsEntryModel]
patterns: list[StatsEntryModel]
#: One entry per `@pyfilter` function, across every pyfilter in the chain.
functions: list[StatsEntryModel]
buckets: list[int]
#: How wide one bucket is. Widens with the range, so a chart stays readable.
bucket_seconds: int
series: list[StatsSeriesModel]
#: Refused in the selected range. Every list above counts the same range.
total: int
#: Refused since each rule was created, which is what the cards show. Reported
#: alongside so a narrow range showing nothing is not read as a contradiction.
all_time: int
#: The range actually served, after clamping to what is still kept.
range_from: int
range_to: int
#: The oldest instant anything is known about, so "everything" has a meaning.
kept_from: int
#: When this service first became able to refuse anything, if it ever did. The
#: effective floor of every range, the same way `kept_from` is.
filtering_since: int | None = None
traffic: TrafficModel
class LogEntryModel(BaseModel):
at: int
level: str
text: str
seq: int
class DebugPatternForm(BaseModel):
id: str
expr: str
case_sensitive: bool = True
class DebugForm(BaseModel):
patterns: list[DebugPatternForm]
#: base64, because a pattern is matched against bytes and a useful sample is not
#: always valid text.
sample: str
class DebugMatchModel(BaseModel):
id: str
start: int
end: int
class DebugErrorModel(BaseModel):
id: str
error: str
class DebugResponse(BaseModel):
matches: list[DebugMatchModel]
errors: list[DebugErrorModel]
error: str | None = None
#: Valid, and will run, but cannot be highlighted here.
unscannable: list[DebugErrorModel] = []
truncated: bool
# --- lifecycle hooks the loader calls ----------------------------------------
async def refresh_frontend(extra: list[str] | None = None):
await socketio_emit(["services"] + (extra or []))
async def startup():
db.init()
_prune_orphan_code()
try:
await firewall.init()
except Exception as e:
print("WARNING cannot start the services firewall:", e)
# Whatever came back up is filtering from now, if it was not already marked. This is
# what covers the services that existed before the mark did, and the ones restored
# from a backup: both are running with a chain the moment `init()` returns, and
# neither passes through a start or an edit to say so.
for row in db.query("SELECT service_id FROM services;"):
_note_filtering(row["service_id"])
def _prune_orphan_code() -> None:
"""Remove the filter files no filter refers to.
The code lives beside the database rather than in it, so the two can drift apart, and
an import is where they did: it replaced the database and wrote the backup's files,
but left every file the backup did not have — the code of every Python filter created
since, kept on disk for filters that no longer existed. Done at every start, which is
also what an import ends with, so the directory holds exactly what the database uses.
Only `<filter id>.py` files are touched; anything else in there is not ours to judge.
"""
if not os.path.isdir(CODE_DIR):
return
used = {
row["filter_id"]
for row in db.query("SELECT filter_id FROM filters WHERE kind = ?;", KIND.PYFILTER)
}
for name in os.listdir(CODE_DIR):
stem, ext = os.path.splitext(name)
if ext != ".py" or stem in used:
continue
try:
os.remove(os.path.join(CODE_DIR, name))
except OSError:
traceback.print_exc()
async def shutdown():
await firewall.close()
# The capture interface outlives any one service but not the process: this is the
# only place it goes away, so a `reset()` — which also closes — leaves it alone.
firewall.release_capture()
db.disconnect()
async def reset(params: ResetRequest):
if not params.delete:
# Into the copy that is put back, or the last second of counters is written
# after it was taken and then overwritten by it.
firewall.flush_blocks()
db.backup()
await firewall.close()
FiregexTables().reset()
if params.delete:
db.delete()
db.init()
# The user's Python and the certificates are not in the database, so wiping
# the database alone would leave them behind to be inherited by whatever
# reuses their id.
for name in os.listdir(CODE_DIR) if os.path.isdir(CODE_DIR) else []:
os.remove(os.path.join(CODE_DIR, name))
else:
db.restore()
try:
await firewall.init()
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
def gen_id() -> str:
return secrets.token_hex(8)
def _service_or_404(service_id: str) -> dict:
res = db.query("SELECT * FROM services WHERE service_id = ?;", service_id)
if not res:
raise HTTPException(status_code=404, detail="This service does not exist")
return res[0]
def _addresses(service_id: str) -> list[dict]:
return db.query(
"SELECT * FROM service_addresses WHERE service_id = ? ORDER BY ip_int, port;",
service_id,
)
def _address_or_404(service_id: str, address_id: str) -> dict:
"""An address is addressed through its service, and has to actually be in it."""
res = db.query(
"SELECT * FROM service_addresses WHERE address_id = ? AND service_id = ?;",
address_id,
service_id,
)
if not res:
raise HTTPException(status_code=404, detail="This address does not exist")
return res[0]
def load_service(service_id: str):
"""The service and everywhere it is reachable, as the datapath sees it."""
from modules.services.models import Address
row = _service_or_404(service_id)
srv = Service.from_dict(row)
srv.addresses = [Address.from_dict(a) for a in _addresses(service_id)]
return srv
def _filter_or_404(service_id: str, filter_id: str) -> dict:
"""A filter is addressed through its service, and has to actually be in it.
Without the second half of that check the nesting would be decoration: any filter
id would work under any service, and the chain the caller thought it was editing
would not be the one that changed.
"""
res = db.query(
"SELECT * FROM filters WHERE filter_id = ? AND service_id = ?;", filter_id, service_id
)
if not res:
raise HTTPException(status_code=404, detail="This filter does not exist")
return res[0]
def _regex_or_404(filter_id: str, regex_id: str) -> dict:
res = db.query(
"SELECT * FROM regexes WHERE regex_id = ? AND filter_id = ?;", regex_id, filter_id
)
if not res:
raise HTTPException(status_code=404, detail="This pattern does not exist")
return res[0]
def _note_filtering(service_id: str) -> None:
"""Remember the first moment this service could have refused something.
Two things have to be true — it is running, and there is at least one active filter
in its chain — and neither of them is where the other is changed, so this is called
from both: starting a service, and every edit that goes through `_apply_chain`.
Asking the database is cheaper than keeping a third copy of the answer in step.
Written once and never rewritten. A service that stops filtering and starts again
has a real gap in the middle of its history, and moving the mark forward would hide
it; clearing it would make the chart claim the earlier hours never happened.
"""
manager = firewall.services.get(service_id)
if manager is not None and not manager.active:
# Meant to run and not running — one that could not start at boot, say — has not
# begun filtering, whatever the stored status says.
return
row = db.query(
"SELECT s.filtering_since since, s.status status, "
"(SELECT COUNT(*) FROM filters f WHERE f.service_id = s.service_id AND f.active = 1) n "
"FROM services s WHERE s.service_id = ?;",
service_id,
)
if not row or row[0]["since"] is not None:
return
if row[0]["status"] != STATUS.ACTIVE or int(row[0]["n"]) < 1:
return
# Stored aligned to its bucket, like everything else the statistics work in: the
# mark names the first bucket that can hold one of this service's blocks. An
# unaligned mark is a few seconds *after* the minute-aligned "now" the stats endpoint
# compares it against, which silently switched the floor off for the first minute of
# every service's life — exactly the minute it matters most.
db.query(
"UPDATE services SET filtering_since = ? WHERE service_id = ?;",
stats.bucket_of(),
service_id,
)
async def _apply_chain(service_id: str, undo=None):
"""Push the chain to a running service, translating a refusal into a 400.
A chain the transport cannot host is the operator's mistake, not the server's, and
the message has to say which combination was impossible — an unqualified 500 would
leave them guessing.
`undo` puts the database back. Without it a refused edit would still have been
written: the operator would be told their filter was rejected and then find it in
the list, and the next start would fail for a reason they thought they had avoided.
A refusal has to mean nothing changed.
"""
undo = _once(undo)
try:
await firewall.get(service_id).update_chain(undo)
except UnsupportedChain as e:
undo()
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
undo()
raise HTTPException(status_code=500, detail=str(e))
_note_filtering(service_id)
def _once(undo):
"""`undo`, or nothing, made safe to call from both the manager and the caller.
A rebuild that fails has the manager put the database back before it starts the
service again, and the caller then reaches its own `except` — where the undo it has
always run is now the second one.
"""
done = []
def run():
if undo is not None and not done:
done.append(True)
undo()
return run
# --- services ----------------------------------------------------------------
SERVICE_QUERY = """
SELECT
s.service_id service_id, s.name name, s.status status, s.proto proto,
s.transport transport, s.fail_open fail_open,
s.max_connections max_connections, s.over_limit_forwards over_limit_forwards,
s.first_byte_timeout first_byte_timeout,
s.over_limit_hits over_limit_hits, s.over_limit_first over_limit_first,
s.over_limit_last over_limit_last,
(s.tls_cert IS NOT NULL AND TRIM(s.tls_cert) != ''
AND s.tls_key IS NOT NULL AND TRIM(s.tls_key) != '') has_tls_material,
COUNT(f.filter_id) n_filters,
COALESCE(SUM(f.blocked), 0) n_blocked
FROM services s LEFT JOIN filters f ON s.service_id = f.service_id
"""
def _address_row(row: dict) -> dict:
"""One address, as it is stored.
It used to grow two loopback ports here, because a TLS service was protected on a
pair of them rather than on the address itself. Nothing is derived any more: the
engine decrypts at the address the world dials.
"""
return dict(row)
def _with_addresses(row: dict) -> dict:
row = dict(row)
row["status"] = _running_status(row)
row["addresses"] = [
_address_row(a) for a in _addresses(row["service_id"])
]
row["problem"] = log_for(row["service_id"]).last_problem()
return row
def _running_status(row: dict) -> str:
"""Whether the service is running, not whether it is meant to.
The stored status is what the operator asked for, and what the next boot starts: a
service that could not come back up at boot kept `active` there, and the interface
showed it as active while nothing was filtering it — the one failure this whole module
is arranged to avoid. The reason is in its log, and on the list as its problem.
"""
manager = firewall.services.get(row["service_id"])
if manager is None:
return row["status"]
return STATUS.ACTIVE if manager.active else STATUS.STOP
@app.get("", response_model=list[ServiceModel])
async def get_services():
"""Every service, with how many filters it runs and how much they have blocked."""
return [_with_addresses(row) for row in db.query(SERVICE_QUERY + " GROUP BY s.service_id;")]
class ApiMemberModel(BaseModel):
name: str
doc: str = ""
#: True for the few things a filter may assign to — the payload, and nothing else.
writable: bool = False
signature: str | None = None
class ApiModelModel(BaseModel):
name: str
doc: str = ""
members: list[ApiMemberModel] = []
#: Which application protocols provide it; a file asking for one of these is one of
#: those. `tcp` and `http` both means it runs on anything.
protocols: list[str] = []
class ApiEntryModel(BaseModel):
name: str
doc: str = ""
value: int | None = None
values: list[str] = []
class PyFilterApiModel(BaseModel):
models: list[ApiModelModel]
verdicts: list[ApiEntryModel]
settings: list[ApiEntryModel]
# Declared before `/{service_id}`, or that route matches this path and looks for a
# service called "pyfilter-api".
@app.get("/pyfilter-api", response_model=PyFilterApiModel)
async def get_pyfilter_api():
"""What the filter library offers, for the editor to suggest.
Introspected from the library rather than written down: hints that disagree with the
thing they describe are worse than none, and a second description would disagree the
first time a model gained a property.
"""
from modules.services.pyapi import describe
return describe()
@app.get("/{service_id}", response_model=ServiceModel)
async def get_service(service_id: str):
res = db.query(SERVICE_QUERY + " WHERE s.service_id = ? GROUP BY s.service_id;", service_id)
if not res:
raise HTTPException(status_code=404, detail="This service does not exist")
return _with_addresses(res[0])
def _insert_address(service_id: str, proto: str, transport: str, form: AddressForm) -> str:
"""Add one address, carrying the transport the kernel will match on.
Not the service's own protocol: `tls` is TCP on the wire, and an address storing it
verbatim would let a TCP service and a TLS one claim one `ip:port` between them.
On an `http` service the address says what is spoken at it, because that service is
reached in more than one way at once — `:80` in the clear, `:443` under TLS, a UDP
port over HTTP/3. What it says is kept beside the transport the kernel matches: the
two are the same answer with the hat on and off.
"""
try:
parsed_ip = parse_ip_or_int(form.ip_int)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
edge = _address_edge(proto, form)
target_port = _address_target(transport, form)
upstream = _address_upstream(proto, edge, form)
proxy_ip = _hijack_ip(transport, parsed_ip, form)
address_id = gen_id()
db.query(
"INSERT INTO service_addresses (address_id, service_id, ip_int, port, proto, "
"edge, target_port, upstream, proxy_ip, proxy_port) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
address_id,
service_id,
parsed_ip,
form.port,
L4.l4_of(edge),
edge,
target_port,
upstream,
proxy_ip,
_hijack_port(transport, form),
)
return address_id
def _hijack_ip(transport: str, service_ip: str, form: AddressForm) -> str | None:
"""Where this address's traffic is handed off, stored rather than defaulted later.
Only the hand-off layer has one; everywhere else it stays NULL, because there is no
endpoint and a value would be a setting that does nothing.
Resolved **here**, at the point the row is written, so that the partial unique index
on `(proxy_ip, proxy_port)` can do its job. It could not before: the column was left
NULL whenever the operator did not type an address, SQLite counts every NULL as
distinct in a UNIQUE index, and the rules then defaulted all of them to the same
loopback address — so any number of addresses could sit behind one endpoint, which is
the single thing that index was added to forbid. The return rule tells them apart by
address and port, so it could not.
"""
if str(transport) != TRANSPORT.EXTERNAL:
return None
if form.proxy_ip and str(form.proxy_ip).strip():
# Checked here, where the row is written: it used to be stored as typed and only
# parsed when the rules were built, so a mistyped endpoint was a service that
# would not start, with the reason in its log rather than in this answer.
proxy = str(form.proxy_ip).strip()
if not is_ip_parse(proxy) or _shown(ip_parse(proxy)) != one_address(ip_parse(proxy)):
raise HTTPException(
status_code=400,
detail=f"Your proxy's address has to be one IP address, not {proxy!r}",
)
if is_ip_parse(service_ip) and ip_family(proxy) != ip_family(service_ip):
raise HTTPException(
status_code=400,
detail="Your proxy's address has to be in the same family as the address "
"it stands in for: the rewrite changes the destination and nothing "
"else, so an IPv4 packet cannot be sent to an IPv6 proxy.",
)
form.proxy_ip = proxy
if not form.proxy_port:
return form.proxy_ip or None
return hijack_endpoint(form.proxy_ip, service_ip)
def _hijack_port(transport: str, form: AddressForm) -> int | None:
"""The port half of the same endpoint: the hand-off layer's alone.
Stored for any layer, it was a value read by nothing that still sat in the partial
unique index — so a proxy address that happened to carry one could refuse a real
hand-off its endpoint.
"""
return form.proxy_port if str(transport) == TRANSPORT.EXTERNAL else None
def _address_edge(proto: str, form: AddressForm) -> str:
"""What clients speak at one address of a service.
Every protocol but `http` has one answer for the whole service, so an address of one
is not asked and a value sent anyway is refused rather than quietly kept: a row whose
edge disagrees with its service is a row that would be intercepted on one transport
and served on another.
`http` is the protocol that means "each address says", which is what it meant when
the choice was TCP or UDP and what it still means now that the same address can say
whether it is encrypted. The default is `tcp`, which is what an HTTP address is
unless somebody says otherwise.
"""
asked = str(form.edge) if form.edge else None
if str(proto) != L4.HTTP:
if asked and asked != str(proto):
raise HTTPException(
status_code=400,
detail=f"A {str(proto).upper()} service speaks {str(proto).upper()} at "
f"every address, so this one cannot be {asked!r}. Only an HTTPS "
f"service is reached in more than one way at once.",
)
return str(proto)
if asked is None:
return L4.TCP
if asked not in (L4.TCP, L4.TLS, L4.QUIC):
raise HTTPException(
status_code=400,
detail=f"An address of an HTTPS service is reached in the clear, under TLS "
f"or over QUIC — not {asked!r}.",
)
return asked
def _address_target(transport: str, form: AddressForm) -> int | None:
"""Where the service reached through one address is, when it is not on that port.
Every protocol can say it, because **publishing is being reachable one more way than
the service is** and nothing about that is HTTP's. One daemon on `:8080` is reached
at `:80` in the clear, at `:443` under TLS and at a UDP port over HTTP/3 — and also,
with nothing HTTP about it, at whatever port the round's scoreboard was told about.
It was refused outside `http` while `http` was the only protocol whose addresses were
not all the same, which confused the shape of one protocol with the shape of the
feature.
What it does need is something that **dials**, and that is the layer rather than the
protocol. The proxy terminates the connection and opens the one to the service, so it
is free to open it somewhere else. NFQUEUE hands the kernel a verdict on packets
already on their way and opens nothing; the hand-off layer rewrites the destination to
your own proxy, which then decides for itself where the service is. On both, a stored
port would be read by nothing at all — refused rather than ignored, because an
operator told their service is published and finding it merely intercepted is exactly
the trap this file keeps arguing against.
"""
if not form.target_port:
return None
if str(transport) != TRANSPORT.PROXY:
raise HTTPException(
status_code=400,
detail="Only the proxy layer can send an address's traffic to a port other "
"than the one it arrived on: it terminates the connection and opens "
"the one to the service, so it is free to open it elsewhere. "
+ ("NFQUEUE passes judgement on packets already on their way and opens "
"nothing."
if str(transport) == TRANSPORT.NFQUEUE else
"This layer hands the traffic to your own proxy, which decides for "
"itself where the service is."),
)
if not 0 < form.target_port < 65536:
raise HTTPException(status_code=400, detail="Invalid port for the service")
return form.target_port
def upstream_applies(proto: str, edge: str) -> bool:
"""Whether one address has an upstream leg of its own to decide.
Only where firegex decrypts what arrives: on a cleartext service it terminates
nothing, and on the cleartext address of an HTTPS service the engine carries what
arrived as it arrived — so an answer stored there is read by nothing. It was accepted
there, and the form offered it: "send it to the service over TLS" on a port clients
reach in the clear was saved, shown back as a tag, and ignored by the engine, whose
cleartext path never asks.
"""
if str(proto) not in (L4.TLS, L4.QUIC, L4.HTTP):
return False
return str(edge) != L4.TCP
def _address_upstream(proto: str, edge: str, form: AddressForm) -> str:
"""What the service behind one address speaks.
Only an address whose traffic is decrypted has an upstream leg of its own to decide
(`upstream_applies`). Anywhere else a stored answer would be a setting that does
nothing, and it is refused rather than ignored for that reason.
"""
asked = str(form.upstream) if form.upstream else UPSTREAM.SAME
if asked == UPSTREAM.SAME:
return UPSTREAM.SAME
if asked not in UPSTREAM.ALL:
raise HTTPException(status_code=400, detail=f"Unknown upstream {asked!r}")
if str(proto) not in (L4.TLS, L4.QUIC, L4.HTTP):
raise HTTPException(
status_code=400,
detail=f"A {str(proto).upper()} service is carried as it arrives: firegex "
f"terminates nothing there, so there is no upstream leg to decide. "
f"Only a service firegex decrypts has one.",
)
if not upstream_applies(proto, edge):
raise HTTPException(
status_code=400,
detail="This address is reached in the clear, and firegex carries what arrives "
"there as it arrived — there is no decrypted leg for it to hand the "
"service differently. Only an address reached over TLS or HTTP/3 has one.",
)
return asked
def _address_taken(e: sqlite3.IntegrityError) -> str:
"""Say which uniqueness rule was broken, because they mean different things.
**Told apart by the columns, not by the index name.** SQLite reports a violated
unique index as `UNIQUE constraint failed: <table>.<column>, ...` — it names the
index only for one on an expression — so keying on `unique_hijack_target` matched
nothing, and the sentence below has never once been shown. What an operator saw
instead was "one of these addresses is already protected", which points at the
service address while the thing actually colliding is their proxy's port: they go
looking for a service that does not exist.
The index name is still accepted, because it costs nothing and is what a future
SQLite, or an index written as an expression, might report instead.
"""
said = str(e)
if "proxy_port" in said or "unique_hijack_target" in said:
return (
"another address already hands its traffic to that proxy endpoint. Give "
"this one its own port: the return rule recognises your proxy by address "
"and port to put the original port back, so two addresses behind the same "
"endpoint could not be told apart on the way out."
)
return "one of these addresses is already protected by a service"
def _check_tls_material(cert: str | None, key: str | None) -> None:
"""Refuse TLS material the engine would refuse, at the point it is handed over.
Only the PEM envelope is checked, not the key itself: the aim is to catch the two
mistakes that are made by hand and cost a restart to discover — a DER or PKCS#12
file pasted in whole, and the certificate and the key swapped between the two
fields. Both used to surface as a service that simply would not start, with the
reason in the service log rather than in the answer to the request that caused it.
Anything that carries the right header is passed through untouched: judging a
certificate here by more than its envelope would be a second opinion on what the
engine accepts, and the second opinion is the one that is wrong.
"""
if cert is not None and cert.strip():
if "-----BEGIN CERTIFICATE-----" not in cert:
raise HTTPException(
status_code=400,
detail="That does not look like a PEM certificate: no "
"'BEGIN CERTIFICATE' block in it."
+ (" It contains a private key — the two fields are the other way "
"round." if "PRIVATE KEY-----" in cert else ""),
)
if key is not None and key.strip():
if "PRIVATE KEY-----" not in key:
raise HTTPException(
status_code=400,
detail="That does not look like a PEM private key: no 'BEGIN PRIVATE "
"KEY' block in it."
+ (" It contains a certificate — the two fields are the other way "
"round." if "-----BEGIN CERTIFICATE-----" in key else ""),
)
if "ENCRYPTED PRIVATE KEY-----" in key:
raise HTTPException(
status_code=400,
detail="That private key is passphrase-protected, and nothing here can "
"be asked for the passphrase. Decrypt it first: "
"openssl pkey -in key.pem -out plain.pem",
)
@app.post("", response_model=ServiceAddResponse)
async def add_service(form: ServiceAddForm):
if form.transport not in TRANSPORT.ALL:
raise HTTPException(status_code=400, detail=f"Unknown transport {form.transport!r}")
if form.proto not in L4.ALL:
raise HTTPException(status_code=400, detail=f"Unknown protocol {form.proto!r}")
if not form.addresses:
raise HTTPException(
status_code=400, detail="A service needs at least one address to protect"
)
if form.proto in (L4.TLS, L4.QUIC, L4.HTTP):
if not (form.tls_cert and form.tls_key):
raise HTTPException(
status_code=400,
detail=f"A service that speaks {form.proto.upper()} needs both a "
f"certificate and a private key"
+ (". An HTTP service covers the TLS and HTTP/3 edges as well as "
"the cleartext one, and neither exists without a certificate — "
"a service that only answers in the clear is a `tcp` one, where "
"HTTP/1.1 and HTTP/2 are already filtered."
if form.proto == L4.HTTP else ""),
)
_check_tls_material(form.tls_cert, form.tls_key)
# Each address answers for itself what the service behind it speaks, and is refused
# here rather than at the first start if it cannot. Everything else an address says
# is checked in the same breath, and for a second reason: the rows are written after
# the service's own, so a refusal arriving then would leave a service behind that
# nobody asked for.
for address in form.addresses:
_address_target(form.transport, address)
_address_upstream(form.proto, _address_edge(form.proto, address), address)
# The transport gets a look before the row exists. A chain it could not host is
# caught later, when there is one; what is caught here is what the *service* makes
# impossible on its own — a protocol this layer cannot carry — and it is worth
# catching now, because the alternative is a row that is created happily and then
# refuses to start every time, with the reason arriving one action too late.
try:
transports.build_class(form.transport).check(
Service.from_dict({
"service_id": "", "name": form.name, "status": STATUS.STOP,
"proto": form.proto, "transport": form.transport,
"fail_open": form.fail_open,
}),
[],
)
except UnsupportedChain as e:
raise HTTPException(status_code=400, detail=str(e))
if form.transport == TRANSPORT.EXTERNAL and any(not a.proxy_port for a in form.addresses):
raise HTTPException(
status_code=400,
detail="A service handing its traffic to your own proxy needs that proxy's "
"port for every address",
)
if form.transport == TRANSPORT.EXTERNAL and any(not is_ip_parse(a.ip_int) for a in form.addresses):
raise HTTPException(
status_code=400,
detail="The external transport hands traffic to your own proxy and rewrites the "
"source address on return, which requires a concrete IP address rather than an interface.",
)
# What the rows will hold, worked out before the first one is written: the addresses
# go in after the service, so a refusal half way through left the service and the
# addresses before it behind — holding the name, unknown to the running firewall.
for address in form.addresses:
try:
_hijack_ip(form.transport, parse_ip_or_int(address.ip_int), address)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
service_id = gen_id()
try:
db.query(
"INSERT INTO services (service_id, name, status, proto, transport, "
"fail_open, max_connections, over_limit_forwards, first_byte_timeout, "
"tls_cert, tls_key) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
service_id,
refactor_name(form.name),
STATUS.STOP,
form.proto,
form.transport,
form.fail_open,
max(0, form.max_connections),
form.over_limit_forwards,
max(0, form.first_byte_timeout),
form.tls_cert,
form.tls_key,
)
except sqlite3.IntegrityError as e:
# The name is the only thing an operator can collide with by accident, so it is
# the answer worth giving — but it is not the only constraint on this row, and
# answering "the name exists" to a CHECK that failed sends whoever reads it
# looking for a service that is not there. Ask the exception which it was.
if "name" not in str(e):
return {"status": f"The service was refused by the database: {e}"}
return {"status": "A service with this name already exists"}
try:
for address in form.addresses:
_insert_address(service_id, form.proto, form.transport, address)
except BaseException as e:
# All of it or none: a service that came up on half the addresses the operator
# listed is one they would believe is protecting the other half. Whatever the
# reason — a refusal from a check on the way in used to leave the service and its
# first addresses behind, holding the name and unknown to the running firewall.
db.query("DELETE FROM service_addresses WHERE service_id = ?;", service_id)
db.query("DELETE FROM services WHERE service_id = ?;", service_id)
if isinstance(e, sqlite3.IntegrityError):
return {"status": _address_taken(e)}
raise
await firewall.reload()
await refresh_frontend()
return {"status": "ok", "service_id": service_id}
def _shown(ip_int: str) -> str:
"""An address as an operator typed it: the host-length prefix it is stored with is
dropped, a real range keeps its own."""
addr, _, prefix = str(ip_int).partition("/")
if prefix in ("32", "128"):
return addr
return str(ip_int)
def _address_under(address: dict, proto: str) -> dict:
"""What one address becomes when its service changes protocol.
The addresses carry the transport the kernel matches on, so that `(ip, port, proto)`
can be unique, and what is spoken at them; both follow the service rather than being
a second thing the operator sets. `http` is the one protocol whose addresses keep
their own answer — rewriting them all to `tcp` would quietly turn an HTTP/3 address
into a TCP one — except that an address reached over plain UDP is, on an HTTPS
service, its HTTP/3 edge. Switching *away* from `http` flattens them, which is honest:
the service now speaks one thing, and a collision that follows from it is refused.
An upstream choice survives only where the address still has a decrypted leg for it
to apply to (`upstream_applies`); anywhere else it would be a stored answer read by
nothing.
"""
if str(proto) == L4.HTTP:
edge = str(address["edge"])
if edge not in (L4.TCP, L4.TLS, L4.QUIC):
edge = L4.QUIC if L4.l4_of(edge) == L4.UDP else L4.TCP
else:
edge = str(proto)
upstream = str(address["upstream"] or UPSTREAM.SAME)
if not upstream_applies(proto, edge):
upstream = UPSTREAM.SAME
return {**address, "edge": edge, "proto": L4.l4_of(edge), "upstream": upstream}
async def _filters_needing_a_stream(service_id: str) -> list[str]:
"""The Python filters of a service that could not run on datagrams, by name.
Asked of the checker, which reports the models a file asks for: the protocol stored
on the row says only whether a file parses HTTP, and a file asking for a TCP stream is
stored as `tcp` and would sit on a UDP service never being called.
"""
names = []
for row in db.query(
"SELECT filter_id, name FROM filters WHERE service_id = ? AND kind = ?;",
service_id, KIND.PYFILTER,
):
code = read_code(row["filter_id"])
if not code.strip():
continue
checked = await validate_code(code)
if checked.get("ok") and needs_a_stream(checked.get("models", [])):
names.append(row["name"])
return names
@app.put("/{service_id}", response_model=StatusMessageModel)
async def edit_service(service_id: str, form: ServiceSettingsForm):
"""Change a service's definition.
Anything here changes what the datapath is, not what it enforces, so a running
service is stopped and started around it. That is a visible interruption, which is
why filters and addresses are edited through their own endpoints instead — those
cost at most the connections on the address that changed.
**A refused edit changes nothing**, and three things make that true. The service it
would become is checked before anything is written — the layer against the protocol
and the chain, the way a start would check it, whether or not the service is running;
this used to happen only by starting it, so a stopped service took any combination
and failed on its next start instead. It is written in one transaction, the service
and its addresses together: a protocol change that collided with another service's
address used to leave the service rewritten and its addresses not. And a running
service whose new definition will not start is put back, and started again as it was,
instead of being left stopped on the definition that failed.
"""
row = _service_or_404(service_id)
fields = {}
if form.name is not None:
fields["name"] = refactor_name(form.name)
if form.proto is not None:
if form.proto not in L4.ALL:
raise HTTPException(status_code=400, detail=f"Unknown protocol {form.proto!r}")
fields["proto"] = form.proto
if form.transport is not None:
if form.transport not in TRANSPORT.ALL:
raise HTTPException(status_code=400, detail=f"Unknown transport {form.transport!r}")
fields["transport"] = form.transport
if form.fail_open is not None:
fields["fail_open"] = form.fail_open
if form.max_connections is not None:
if form.max_connections < 0:
raise HTTPException(status_code=400, detail="A limit cannot be negative; 0 means none")
fields["max_connections"] = form.max_connections
if form.over_limit_forwards is not None:
fields["over_limit_forwards"] = form.over_limit_forwards
if form.first_byte_timeout is not None:
if form.first_byte_timeout < 0:
raise HTTPException(status_code=400, detail="A deadline cannot be negative; 0 means none")
fields["first_byte_timeout"] = form.first_byte_timeout
_check_tls_material(form.tls_cert, form.tls_key)
# What has to hold is the state this edit *lands on*, not what it carries: turning
# TLS on for a service that has never been given a certificate is a valid-looking
# request whose only symptom is the engine refusing to start later, on a restart the
# operator has by then attributed to something else. So the stored material counts
# towards it, and a service already holding a certificate can be switched on without
# pasting it again.
lands_on = form.proto if form.proto is not None else row["proto"]
if lands_on in (L4.TLS, L4.QUIC, L4.HTTP):
cert = form.tls_cert if form.tls_cert is not None else row["tls_cert"]
key = form.tls_key if form.tls_key is not None else row["tls_key"]
if not (cert and cert.strip()) or not (key and key.strip()):
missing = "a certificate and a private key" if not (cert and cert.strip()) and not (key and key.strip()) \
else "a certificate" if not (cert and cert.strip()) else "a private key"
raise HTTPException(
status_code=400,
detail=f"This service terminates {str(lands_on).upper()}, so it needs "
f"{missing}. Give it one here — the engine would refuse to start "
f"without it, and the service would fail on its next start rather "
f"than now.",
)
if form.tls_cert is not None:
fields["tls_cert"] = form.tls_cert
if form.tls_key is not None:
fields["tls_key"] = form.tls_key
# Only what actually changes. The form sends every setting on every save, and a field
# that already holds the value it is sent is not an edit: counted as one, opening the
# dialog and saving it untouched restarted a running service and dropped every
# connection it was carrying.
fields = {key: value for key, value in fields.items() if row[key] != value}
if not fields:
return {"status": "ok"}
proto = str(fields.get("proto", row["proto"]))
transport = str(fields.get("transport", row["transport"]))
old_addresses = _addresses(service_id)
new_addresses = [_address_under(address, proto) for address in old_addresses]
if transport != TRANSPORT.EXTERNAL:
# The endpoint of the operator's own proxy is the hand-off layer's alone. Left
# on the rows it is read by nothing and still sits in the unique index, refusing
# another service's hand-off the endpoint this one no longer uses.
new_addresses = [{**address, "proxy_ip": None, "proxy_port": None}
for address in new_addresses]
# What the addresses say has to mean something on the layer they land on. A port the
# service is published on takes a layer that dials; on the others it would be read by
# nothing, and the address would look published while being merely intercepted.
if transport != TRANSPORT.PROXY:
published = [a for a in new_addresses if a["target_port"]]
if published:
raise HTTPException(
status_code=400,
detail="Only the proxy layer can send an address's traffic to a port other "
"than the one it arrived on, and "
+ ", ".join(f"{_shown(a['ip_int'])}:{a['port']}" for a in published)
+ " is published on another port. Take that off the address first.",
)
# The service it would become, asked what a start would ask of the layer and the
# chain. Without its addresses, the way a service is checked when it is created: what
# an address still lacks on the new layer — the port of the operator's own proxy, say
# — is filled in on the address afterwards, and a start says so if it is not.
manager = firewall.get(service_id)
candidate = Service.from_dict({**row, **fields})
try:
transports.build_class(transport).check(candidate, manager.chain())
except UnsupportedChain as e:
raise HTTPException(status_code=400, detail=str(e))
if proto == L4.UDP:
needing = await _filters_needing_a_stream(service_id)
if needing:
raise HTTPException(
status_code=400,
detail=f"{', '.join(needing)} {'ask' if len(needing) > 1 else 'asks'} for a "
f"stream — an assembled TCP stream or a parsed HTTP message — and a "
f"UDP service carries datagrams, so on it the library would never "
f"call {'them' if len(needing) > 1 else 'it'}. Rewrite them against "
f"RawPacket, or keep the service on a stream.",
)
def rewrite(values: dict, addresses: list[dict]) -> list[tuple]:
queries = [(
f"UPDATE services SET {', '.join(f'{k} = ?' for k in values)} WHERE service_id = ?;",
*values.values(),
service_id,
)]
for address in addresses:
queries.append((
"UPDATE service_addresses SET proto = ?, edge = ?, upstream = ?, "
"proxy_ip = ?, proxy_port = ? WHERE address_id = ?;",
address["proto"], address["edge"], address["upstream"],
address["proxy_ip"], address["proxy_port"], address["address_id"],
))
return queries
# All of it or none: a protocol change that collided with another service's address
# used to leave the service rewritten and its addresses as they were.
try:
db.queries(rewrite(fields, new_addresses))
except sqlite3.IntegrityError as e:
if "services.name" in str(e):
raise HTTPException(status_code=400, detail="That name is already taken")
raise HTTPException(
status_code=400,
detail=f"Another service already protects one of these addresses on {proto}",
)
if set(fields) == {"name"}:
# A name is the operator's, not the datapath's: nothing running reads it, so it
# is not a reason to take the service down and bring it back.
manager.srv.name = fields["name"]
await refresh_frontend()
return {"status": "ok"}
was_active = manager.active
try:
await manager.refresh(load_service(service_id))
except Exception as e:
# The new definition would not start. Put the old one back, and the service with
# it, rather than leaving it stopped on the definition that failed — which is what
# happened, with a 500, to a service handed a key its engine would not sign with.
db.queries(rewrite({k: row[k] for k in fields}, old_addresses))
manager.srv = load_service(service_id)
if was_active:
try:
await manager.enable()
except Exception:
traceback.print_exc()
raise HTTPException(
status_code=400 if isinstance(e, UnsupportedChain) else 500,
detail=f"The service would not start with that, so nothing was changed: {e}",
)
await refresh_frontend()
return {"status": "ok"}
# --- where a service is reachable ---------------------------------------------
@app.get("/{service_id}/addresses", response_model=list[AddressModel])
async def get_addresses(service_id: str):
_service_or_404(service_id)
return [_address_row(a) for a in _addresses(service_id)]
@app.post("/{service_id}/addresses", response_model=StatusMessageModel)
async def add_address(service_id: str, form: AddressForm):
"""Protect one more address with this service's existing chain.
Not a restart: the datapath is already running and already enforcing the chain, so
this installs the rules that point the new address at it. The connections on the
other addresses are untouched. The one exception is a proxy service gaining its
first IPv6 address, because the listener has to be reopened in a family that can
accept it — the manager says so and rebuilds.
"""
row = _service_or_404(service_id)
if row["transport"] == TRANSPORT.EXTERNAL and not form.proxy_port:
raise HTTPException(
status_code=400,
detail="This service hands its traffic to your own proxy, so the new "
"address needs the port that proxy listens on for it",
)
if row["transport"] == TRANSPORT.EXTERNAL and not is_ip_parse(form.ip_int):
raise HTTPException(
status_code=400,
detail="The external transport hands traffic to your own proxy and rewrites the "
"source address on return, which requires a concrete IP address rather than an interface.",
)
# Before the row exists: what the address says has to mean something on this service.
_address_upstream(row["proto"], _address_edge(row["proto"], form), form)
try:
address_id = _insert_address(service_id, row["proto"], row["transport"], form)
except sqlite3.IntegrityError as e:
raise HTTPException(status_code=400, detail=_address_taken(e))
@_once
def undo():
# The row *and* the manager's copy of it: the manager re-read the list on its
# way in, so leaving it there would mean the next start protecting an address
# that was refused and no longer exists.
db.query("DELETE FROM service_addresses WHERE address_id = ?;", address_id)
firewall.get(service_id).reload_addresses()
try:
await firewall.get(service_id).address_added(address_id, undo)
except UnsupportedChain as e:
undo()
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
undo()
raise HTTPException(status_code=500, detail=str(e))
await refresh_frontend()
return {"status": "ok"}
@app.put("/{service_id}/addresses/{address_id}", response_model=StatusMessageModel)
async def edit_address(service_id: str, address_id: str, form: AddressForm):
"""Change one address: where it is, and everything it says.
The rules for it are taken back and reinstalled, which is what makes the whole row
rewritable rather than only its address. It used to touch `ip_int`, `port` and the
proxy endpoint alone, so the edge, the port the service is on and what that service
speaks were add-only — and the operator who set one of them while creating the
service had no way back to it except deleting the address and adding it again. That
is a worse answer than reinstalling rules the edit already reinstalls.
Only this address stops being steered while that happens; the rest of the service
keeps its connections. A refusal puts the row back exactly as it was.
"""
row = _service_or_404(service_id)
was = _address_or_404(service_id, address_id)
if row["transport"] == TRANSPORT.EXTERNAL and not form.proxy_port:
raise HTTPException(
status_code=400,
detail="This service hands its traffic to your own proxy, so every address "
"needs the port that proxy listens on for it",
)
if row["transport"] == TRANSPORT.EXTERNAL and not is_ip_parse(form.ip_int):
raise HTTPException(
status_code=400,
detail="The external transport hands traffic to your own proxy and rewrites the "
"source address on return, which requires a concrete IP address rather than an interface.",
)
# A field the caller did not mention keeps what the row already had. This is the
# endpoint that *moves* an address as well as the one that changes what it says, and
# a move sending only an address and a port would otherwise quietly unpublish it.
# Taking one off is a value, not an absence: `0` for the port, `same` for the
# upstream — which is what the interface sends when it is emptied.
upstream_mentioned = form.upstream is not None
if form.edge is None:
form.edge = was["edge"]
if form.target_port is None:
form.target_port = was["target_port"]
if form.upstream is None:
form.upstream = was["upstream"]
# Everything the row says is worked out *before* the old rules come back, so a
# refusal leaves an address that is still steered exactly as it was.
edge = _address_edge(row["proto"], form)
if not upstream_mentioned and not upstream_applies(row["proto"], edge):
# Moved to the clear without a word about the upstream: what the row had was an
# answer for the decrypted leg this address no longer has, and keeping it would
# refuse the very edit that made it meaningless.
form.upstream = UPSTREAM.SAME
target_port = _address_target(row["transport"], form)
upstream = _address_upstream(row["proto"], edge, form)
def write(ip_int, port, proto, edge, target_port, upstream, proxy_ip, proxy_port):
db.query(
"UPDATE service_addresses SET ip_int = ?, port = ?, proto = ?, edge = ?, "
"target_port = ?, upstream = ?, proxy_ip = ?, proxy_port = ? "
"WHERE address_id = ?;",
ip_int, port, proto, edge, target_port, upstream, proxy_ip, proxy_port,
address_id,
)
async def restore():
"""Put the address back exactly as it was, and steer it again.
Best effort by design: the edit has already taken the old rules back, so the
alternative to trying is an address left unprotected because the *new* one was
impossible. A failure here is reported by the exception that caused it.
"""
write(was["ip_int"], was["port"], was["proto"], was["edge"],
was["target_port"], was["upstream"], was["proxy_ip"], was["proxy_port"])
try:
await firewall.get(service_id).address_added(address_id)
except Exception:
pass
try:
parsed_ip = parse_ip_or_int(form.ip_int)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
proxy_ip = _hijack_ip(row["transport"], parsed_ip, form)
manager = firewall.get(service_id)
await manager.address_removed(address_id)
try:
write(parsed_ip, form.port, L4.l4_of(edge), edge, target_port, upstream,
proxy_ip, _hijack_port(row["transport"], form))
except sqlite3.IntegrityError as e:
await restore()
raise HTTPException(status_code=400, detail=_address_taken(e))
try:
# A rebuild failing on the new address stops the service before it finds out;
# the manager puts the old row back and starts it again on that.
await manager.address_added(address_id, lambda: write(
was["ip_int"], was["port"], was["proto"], was["edge"], was["target_port"],
was["upstream"], was["proxy_ip"], was["proxy_port"]))
except UnsupportedChain as e:
await restore()
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
await restore()
raise HTTPException(status_code=500, detail=str(e))
await refresh_frontend()
return {"status": "ok"}
@app.delete("/{service_id}/addresses/{address_id}", response_model=StatusMessageModel)
async def delete_address(service_id: str, address_id: str):
_service_or_404(service_id)
_address_or_404(service_id, address_id)
if len(_addresses(service_id)) == 1:
raise HTTPException(
status_code=400,
detail="This is the only address this service protects. Delete the service "
"itself, or add another address first.",
)
await firewall.get(service_id).address_removed(address_id)
db.query("DELETE FROM service_addresses WHERE address_id = ?;", address_id)
await refresh_frontend()
return {"status": "ok"}
@app.delete("/{service_id}", response_model=StatusMessageModel)
async def delete_service(service_id: str):
_service_or_404(service_id)
await firewall.remove(service_id)
for row in db.query("SELECT filter_id FROM filters WHERE service_id = ?;", service_id):
clear_code(row["filter_id"])
db.query("DELETE FROM regexes WHERE filter_id = ?;", row["filter_id"])
db.query("DELETE FROM pyfilters WHERE filter_id = ?;", row["filter_id"])
db.query("DELETE FROM filters WHERE service_id = ?;", service_id)
db.query("DELETE FROM service_addresses WHERE service_id = ?;", service_id)
stats.forget(db, service_id)
db.query("DELETE FROM services WHERE service_id = ?;", service_id)
await refresh_frontend()
return {"status": "ok"}
@app.post("/{service_id}/start", response_model=StatusMessageModel)
async def start_service(service_id: str):
_service_or_404(service_id)
try:
await firewall.get(service_id).next(STATUS.ACTIVE)
except UnsupportedChain as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
_note_filtering(service_id)
await refresh_frontend()
return {"status": "ok"}
@app.post("/{service_id}/stop", response_model=StatusMessageModel)
async def stop_service(service_id: str):
_service_or_404(service_id)
await firewall.get(service_id).next(STATUS.STOP)
await refresh_frontend()
return {"status": "ok"}
# --- the chain ---------------------------------------------------------------
@app.get("/{service_id}/filters", response_model=list[FilterModel])
async def get_filters(service_id: str):
"""The chain, in the order it runs."""
_service_or_404(service_id)
return db.query(
"SELECT f.filter_id filter_id, f.service_id service_id, f.position position, "
"f.kind kind, f.proto proto, f.name name, f.active active, f.blocked blocked, "
"(SELECT COUNT(*) FROM regexes r WHERE r.filter_id = f.filter_id) n_regexes, "
"(SELECT COUNT(*) FROM pyfilters p WHERE p.filter_id = f.filter_id) n_functions, "
"(SELECT COUNT(*) FROM pyfilters p WHERE p.filter_id = f.filter_id AND p.active = 1) "
"n_functions_active "
"FROM filters f WHERE f.service_id = ? ORDER BY f.position ASC;",
service_id,
)
@app.post("/{service_id}/filters", response_model=StatusMessageModel)
async def add_filter(service_id: str, form: FilterAddForm):
"""Attach a filter to the end of a service's chain."""
_service_or_404(service_id)
if form.kind not in KIND.ALL:
raise HTTPException(status_code=400, detail=f"Unknown filter kind {form.kind!r}")
last = db.query(
"SELECT COALESCE(MAX(position), -1) p FROM filters WHERE service_id = ?;", service_id
)[0]["p"]
filter_id = gen_id()
db.query(
"INSERT INTO filters (filter_id, service_id, position, kind, proto, name, active) "
"VALUES (?, ?, ?, ?, ?, ?, ?);",
filter_id,
service_id,
last + 1,
form.kind,
# A new pyfilter has no code yet, so it asks for nothing and speaks the simplest
# protocol there is. Saving code is what settles it.
PROTO.TCP,
refactor_name(form.name or form.kind),
form.active,
)
if form.kind == KIND.PYFILTER:
write_code(filter_id, "")
def undo():
db.query("DELETE FROM filters WHERE filter_id = ?;", filter_id)
clear_code(filter_id)
await _apply_chain(service_id, undo)
await refresh_frontend()
return {"status": "ok"}
@app.put("/{service_id}/filters/{filter_id}", response_model=StatusMessageModel)
async def edit_filter(service_id: str, filter_id: str, form: FilterEditForm):
was = _filter_or_404(service_id, filter_id)
fields = {}
if form.name is not None:
fields["name"] = refactor_name(form.name)
if form.active is not None:
fields["active"] = form.active
if fields:
db.query(
f"UPDATE filters SET {', '.join(f'{k} = ?' for k in fields)} WHERE filter_id = ?;",
*fields.values(),
filter_id,
)
def undo():
db.query(
f"UPDATE filters SET {', '.join(f'{k} = ?' for k in fields)} WHERE filter_id = ?;",
*[was[k] for k in fields],
filter_id,
)
await _apply_chain(service_id, undo)
await refresh_frontend()
return {"status": "ok"}
@app.delete("/{service_id}/filters/{filter_id}", response_model=StatusMessageModel)
async def delete_filter(service_id: str, filter_id: str):
_filter_or_404(service_id, filter_id)
db.query("DELETE FROM regexes WHERE filter_id = ?;", filter_id)
db.query("DELETE FROM pyfilters WHERE filter_id = ?;", filter_id)
db.query("DELETE FROM filters WHERE filter_id = ?;", filter_id)
# Its history goes with it, or the chart would keep drawing a line for something
# that no longer exists, labelled with an id nobody can look up.
stats.forget_filter(db, filter_id)
clear_code(filter_id)
# Positions must stay contiguous: a gap is invisible until something orders by
# position and gets a different answer than the operator saw.
_renumber(service_id)
await _apply_chain(service_id)
await refresh_frontend()
return {"status": "ok"}
def _renumber_to(order: list[str]):
"""Write an explicit order. Two passes because `position` is unique per service:
writing the final numbers directly would collide with the ones not yet moved."""
for offset, filter_id in enumerate(order):
db.query("UPDATE filters SET position = ? WHERE filter_id = ?;", -1000 - offset, filter_id)
for position, filter_id in enumerate(order):
db.query("UPDATE filters SET position = ? WHERE filter_id = ?;", position, filter_id)
def _renumber(service_id: str):
rows = db.query(
"SELECT filter_id FROM filters WHERE service_id = ? ORDER BY position ASC;", service_id
)
# Out of the way first: `position` is unique per service, so writing the final
# numbers directly would collide with the ones not yet moved.
for offset, row in enumerate(rows):
db.query(
"UPDATE filters SET position = ? WHERE filter_id = ?;",
-1000 - offset,
row["filter_id"],
)
for position, row in enumerate(rows):
db.query(
"UPDATE filters SET position = ? WHERE filter_id = ?;", position, row["filter_id"]
)
@app.post("/{service_id}/filters/order", response_model=StatusMessageModel)
async def reorder_filters(service_id: str, form: ReorderForm):
"""Set the order the chain runs in.
The whole list at once rather than a move: a partial reorder has to be reconciled
against whatever the operator's page last saw, and two tabs doing that at once
produce an order neither of them asked for.
"""
_service_or_404(service_id)
existing = [
r["filter_id"]
for r in db.query("SELECT filter_id FROM filters WHERE service_id = ?;", service_id)
]
if sorted(existing) != sorted(form.filters):
raise HTTPException(
status_code=400,
detail="The order must list exactly the filters this service has",
)
previous = {
r["filter_id"]: r["position"]
for r in db.query(
"SELECT filter_id, position FROM filters WHERE service_id = ?;", service_id
)
}
_renumber_to(form.filters)
def undo():
_renumber_to(sorted(previous, key=lambda fid: previous[fid]))
await _apply_chain(service_id, undo)
await refresh_frontend()
return {"status": "ok"}
# --- a pyfilter's code -------------------------------------------------------
class CodeCheckError(BaseModel):
type: str
message: str
#: 1-based, or 0 when nothing in the file could be pointed at.
line: int = 0
column: int = 0
#: The offending source line, so the reason can be shown without a second lookup.
text: str = ""
traceback: str = ""
class CodeCheckResult(BaseModel):
ok: bool
#: Which application protocol the code speaks, when it loads.
proto: str | None = None
#: The `@pyfilter` functions it defines, in the order the library found them.
filters: list[str] = []
#: The models it asks for, which is what decides where it can run.
models: list[str] = []
error: CodeCheckError | None = None
@app.post("/{service_id}/filters/{filter_id}/check", response_model=CodeCheckResult)
async def check_filter_code(service_id: str, filter_id: str, form: CodeForm):
"""Would this code load? Answered without saving it.
Always 200, even when the answer is no: a refusal here is the result, not a failure
of the request, and the editor wants a position to put a marker on rather than a
string to print.
"""
row = _filter_or_404(service_id, filter_id)
if row["kind"] != KIND.PYFILTER:
raise HTTPException(status_code=400, detail="This filter is not a pyfilter")
return await validate_code(form.code)
@app.get("/{service_id}/filters/{filter_id}/code", response_class=PlainTextResponse)
async def get_filter_code(service_id: str, filter_id: str):
"""The user's Python. Plain text, not JSON: it is a file, not a value."""
row = _filter_or_404(service_id, filter_id)
if row["kind"] != KIND.PYFILTER:
raise HTTPException(status_code=400, detail="This filter is not a pyfilter")
return read_code(filter_id)
@app.put("/{service_id}/filters/{filter_id}/code", response_model=StatusMessageModel)
async def set_filter_code(service_id: str, filter_id: str, form: CodeForm):
"""Replace a pyfilter's code, and record which protocol it now speaks.
The protocol is not asked for and cannot be set: the library reads it off the
filters' parameter annotations, because that is already what decides when each one
is called. Asking would mean two places that can disagree, and the symptom of a
disagreement is a filter that silently never runs — or, as it happened here, one
refused for asking for an `HttpRequest`, which is the only thing an HTTP filter does.
A file whose filters want two *different* application protocols is refused, naming
both: a connection is only ever one of them.
"""
row = _filter_or_404(service_id, filter_id)
if row["kind"] != KIND.PYFILTER:
raise HTTPException(status_code=400, detail="This filter is not a pyfilter")
# Refused here, with the reason and the line, rather than by the datapath a moment
# later with "the filter code did not load: worker exited" — which is true, useless,
# and leaves the operator reading a traceback out of the service log.
checked = await validate_code(form.code)
if not checked.get("ok"):
raise HTTPException(status_code=400, detail=describe_error(checked["error"]))
service = _service_or_404(service_id)
if str(service["proto"]) == L4.UDP:
# Refused rather than accepted and never called: on a datagram these models
# decline to be built, so the filter would sit in the chain doing nothing and
# nothing would say so.
wanted = needs_a_stream(checked.get("models", []))
if wanted:
raise HTTPException(
status_code=400,
detail=f"This service speaks UDP, and {', '.join(wanted)} needs a TCP "
f"stream underneath — an assembled stream or a parsed HTTP "
f"message. On datagrams the library would decline to build it and "
f"the filter would never be called. Ask for a RawPacket instead, "
f"which carries the datagram and its metadata.",
)
proto = checked.get("proto", PROTO.TCP)
was = read_code(filter_id)
was_functions = db.query(
"SELECT name, active, blocked, position FROM pyfilters WHERE filter_id = ?;", filter_id
)
write_code(filter_id, form.code)
db.query("UPDATE filters SET proto = ? WHERE filter_id = ?;", proto, filter_id)
_reconcile_functions(filter_id, checked.get("filters") or [])
def undo():
write_code(filter_id, was)
db.query("UPDATE filters SET proto = ? WHERE filter_id = ?;", row["proto"], filter_id)
db.query("DELETE FROM pyfilters WHERE filter_id = ?;", filter_id)
for fn in was_functions:
# With its place in the file, or a refused edit would hand back the list in
# an order the file does not have.
db.query(
"INSERT INTO pyfilters (filter_id, name, active, blocked, position) "
"VALUES (?, ?, ?, ?, ?);",
filter_id, fn["name"], fn["active"], fn["blocked"], fn["position"],
)
await _apply_chain(service_id, undo)
await refresh_frontend()
return {"status": "ok"}
def _reconcile_functions(filter_id: str, defined: list[str]) -> None:
"""Line the stored function list up with what the code now defines.
The code decides which functions exist; the operator decides which of them run. So a
function that has gone from the file is dropped, one that has appeared is added
switched on, and one that is still there keeps whatever the operator had set — an
edit elsewhere in the file must not silently switch a function back on.
`defined` is what `pyworker.py --check` found, in definition order: the file is not
run again here to ask. It used to be — `exec`ed a second time inside the backend's
own event loop, after the checker had already answered the same question in a process
of its own, which is exactly the place the checker exists to keep the operator's
module body out of.
"""
existing = {
row["name"] for row in db.query(
"SELECT name FROM pyfilters WHERE filter_id = ?;", filter_id
)
}
for name in existing - set(defined):
db.query("DELETE FROM pyfilters WHERE filter_id = ? AND name = ?;", filter_id, name)
for position, name in enumerate(defined):
if name in existing:
# A survivor keeps its `active` and its counters, but not its old place: a
# function inserted above it moved it down in the file, and the file is what
# the order means.
db.query(
"UPDATE pyfilters SET position = ? WHERE filter_id = ? AND name = ?;",
position, filter_id, name,
)
else:
db.query(
"INSERT INTO pyfilters (filter_id, name, position) VALUES (?, ?, ?);",
filter_id, name, position,
)
#: How long the operator's code gets to load before the check gives up on it.
CHECK_TIMEOUT = 8
async def _run_helper(argv: list[str], stdin: bytes | None,
timeout: float) -> tuple[int, bytes, bytes] | None:
"""Run one of the helper processes without holding up everything else.
The checker and the regex tester are separate processes on purpose, and they were
waited for with `subprocess.run` from inside `async` endpoints — which blocks the one
event loop the whole backend runs on. The editor asks for a check 700 ms after every
pause in typing, and a check is a Python start and an import at the very least, up to
`CHECK_TIMEOUT` for slow module code: for that long no acknowledgement from a datapath
was read (so a ruleset push or a service start could time out), and no `BLOCKED` line
either, until the engine's pipe filled and its writes stalled the traffic itself.
`None` when it ran out of time, and it is killed rather than left behind.
"""
proc = await asyncio.create_subprocess_exec(
*argv,
stdin=asyncio.subprocess.PIPE if stdin is not None else asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await asyncio.wait_for(proc.communicate(stdin), timeout)
except (asyncio.TimeoutError, TimeoutError):
proc.kill()
await proc.wait()
return None
return proc.returncode, stdout, stderr
async def validate_code(code: str) -> dict:
"""Would this filter load, and if not, exactly where does it go wrong?
Asked of `pyworker.py --check`, which is the process that will run the filter — the
same build, the same compile, the same library. A stand-in that merely parsed the
file would accept things the datapath refuses, which is the class of lie the regex
tester exists to avoid.
In a subprocess, and not only for the answer: the module body of a filter is
arbitrary code, and this used to be `exec`ed inside the backend's own event loop. A
`while True:` at module level would have taken the whole interface down with it.
Here it costs a timeout — the check's own, not the backend's (see `_run_helper`).
"""
import tempfile
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
f.write(code)
probe = f.name
try:
ran = await _run_helper([sys.executable or "python3", PYWORKER, "--check", probe],
None, CHECK_TIMEOUT)
finally:
try:
os.remove(probe)
except OSError:
pass
if ran is None:
return {
"ok": False,
"error": {
"type": "Timeout", "line": 0, "column": 0, "text": "",
"message": f"the file did not finish loading within {CHECK_TIMEOUT}s. "
f"Code at module level runs once per connection, so it has to "
f"return — a loop or a blocking call there never will.",
"traceback": "",
},
}
_, stdout, stderr = ran
try:
return json.loads(stdout.decode())
except (ValueError, UnicodeDecodeError):
# The checker itself failed, which is ours to explain rather than theirs.
detail = stderr.decode(errors="replace").strip() or "the check produced no answer"
return {"ok": False, "error": {"type": "CheckFailed", "message": detail,
"line": 0, "column": 0, "text": "",
"traceback": ""}}
#: The one model that does not need a connection under it.
#:
#: Every other model reaches for something only TCP has — an assembled stream, a parsed
#: HTTP message — and answers `NotReadyToRun` on a datagram, which means the filter is
#: never called. Stated here rather than inferred because the library expresses it by
#: declining at runtime, and a filter that silently never fires is the failure this
#: whole module is arranged to prevent.
DATAGRAM_SAFE_MODELS = {"RawPacket"}
def needs_a_stream(models: list[str]) -> list[str]:
return sorted(set(models) - DATAGRAM_SAFE_MODELS)
def describe_error(error: dict) -> str:
"""The one-line form, for a caller that has nowhere to put a marker."""
where = f" (line {error['line']})" if error.get("line") else ""
return f"{error.get('type', 'Error')}{where}: {error.get('message', '')}"
# --- the functions inside one pyfilter ---------------------------------------
@app.get("/{service_id}/filters/{filter_id}/functions", response_model=list[PyFunctionModel])
async def get_functions(service_id: str, filter_id: str):
"""Every `@pyfilter` the code defines, and whether it is switched on."""
row = _filter_or_404(service_id, filter_id)
if row["kind"] != KIND.PYFILTER:
raise HTTPException(status_code=400, detail="This filter is not a pyfilter")
return db.query(
"SELECT filter_id, name, active, blocked FROM pyfilters "
"WHERE filter_id = ? ORDER BY position ASC, name ASC;",
filter_id,
)
@app.put(
"/{service_id}/filters/{filter_id}/functions/{name}", response_model=StatusMessageModel
)
async def edit_function(service_id: str, filter_id: str, name: str, form: PyFunctionEditForm):
"""Switch one function of a pyfilter on or off.
The code is untouched: only the list of names handed to the library changes, which
is exactly what decides whether a function is ever called. Deleting the code to stop
consulting it, and pasting it back to resume, is the thing this exists to avoid.
"""
row = _filter_or_404(service_id, filter_id)
if row["kind"] != KIND.PYFILTER:
raise HTTPException(status_code=400, detail="This filter is not a pyfilter")
found = db.query(
"SELECT active FROM pyfilters WHERE filter_id = ? AND name = ?;", filter_id, name
)
if not found:
raise HTTPException(status_code=404, detail="This filter defines no such function")
db.query(
"UPDATE pyfilters SET active = ? WHERE filter_id = ? AND name = ?;",
form.active, filter_id, name,
)
await _apply_chain(service_id, lambda: db.query(
"UPDATE pyfilters SET active = ? WHERE filter_id = ? AND name = ?;",
found[0]["active"], filter_id, name))
await refresh_frontend()
return {"status": "ok"}
# --- a regex filter's patterns -----------------------------------------------
@app.get("/{service_id}/filters/{filter_id}/regexes", response_model=list[RegexModel])
async def get_regexes(service_id: str, filter_id: str):
_filter_or_404(service_id, filter_id)
return db.query("SELECT * FROM regexes WHERE filter_id = ?;", filter_id)
def _decoded_pattern(encoded: str) -> str:
"""The pattern a request carries, or a 400 saying why it is not one.
Two things the obvious `b64decode(...)` does not do, and both of them matter because
what gets through here is compiled into a live service's chain.
**Strict decoding.** Python's decoder ignores characters outside the alphabet unless
it is asked not to, so `"!!!"` is not an error — it is `b""`. A malformed field
therefore arrived as an empty pattern rather than as a refusal.
**And an empty pattern is refused.** hyperscan compiles it happily (the flags allow a
pattern that matches an empty buffer, which real patterns like `a*` need) and then it
matches every buffer it is shown — so a service whose chain holds one refuses every
connection it carries. There is no legitimate way to ask for that, an empty field in
the form is the only way to arrive at it, and the symptom is a CTF service that has
silently stopped answering. It is worth one sentence at the door.
"""
try:
# Whitespace out first, then strict: a base64 field that arrived wrapped across
# lines is still a perfectly good one, and `validate=True` would refuse it for a
# newline. What is being caught here is a field that is not base64 at all.
raw = base64.b64decode("".join(str(encoded).split()), validate=True)
except (binascii.Error, ValueError):
raise HTTPException(status_code=400, detail="The pattern must be base64-encoded")
if not raw:
raise HTTPException(
status_code=400,
detail="A pattern cannot be empty: an empty one matches every byte of every "
"connection, so the service would refuse all of its traffic.",
)
return raw.decode(errors="replace")
@app.post("/{service_id}/filters/{filter_id}/regexes", response_model=StatusMessageModel)
async def add_regex(service_id: str, filter_id: str, form: RegexAddForm):
row = _filter_or_404(service_id, filter_id)
if row["kind"] != KIND.REGEX:
raise HTTPException(status_code=400, detail="This filter does not hold patterns")
if form.mode not in MODE.ALL:
raise HTTPException(status_code=400, detail=f"Unknown mode {form.mode!r}")
pattern = _decoded_pattern(form.regex)
# Checked by the engine that will run it, not by a stand-in. A pattern accepted
# here and refused at start time would leave the operator with a service that
# will not come up and no idea which rule is at fault.
ok, why = await check_pattern(pattern, form.case_sensitive)
if not ok:
raise HTTPException(status_code=400, detail=f"Invalid pattern: {why}")
try:
regex_id = gen_id()
db.query(
"INSERT INTO regexes (regex_id, filter_id, regex, mode, case_sensitive, active) "
"VALUES (?, ?, ?, ?, ?, ?);",
regex_id,
filter_id,
form.regex,
form.mode,
form.case_sensitive,
form.active,
)
except sqlite3.IntegrityError:
raise HTTPException(status_code=400, detail="This pattern is already in this filter")
await _apply_chain(service_id, lambda: db.query(
"DELETE FROM regexes WHERE regex_id = ?;", regex_id))
await refresh_frontend()
return {"status": "ok"}
@app.put("/{service_id}/filters/{filter_id}/regexes/{regex_id}", response_model=StatusMessageModel)
async def edit_regex(service_id: str, filter_id: str, regex_id: str, form: RegexEditForm):
"""Change a pattern in place, without dropping the connections it is matching.
The same rule keeps its row, so it keeps its place and its counters — except when the
*pattern itself* changes. Then its history stops being its own: a count is about what
a rule matched, and carrying yesterday's blocks over to a matcher that now matches
something else would credit them to something that never made them. It is handed to
the filter rather than deleted, so the timeline keeps its shape — those connections
were refused, and correcting a typo afterwards does not unrefuse them.
"""
_filter_or_404(service_id, filter_id)
was = _regex_or_404(filter_id, regex_id)
fields: dict = {}
if form.active is not None:
fields["active"] = form.active
if form.mode is not None:
if form.mode not in MODE.ALL:
raise HTTPException(status_code=400, detail=f"Unknown mode {form.mode!r}")
fields["mode"] = form.mode
if form.case_sensitive is not None:
fields["case_sensitive"] = form.case_sensitive
if form.regex is not None:
fields["regex"] = form.regex
if not fields:
return {"status": "ok"}
# Whatever the row will hold once this is applied, checked as a whole rather than
# field by field: what the engine has to accept is the resulting pattern.
pattern = _decoded_pattern(fields.get("regex", was["regex"]))
ok, why = await check_pattern(pattern, fields.get("case_sensitive", was["case_sensitive"]))
if not ok:
raise HTTPException(status_code=400, detail=f"Invalid pattern: {why}")
changed_matcher = "regex" in fields and fields["regex"] != was["regex"]
if changed_matcher:
fields["blocked"] = 0
try:
db.query(
f"UPDATE regexes SET {', '.join(f'{k} = ?' for k in fields)} WHERE regex_id = ?;",
*fields.values(),
regex_id,
)
except sqlite3.IntegrityError:
raise HTTPException(
status_code=400, detail="This filter already holds that exact pattern"
)
def undo():
db.query(
f"UPDATE regexes SET {', '.join(f'{k} = ?' for k in fields)} WHERE regex_id = ?;",
*[was[k] for k in fields],
regex_id,
)
await _apply_chain(service_id, undo)
# After the chain has accepted it, and not before: a refusal has to leave the rule
# exactly as it was, and history moved ahead of the verdict could not be put back.
if changed_matcher:
stats.disown_rule(db, filter_id, regex_id)
await refresh_frontend()
return {"status": "ok"}
@app.delete("/{service_id}/filters/{filter_id}/regexes/{regex_id}", response_model=StatusMessageModel)
async def delete_regex(service_id: str, filter_id: str, regex_id: str):
_filter_or_404(service_id, filter_id)
_regex_or_404(filter_id, regex_id)
db.query("DELETE FROM regexes WHERE regex_id = ?;", regex_id)
await _apply_chain(service_id)
await refresh_frontend()
return {"status": "ok"}
# --- what the service is doing -----------------------------------------------
@app.get("/{service_id}/stats", response_model=StatsModel)
async def get_stats(
service_id: str,
range_from: int | None = None,
range_to: int | None = None,
buckets: int = 60,
step: int | None = None,
):
"""What this service's filters have refused, over a range you choose.
`range_from`/`range_to` are unix seconds; leaving them out means the last hour.
Everything in the answer counts the same range — the chart, the per-filter,
per-pattern and per-function totals, and the shares — because a page where the chart
honours the range and the table beside it does not is a page that contradicts itself.
The bucket widens with the range rather than the count growing: twelve hours at
one-minute resolution is seven hundred bars in a few hundred pixels. `step` overrides
that with a width in seconds, for an operator who wants two ranges to be comparable
by eye; it is rounded to whole stored buckets and widened if it would draw more bars
than a chart can carry, and `bucket_seconds` reports what was actually used.
"""
service = _service_or_404(service_id)
buckets = max(5, min(int(buckets), 400))
# The real instant, not the bucket it falls in: the window is aligned to buckets
# inside `stats`, and truncating here would report the current, partial minute as
# ending at the moment it began — which on a service a few seconds old prints as
# "23:35 → 23:35".
now = int(time.time())
kept_from = stats.retention_start()
began = service.get("filtering_since")
end = min(int(range_to) if range_to else now, now)
start = int(range_from) if range_from else end - 3600
if end < start:
start, end = end, start # asked for the wrong way round
# Clamped rather than refused: asking for a week of a two-day history is a reasonable
# thing to do, and so is asking for a day of a service that has been up for ten
# minutes. The answer is what there is, with the range actually served reported back
# so the interface can say what it drew.
start = max(start, kept_from)
# The second floor, and the one that bites every day: without it a new service is a
# chart of twenty-three empty hours with a smudge at the right-hand edge, which reads
# as "nothing is happening" when what it means is "this only started existing a
# moment ago". Only when the window reaches into the service's life at all — a window
# that ended before it began is answered as asked, and is empty because it is, rather
# than being dragged forward onto minutes nobody asked about.
if began is not None and end >= began:
start = max(start, began)
window = stats.window(db, service_id, start, end, buckets, step if step else None)
counted = stats.totals(db, service_id, start, end)
by_filter = stats.totals_by_filter(db, service_id, start, end)
total = sum(counted.values())
def with_share(rows: list[dict]) -> list[dict]:
for row in rows:
row["share"] = round(100 * int(row["blocked"]) / total, 1) if total else 0.0
return rows
filters = db.query(
"SELECT filter_id id, name, kind, blocked all_time FROM filters "
"WHERE service_id = ? ORDER BY position ASC;",
service_id,
)
for row in filters:
# Summed over the rules inside it, not looked up by its own id: a regex block is
# reported by its pattern and a Python one by its function, so a filter's own
# token appears only when neither named itself.
row["blocked"] = by_filter.get(row["id"], 0)
patterns = db.query(
"SELECT r.regex_id id, r.regex name, f.kind kind, r.filter_id filter_id "
"FROM regexes r JOIN filters f ON r.filter_id = f.filter_id WHERE f.service_id = ?;",
service_id,
)
for row in patterns:
row["blocked"] = counted.get(row["id"], 0)
# The pattern is stored base64 because it is bytes; a chart legend wants text.
try:
row["name"] = base64.b64decode(row["name"]).decode(errors="replace")
except (binascii.Error, ValueError):
pass
functions = db.query(
"SELECT p.name name, 'pyfilter' kind, p.filter_id filter_id FROM pyfilters p "
"JOIN filters f ON p.filter_id = f.filter_id WHERE f.service_id = ?;",
service_id,
)
for row in functions:
# A function's name is unique inside its file, not across the chain, so the
# token the datapath reports is what identifies it here too.
row["id"] = f"{row['filter_id']}/{row['name']}"
row["blocked"] = counted.get(row["id"], 0)
# What a filter was credited with that no rule still in it accounts for: a pattern
# that was deleted, or one whose text was edited and so handed its history to the
# filter. Those blocks are in the chart and in the total because they happened, so
# the breakdown has to name them — otherwise it quietly adds up to less than the
# chart above it, and the operator is left looking for the difference.
accounted: dict[str, int] = {}
for row in patterns + functions:
accounted[row["filter_id"]] = accounted.get(row["filter_id"], 0) + int(row["blocked"])
for row in filters:
left = int(row["blocked"]) - accounted.get(row["id"], 0)
if left <= 0:
continue
is_py = row["kind"] == KIND.PYFILTER
(functions if is_py else patterns).append({
"id": f"{row['id']}/",
"name": "functions since removed" if is_py else "patterns since edited or removed",
"kind": row["kind"],
"filter_id": row["id"],
"blocked": left,
"all_time": None,
"residual": True,
})
named = {row["id"]: row["name"] for row in filters}
manager = None
try:
manager = firewall.get(service_id)
except Exception:
pass
traffic = manager.traffic() if manager else {}
seen, refused = traffic.get("connections"), traffic.get("connections_refused")
return {
"filters": with_share(filters),
"patterns": with_share(sorted(patterns, key=lambda r: -r["blocked"])),
"functions": with_share(sorted(functions, key=lambda r: -r["blocked"])),
"buckets": window["edges"],
"bucket_seconds": window["width"],
# Only the filters that actually refused something in the range: an empty line
# per filter would make a busy chain unreadable to say nothing.
"series": [
{"id": fid, "name": named.get(fid, fid), "counts": counts}
for fid, counts in window["series"].items()
],
"total": total,
"all_time": sum(int(row["all_time"]) for row in filters),
"range_from": start,
"range_to": end,
"kept_from": kept_from,
"filtering_since": began,
"traffic": {
**traffic,
# Only where both numbers are connections. Dividing refused connections by
# a packet count would produce a number that looks like a percentage and
# means nothing.
"refused_share": (
round(100 * refused / seen, 2) if seen and refused is not None else None
),
},
}
@app.get("/{service_id}/logs", response_model=list[LogEntryModel])
async def get_logs(service_id: str):
"""The tail of what this service has been doing.
Only the tail: new lines arrive over the `log` socket event as they happen, so this
exists to fill the panel when it opens rather than to be polled. It is bounded, so a
service that has blocked a million connections still answers instantly.
"""
_service_or_404(service_id)
return log_for(service_id).entries()
@app.delete("/{service_id}/logs", response_model=StatusMessageModel)
async def clear_logs(service_id: str):
_service_or_404(service_id)
log_for(service_id).clear()
# The list shows the newest problem in the log, and clearing is how it is dismissed.
await refresh_frontend()
return {"status": "ok"}
# --- the debugger ------------------------------------------------------------
async def _ask_engine(request: dict) -> dict:
"""Put the question to the binary that would enforce the answer."""
try:
ran = await _run_helper([PROXY_ENGINE, "--debug-regex"],
json.dumps(request).encode(), 10)
except FileNotFoundError:
raise HTTPException(status_code=500, detail="The matching engine is not installed")
if ran is None:
raise HTTPException(status_code=400, detail="The pattern took too long to evaluate")
returncode, stdout, stderr = ran
if returncode != 0:
detail = stderr.decode(errors="replace").strip() or "the engine refused the request"
raise HTTPException(status_code=400, detail=detail)
return json.loads(stdout.decode())
async def check_pattern(pattern: str, case_sensitive: bool = True) -> tuple[bool, str]:
"""Would the engine accept this pattern?
Asked of the engine that will run it rather than of a stand-in, and compiled for
stream matching, which is the only mode anything runs in now: a pattern blocks, and
blocking follows the stream across chunk boundaries. This used to take the action
too, because rewriting compiled in block mode and the two do not accept quite the
same patterns.
`case_sensitive` is coerced rather than trusted: a caller checking a pattern it just
read back from the database hands over SQLite's 1, and the engine's request format
is strict about the difference — which came out as "invalid type: integer 1" against
a pattern that was perfectly fine.
"""
try:
res = await _ask_engine(
{
"patterns": [
{
"id": "x",
"expr": pattern,
"case_sensitive": bool(case_sensitive),
}
],
"sample": "",
}
)
except HTTPException as e:
return False, str(e.detail)
if res.get("error"):
return False, res["error"]
if res.get("errors"):
return False, res["errors"][0]["error"]
return True, "ok"
@app.post("/debug-regex", response_model=DebugResponse)
async def debug_regexes(form: DebugForm):
"""Try patterns against a sample, with the engine that will run them.
Deliberately not a `re`-based approximation. hyperscan takes a subset of PCRE and
rejects things Python accepts, so anything else would bless patterns that cannot
be saved and disagree about what matches — a tester the operator would learn not
to trust, which is worse than none.
"""
res = await _ask_engine(
{
"patterns": [
{
"id": p.id,
"expr": p.expr,
"case_sensitive": p.case_sensitive,
}
for p in form.patterns
],
"sample": form.sample,
}
)
return {
"matches": [
{"id": m["id"], "start": m["from"], "end": m["to"]} for m in res.get("matches", [])
],
"errors": res.get("errors", []),
"error": res.get("error"),
"unscannable": res.get("unscannable", []),
"truncated": res.get("truncated", False),
}
Reference in New Issue View Git Blame Copy Permalink