diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33d5bba..c94d618 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -248,6 +248,55 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # The README's picture of the interface, retaken for every release so it never shows a + # version that no longer exists. From the demo build served locally rather than from the + # published demo: that deploys on its own schedule, and could still be the old one. + update_readme_screenshot: + needs: [run_tests, engine_tests] + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: main + + - name: Set up bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Playwright + run: | + python -m pip install --upgrade pip + pip install playwright + playwright install --with-deps chromium + + - name: Build the demo + working-directory: frontend + run: bun install --frozen-lockfile && VITE_VERSION="${{ github.ref_name }}" bun run build:demo + + - name: Screenshot it + working-directory: frontend + run: | + bunx vite preview --port 4173 --strictPort & + timeout 60 bash -c 'until curl -sf http://localhost:4173/ > /dev/null; do sleep 0.5; done' + python3 ../docs/screenshot_demo.py http://localhost:4173/ + + - name: Commit the new screenshot + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "Update README screenshot for ${{ github.ref_name }}" + file_pattern: 'docs/Firegex_Screenshot.png' + branch: main + # `firegex` carries a C extension — the llhttp binding in `fgex-lib/llhttp/` — so it # is no longer one universal wheel. A wheel is built per platform and per interpreter # here, and the sdist goes up beside them: where no wheel matches, pip compiles from @@ -351,8 +400,11 @@ jobs: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + # After `firegex`, never beside it: this package is nothing but `firegex==`, so + # published while a wheel job failed it is an alias for a version that does not exist, + # and `pip install fgex` fails for everybody until somebody notices. pypi_publish_fgex_alias: - needs: [run_tests, engine_tests] + needs: pypi_publish_firegex runs-on: ubuntu-latest permissions: contents: read diff --git a/AGENTS.md b/AGENTS.md index f7583ec..81b6835 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ there — a file only processed for *site* directories, so putting that path in imports pytest and still leaves `pyworker.py --check` unable to find the filter library. The script installs into the environment it is running in, which resolves all of it. Two things that trip a root run were found by trying it, and both are fixed rather than worked -around: `tests/requirements.txt` was missing `fastapi` (see the file for why one unit test +around: the `test` group in `tests/pyproject.toml` was missing `fastapi` (see the file for why one unit test needs the backend's own dependency, and why its absence stops the *whole* run rather than one file), and `./run_tests.sh unit` — advertised in the script's own help — took `unit` as a **password** and ran everything against an instance it could not authenticate to. @@ -108,7 +108,7 @@ Mount `backend/`, not `backend/proxysrc/`: `tests/pyworker.rs` launches `../modu The suite runs against an instance with authentication turned off as well: `login()` reads a 403 there as "there is nothing to log in to" rather than as a failure, since every request is already allowed. `integration/test_auth.py` is the exception and skips itself — it is about passwords, and those endpoints answer 403 in that mode by design. Note `BearerSession.post` copies the header dict rather than writing into it: the form `Content-Type` belongs to the login request alone, and it used to stick, which went unnoticed only because a successful login replaces the whole dict a moment later. -CI (`.github/workflows/release.yml`) runs **only on `release: published`**, not on pushes or pull requests — nothing validates a PR automatically. Two jobs gate every publish job below them, both on `ubuntu-latest` and `ubuntu-24.04-arm`: `run_tests` does `python3 run.py start -P testpassword` then `cd tests && sudo ./run_tests.sh`, and `engine_tests` runs the Rust suite (`cargo test` in `proxysrc`) in the Dockerfile's `enginetest` stage — the compiler stage plus the filter library the pyworker tests start. That suite was not run before a release at all, though much of what it covers (HTTP/2, HTTP/3 and QUIC termination, the worker's framing, the UDP relay) no integration test reaches. `enginetest` sits **before** `final` on purpose: a build with no target, which is what compose does, takes the last stage. The tests' dependencies are pure python; `iperf3`, matplotlib and numpy belong to `bench/requirements.txt` and CI installs none of them. +CI (`.github/workflows/release.yml`) runs **only on `release: published`**, not on pushes or pull requests — nothing validates a PR automatically. Two jobs gate every publish job below them, both on `ubuntu-latest` and `ubuntu-24.04-arm`: `run_tests` does `python3 run.py start -P testpassword` then `cd tests && sudo ./run_tests.sh`, and `engine_tests` runs the Rust suite (`cargo test` in `proxysrc`) in the Dockerfile's `enginetest` stage — the compiler stage plus the filter library the pyworker tests start. That suite was not run before a release at all, though much of what it covers (HTTP/2, HTTP/3 and QUIC termination, the worker's framing, the UDP relay) no integration test reaches. `enginetest` sits **before** `final` on purpose: a build with no target, which is what compose does, takes the last stage. The tests' dependencies are pure python; `iperf3`, matplotlib and numpy belong to the `bench` group of `tests/pyproject.toml` and CI installs none of them. ## Architecture diff --git a/Dockerfile b/Dockerfile index 9f7d96f..2318ede 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,12 +57,12 @@ RUN python3.14 -m pip install --no-cache-dir --break-system-packages /fgex-lib #Building main conteiner FROM --platform=$TARGETARCH base AS final -COPY ./backend/requirements.txt /execute/requirements.txt +COPY ./backend/pyproject.toml /execute/pyproject.toml COPY ./fgex-lib /execute/fgex-lib RUN dnf -y update && dnf install -y gcc-c++ python3.14-devel uv git &&\ uv pip install --no-cache --system ./fgex-lib &&\ - uv pip install --no-cache --system -r /execute/requirements.txt &&\ + uv pip install --no-cache --system -r /execute/pyproject.toml &&\ uv cache clean && dnf remove -y gcc-c++ python3.14-devel uv git && dnf clean all COPY ./backend/ /execute/ diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 0000000..bcf38e4 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,15 @@ +# Not a package: only read for its dependencies (`uv pip install -r pyproject.toml` in the +# Dockerfile). The version firegex reports is in utils/__init__.py. +[project] +name = "firegex-backend" +version = "0.0.0" +dependencies = [ + "fastapi[all]", + "httpx", + "uvicorn[standard]", + "psutil", + "python-jose[cryptography]", + "python-socketio", + "brotli", + # nftables comes from the distribution (python3-nftables), not from PyPI. +] diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index 55d6516..0000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -fastapi[all] -httpx -uvicorn[standard] -psutil -python-jose[cryptography] -python-socketio -brotli -#git+https://salsa.debian.org/pkg-netfilter-team/pkg-nftables#egg=nftables&subdirectory=py diff --git a/docs/Firegex_Screenshot.png b/docs/Firegex_Screenshot.png index 06fa7ed..330ad54 100644 Binary files a/docs/Firegex_Screenshot.png and b/docs/Firegex_Screenshot.png differ diff --git a/docs/screenshot_demo.py b/docs/screenshot_demo.py new file mode 100644 index 0000000..3322996 --- /dev/null +++ b/docs/screenshot_demo.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +""" +Take the README's screenshot of the interface, from the demo build. + +Usage: + python3 docs/screenshot_demo.py [url] [output_path] + +`url` defaults to the published demo; the release workflow builds the demo from the +release being published and serves it locally instead, so the picture is of that version +and not of whatever the Pages deploy last finished (built with `VITE_VERSION=`, the +header then names it instead of saying "demo"). Needs Playwright with chromium +(`pip install playwright && playwright install --with-deps chromium`). + +The viewport is fixed, so the image keeps its size from one release to the next however +much the demo's seed data happens to render. +""" +import sys +from pathlib import Path + +from playwright.sync_api import sync_playwright + +URL = sys.argv[1] if len(sys.argv) > 1 else "https://demo.firegex.pwnzer0tt1.it/" +OUTPUT = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(__file__).parent / "Firegex_Screenshot.png" +VIEWPORT = {"width": 1600, "height": 960} + + +def main(): + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page(viewport=VIEWPORT, device_scale_factor=2, color_scheme="dark") + page.goto(URL, wait_until="networkidle") + # The demo greets its visitor with a toast saying it is one, which is not part of + # the interface being pictured. + page.add_style_tag(content=".mantine-Notifications-root { display: none !important; }") + # The list fills in from the demo's in-memory backend after the page has loaded; + # a seeded service's name is what says it has. + page.wait_for_selector("text=shop-api", timeout=15000) + page.wait_for_timeout(1000) + page.screenshot(path=str(OUTPUT)) + browser.close() + print(f"Saved {OUTPUT} ({VIEWPORT['width']}x{VIEWPORT['height']} @2x)") + + +if __name__ == "__main__": + main() diff --git a/fgex-lib/MANIFEST.in b/fgex-lib/MANIFEST.in index 5a6efc3..23f4f83 100644 --- a/fgex-lib/MANIFEST.in +++ b/fgex-lib/MANIFEST.in @@ -1,4 +1,3 @@ -include requirements.txt recursive-include llhttp *.c *.h include llhttp/PROVENANCE.md include llhttp/LICENSE.pyllhttp diff --git a/fgex-lib/pyproject.toml b/fgex-lib/pyproject.toml new file mode 100644 index 0000000..89ccc36 --- /dev/null +++ b/fgex-lib/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "firegex" +# Set in setup.py, where the release workflow writes the tag. +dynamic = ["version"] +description = "Firegex client" +readme = "README.md" +authors = [{ name = "Pwnzer0tt1", email = "pwnzer0tt1@poliba.it" }] +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", +] +dependencies = [ + "typer", + # Imported directly by the CLI and both simulators for their output. `typer` happens to + # bring it today; `typer-slim` does not, and nothing here should rest on which one it is. + "rich", + "pydantic>=2", + "typing-extensions>=4.7.1", + "pycryptodome", + "watchfiles", + "websockets", + # 1.2 is the first with `output_buffer_limit`, which is what lets a body be decoded only + # as far as a filter may hold it. See `_unbrotli` in pyfilters/models/http.py. + "brotli>=1.2.0", + # Python 3.14 has `compression.zstd` in the standard library; this fills the gap + # below it. See firegex/pyfilters/internals/zstd_compat.py. + "zstandard; python_version < '3.14'", +] + +[project.urls] +Homepage = "https://github.com/pwnzer0tt1/firegex" + +[project.scripts] +fgex = "firegex.cli:run" + +[tool.setuptools.packages.find] +include = ["firegex*"] diff --git a/fgex-lib/requirements.txt b/fgex-lib/requirements.txt deleted file mode 100644 index 7f20801..0000000 --- a/fgex-lib/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -typer -# Imported directly by the CLI and both simulators for their output. `typer` happens to -# bring it today; `typer-slim` does not, and nothing here should rest on which one it is. -rich -pydantic>=2 -typing-extensions>=4.7.1 -pycryptodome -watchfiles -websockets -# 1.2 is the first with `output_buffer_limit`, which is what lets a body be decoded only -# as far as a filter may hold it. See `_unbrotli` in pyfilters/models/http.py. -brotli>=1.2.0 -# Python 3.14 has `compression.zstd` in the standard library; this fills the gap -# below it. See firegex/pyfilters/internals/zstd_compat.py. -zstandard; python_version < '3.14' diff --git a/fgex-lib/setup.py b/fgex-lib/setup.py index d7758a8..32dbedd 100644 --- a/fgex-lib/setup.py +++ b/fgex-lib/setup.py @@ -23,32 +23,12 @@ LLHTTP = setuptools.Extension( extra_compile_args=["-O3"], ) -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -with open('requirements.txt', 'r', encoding='utf-8') as f: - required = [ele.strip() for ele in f.read().splitlines() if not ele.strip().startswith("#") and ele.strip() != ""] - +# Everything else about the package is in pyproject.toml. The version stays here because +# the release workflow writes the tag over the placeholder, which is not a valid version +# until it has. VERSION = "{{VERSION_PLACEHOLDER}}" setuptools.setup( - name="firegex", - version= VERSION if "{" not in VERSION else "0.0.0", #uv pip install -U . --no-cache-dir for testing - author="Pwnzer0tt1", - author_email="pwnzer0tt1@poliba.it", - scripts=["fgex"], - py_modules=["firegex"], - install_requires=required, - include_package_data=True, - description="Firegex client", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/pwnzer0tt1/firegex", - packages=setuptools.find_packages(), + version=VERSION if "{" not in VERSION else "0.0.0", #uv pip install -U . --no-cache-dir for testing ext_modules=[LLHTTP], - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", - ], - python_requires='>=3.10', ) diff --git a/frontend/src/js/demo.ts b/frontend/src/js/demo.ts index d495b7d..b5b3a2d 100644 --- a/frontend/src/js/demo.ts +++ b/frontend/src/js/demo.ts @@ -379,7 +379,7 @@ type Handler = (m: RegExpMatchArray, body: Json, query: Record) const routes: [string, RegExp, Handler][] = [ // ---- global - ["GET", /^status$/, () => ({ status: "run", loggined: true, version: "demo", auth_disabled: false })], + ["GET", /^status$/, () => ({ status: "run", loggined: true, version: import.meta.env.VITE_VERSION || "demo", auth_disabled: false })], ["POST", /^login$/, () => ({ access_token: "demo-token", token_type: "bearer" })], ["POST", /^set-password$/, () => ok], ["POST", /^change-password$/, () => ({ ...ok, access_token: "demo-token" })], diff --git a/tests/bench/README.md b/tests/bench/README.md index d3c885d..10cd406 100644 --- a/tests/bench/README.md +++ b/tests/bench/README.md @@ -12,9 +12,9 @@ most of this page is about. `pytest` does not collect this directory. | `stress_test_history.py` | a soak test for `HttpHistory` under load | | `results_plotter.py` | redraws the SVGs in `results/` from the CSVs beside them | -These need more than the tests do — `pip3 install -r requirements.txt` here, on top of -the one a directory up, and `iperf3` itself installed as a package because the python -binding links against it. +These need more than the tests do — the `bench` group of `../pyproject.toml` +(`pip3 install --group ../pyproject.toml:bench` from here, pip 25.1 or later), and `iperf3` +itself installed as a package because the python binding links against it. # Running a benchmark diff --git a/tests/bench/requirements.txt b/tests/bench/requirements.txt deleted file mode 100644 index 950c6a3..0000000 --- a/tests/bench/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# On top of ../requirements.txt. -# `iperf3` is a binding against libiperf, so the library has to be installed first -# (`apt-get install iperf3`, `dnf install iperf3`). -iperf3 -matplotlib -numpy diff --git a/tests/integration/test_service_lifecycle.py b/tests/integration/test_service_lifecycle.py index 73bc517..01bdf48 100644 --- a/tests/integration/test_service_lifecycle.py +++ b/tests/integration/test_service_lifecycle.py @@ -110,7 +110,10 @@ def test_stopping_a_proxied_service_does_not_cut_its_connections(api, protected, "a stopped service went on filtering the connections it had" time.sleep(3) assert _held_open(server, b"a while later") == b"a while later" - assert _log_says(api, service_id, "carry on unfiltered"), \ + # The line waits for a STATS report taken after the stop, which the engine sends + # every two seconds and drops while the backend is behind — under a full suite on + # a busy machine, ten seconds was once not enough to see one. + assert _log_says(api, service_id, "carry on unfiltered", wait=30), \ "the log did not say connections were still being carried" finally: server.close_client() diff --git a/tests/pyproject.toml b/tests/pyproject.toml new file mode 100644 index 0000000..3d5df07 --- /dev/null +++ b/tests/pyproject.toml @@ -0,0 +1,55 @@ +# Not a package: dependency groups only (PEP 735). `run_tests.sh` installs `test`; +# `bench` is what the benchmarks need on top of it. +[dependency-groups] +test = [ + "pytest", + "requests", + "cryptography", + "psutil", + "pydantic", + "python-socketio[client]", + # The only way to speak QUIC from Python, and the integration tests for that layer have + # to speak it: what they check is the half the engine's own tests cannot reach — the + # rules that steer a QUIC address, the relay bound per address, and a filter attached + # through the API. Everything that needs it skips with a reason when it is absent, so a + # machine without it still runs the rest. + "aioquic", + # And the only way to speak gRPC from Python, which is what says HTTP/2 is filtered: a + # gRPC service is a real HTTP/2 service, its method name is an HPACK-compressed `:path`, + # and its status is a trailer section — the three things a firegex that only forwarded + # HTTP/2 could not show a filter. Everything that needs it skips with a reason when it is + # absent, so a machine without it still runs the rest. + "grpcio", + # `unit/test_nftables_addresses.py` imports the **backend** — the address-to-nftables + # translation is the backend's, and testing a copy of it would be testing a copy. That + # drags in `backend/utils`, which raises FastAPI's `HTTPException`, so FastAPI has to be + # here even though nothing in this suite serves a request. Without it the file cannot be + # collected, and a collection error is not one test failing: pytest stops the entire run + # before anything has executed. It went unnoticed for as long as it did because a machine + # that has ever run the backend outside the container already has it. + "fastapi", + # `unit/` imports the filter library straight out of `../fgex-lib`, so its runtime + # dependencies have to be here too. Not the published `firegex` itself, which would + # quietly shadow the source tree the suite is meant to be testing. + # + # The HTTP parser is deliberately NOT here. It is a C extension that ships inside + # firegex itself, so it has to be *built*: `run_tests.sh` does that with an editable + # install of `../fgex-lib`. Running `pytest` by hand without it gives + # `ImportError: cannot import name '_llhttp' from 'firegex'`. + "brotli", + "websockets", + # Unconditional here, unlike in the library, which only needs it below 3.14: the + # suite uses it to build the corpus *and* to exercise the fallback backend, so + # without it the cross-backend agreement test silently skips on the one Python + # that matters most. + "zstandard", +] +# The tests' dependencies stay pure python, so a CI run that only tests does not build a +# binding against a C library it will never call. `iperf3` is one: the library has to be +# installed first (`apt-get install iperf3`, `dnf install iperf3`). +bench = [ + { include-group = "test" }, + "iperf3", + "matplotlib", + "numpy", +] diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 221581c..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,48 +0,0 @@ -# What the tests need. The benchmarks need more — iperf3, matplotlib, numpy — and those -# are in bench/requirements.txt, so a CI run that only tests does not build a binding -# against a C library it will never call. -pytest -requests -cryptography -psutil -pydantic -python-socketio[client] -# The only way to speak QUIC from Python, and the integration tests for that layer have -# to speak it: what they check is the half the engine's own tests cannot reach — the -# rules that steer a QUIC address, the relay bound per address, and a filter attached -# through the API. Everything that needs it skips with a reason when it is absent, so a -# machine without it still runs the rest. -aioquic -# And the only way to speak gRPC from Python, which is what says HTTP/2 is filtered: a -# gRPC service is a real HTTP/2 service, its method name is an HPACK-compressed `:path`, -# and its status is a trailer section — the three things a firegex that only forwarded -# HTTP/2 could not show a filter. Everything that needs it skips with a reason when it is -# absent, so a machine without it still runs the rest. -grpcio - -# `unit/test_nftables_addresses.py` imports the **backend** — the address-to-nftables -# translation is the backend's, and testing a copy of it would be testing a copy. That -# drags in `backend/utils`, which raises FastAPI's `HTTPException`, so FastAPI has to be -# here even though nothing in this suite serves a request. Without it the file cannot be -# collected, and a collection error is not one test failing: pytest stops the entire run -# before anything has executed. It went unnoticed for as long as it did because a machine -# that has ever run the backend outside the container already has it. -fastapi - -# `unit/` imports the filter library straight out of `../fgex-lib`, so its runtime -# dependencies have to be here too. Listed one by one rather than with -# `-r ../fgex-lib/requirements.txt`, because that file lists `fgex` — the alias package, -# which depends on a *published* `firegex` and would quietly shadow the source tree the -# suite is meant to be testing. -# -# The HTTP parser is deliberately NOT here. It is a C extension that ships inside -# firegex itself, so it has to be *built*: `run_tests.sh` does that with an editable -# install of `../fgex-lib`. Running `pytest` by hand without it gives -# `ImportError: cannot import name '_llhttp' from 'firegex'`. -brotli -websockets -# Unconditional here, unlike in the library, which only needs it below 3.14: the -# suite uses it to build the corpus *and* to exercise the fallback backend, so -# without it the cross-backend agreement test silently skips on the one Python -# that matters most. -zstandard diff --git a/tests/run_tests.sh b/tests/run_tests.sh index b47d03e..1a2d640 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -47,7 +47,10 @@ fi ADDRESS="${FIREGEX_ADDRESS:-http://127.0.0.1:4444/}" ADDRESS="${ADDRESS%/}/" -pip3 install -q -r requirements.txt +# The `test` group of pyproject.toml, read here rather than with `pip install --group`, +# which needs pip 25.1 — older than what CI's runners ship. +python3 -c 'import tomllib; print("\n".join(tomllib.load(open("pyproject.toml", "rb"))["dependency-groups"]["test"]))' \ + | pip3 install -q -r /dev/stdin # The filter library is imported straight out of the source tree, and it carries a C # extension — the llhttp binding that parses HTTP — which therefore has to be built. An # editable install builds it in place; without this `unit/` cannot even be collected.