mirror of
https://github.com/Av0idD4rk/firegex.git
synced 2026-10-23 16:31:00 +00:00
f6d9d70042
cibuildwheel ran wheel_smoke.py from fgex-lib/, so Python put that directory first on sys.path and imported the unbuilt source package: no _llhttp, every wheel job failed. python -I leaves it off.
435 lines
17 KiB
YAML
435 lines
17 KiB
YAML
name: Test, build and publish release artifacts
|
|
|
|
on:
|
|
release:
|
|
types:
|
|
- published
|
|
|
|
env:
|
|
REGISTRY: ghcr.io
|
|
IMAGE_NAME: ${{ github.repository }}
|
|
|
|
jobs:
|
|
# Gate for every publish target below: nothing gets pushed to Docker/PyPI unless this passes.
|
|
run_tests:
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
matrix:
|
|
os: [ubuntu-latest, ubuntu-24.04-arm]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Build and run firegex
|
|
run: python3 run.py start -P testpassword
|
|
|
|
# No apt-get: the tests' dependencies are pure python. `iperf3`, matplotlib and
|
|
# numpy belong to the benchmarks, which CI does not run — a measurement has no
|
|
# pass or fail to gate a release on.
|
|
#
|
|
# As root, because what needs root **skips** otherwise — and a case that always
|
|
# skips is not a case this gate is checking. Through the script rather than `sudo
|
|
# pytest`, because the script installs the dependencies into the environment it is
|
|
# running in; a root process does not read the invoking user's site-packages.
|
|
- name: Run tests
|
|
run: cd tests && sudo ./run_tests.sh
|
|
|
|
# The proxy engine's own suite: HTTP/2, HTTP/3 and QUIC termination, the Python worker's
|
|
# framing and deadlines, the UDP relay — much of it reachable from no integration test,
|
|
# and none of it was run before a release. In the image stage built for it, which is
|
|
# the compiler plus the filter library the worker tests start. `backend/` is mounted
|
|
# rather than baked in because the worker tests launch `../modules/services/pyworker.py`.
|
|
engine_tests:
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
matrix:
|
|
os: [ubuntu-latest, ubuntu-24.04-arm]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Build the engine's test image
|
|
run: docker build --target enginetest -t firegex-enginetest .
|
|
|
|
- name: Run the engine's tests
|
|
run: >-
|
|
docker run --rm -v "$PWD/backend:/execute" -w /execute/proxysrc
|
|
firegex-enginetest cargo test
|
|
|
|
docker_build:
|
|
needs: [run_tests, engine_tests]
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- os: ubuntu-latest
|
|
arch: amd64
|
|
- os: ubuntu-24.04-arm
|
|
arch: arm64
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Convert repository name to lowercase
|
|
id: lowercase
|
|
run: echo "image_name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Set up Docker Buildx
|
|
id: buildx
|
|
uses: docker/setup-buildx-action@master
|
|
|
|
- name: Log in to the Container registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Extract metadata (tags, labels) for Docker
|
|
id: meta
|
|
uses: docker/metadata-action@v5
|
|
with:
|
|
images: ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}
|
|
|
|
- name: Extract tag name
|
|
id: tag
|
|
run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
|
|
|
|
- name: Update version in setup.py
|
|
run: >-
|
|
sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" backend/utils/__init__.py;
|
|
sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/setup.py;
|
|
sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/firegex/__init__.py;
|
|
|
|
- name: Build and push Docker image
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: .
|
|
builder: ${{ steps.buildx.outputs.name }}
|
|
platforms: linux/${{ matrix.arch }}
|
|
push: true
|
|
tags: |
|
|
${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-${{ matrix.arch }}
|
|
${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-${{ matrix.arch }}
|
|
labels: ${{ steps.meta.outputs.labels }}
|
|
cache-from: type=gha,scope=${{ matrix.arch }}
|
|
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
|
|
provenance: false
|
|
sbom: false
|
|
|
|
docker_manifest:
|
|
needs: docker_build
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
|
|
steps:
|
|
- name: Convert repository name to lowercase
|
|
id: lowercase
|
|
run: echo "image_name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Log in to the Container registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Extract tag name
|
|
id: tag
|
|
run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
|
|
|
|
- name: Create and push multi-platform manifest
|
|
run: |
|
|
# Create manifest list for specific tag
|
|
docker manifest create ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }} \
|
|
--amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-amd64 \
|
|
--amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-arm64
|
|
|
|
# Annotate the manifest with architecture info
|
|
docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }} \
|
|
${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-amd64 \
|
|
--arch amd64 --os linux
|
|
|
|
docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }} \
|
|
${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-arm64 \
|
|
--arch arm64 --os linux
|
|
|
|
docker manifest push ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}
|
|
|
|
# Create manifest list for latest tag
|
|
docker manifest create ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest \
|
|
--amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-amd64 \
|
|
--amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-arm64
|
|
|
|
# Annotate the latest manifest with architecture info
|
|
docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest \
|
|
${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-amd64 \
|
|
--arch amd64 --os linux
|
|
|
|
docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest \
|
|
${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-arm64 \
|
|
--arch arm64 --os linux
|
|
|
|
docker manifest push ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest
|
|
|
|
create-rootfs-assets:
|
|
runs-on: ubuntu-latest
|
|
needs: [docker_manifest]
|
|
permissions:
|
|
contents: write
|
|
packages: read
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Convert repository name to lowercase
|
|
id: lowercase
|
|
run: echo "image_name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Set up QEMU
|
|
uses: docker/setup-qemu-action@master
|
|
with:
|
|
platforms: all
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@master
|
|
|
|
- name: Log in to the Container registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ${{ env.REGISTRY }}
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Get latest release tag
|
|
id: get_tag
|
|
run: |
|
|
LATEST_TAG=$(curl -s https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.tag_name')
|
|
echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT
|
|
echo "Latest release tag: $LATEST_TAG"
|
|
|
|
- name: Export rootfs for amd64
|
|
run: |
|
|
echo "Creating and exporting amd64 container..."
|
|
CONTAINER_ID=$(docker create --platform linux/amd64 ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.get_tag.outputs.tag }})
|
|
docker export $CONTAINER_ID --output="firegex-rootfs-amd64.tar"
|
|
docker rm $CONTAINER_ID
|
|
echo "Compressing amd64 rootfs..."
|
|
gzip firegex-rootfs-amd64.tar
|
|
ls -lh firegex-rootfs-amd64.tar.gz
|
|
|
|
- name: Export rootfs for arm64
|
|
run: |
|
|
echo "Creating and exporting arm64 container..."
|
|
CONTAINER_ID=$(docker create --platform linux/arm64 ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.get_tag.outputs.tag }})
|
|
docker export $CONTAINER_ID --output="firegex-rootfs-arm64.tar"
|
|
docker rm $CONTAINER_ID
|
|
echo "Compressing arm64 rootfs..."
|
|
gzip firegex-rootfs-arm64.tar
|
|
ls -lh firegex-rootfs-arm64.tar.gz
|
|
|
|
- name: Upload rootfs assets to release
|
|
run: |
|
|
echo "Uploading assets to release ${{ steps.get_tag.outputs.tag }}..."
|
|
gh release upload ${{ steps.get_tag.outputs.tag }} \
|
|
firegex-rootfs-amd64.tar.gz \
|
|
firegex-rootfs-arm64.tar.gz \
|
|
--clobber
|
|
echo "Assets uploaded successfully!"
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
# The README's picture of the interface, retaken for every release so it never shows a
|
|
# version that no longer exists. From the demo build served locally rather than from the
|
|
# published demo: that deploys on its own schedule, and could still be the old one.
|
|
update_readme_screenshot:
|
|
needs: [run_tests, engine_tests]
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: write
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: main
|
|
|
|
- name: Set up bun
|
|
uses: oven-sh/setup-bun@v2
|
|
with:
|
|
bun-version: latest
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Install Playwright
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install playwright
|
|
playwright install --with-deps chromium
|
|
|
|
- name: Build the demo
|
|
working-directory: frontend
|
|
run: bun install --frozen-lockfile && VITE_VERSION="${{ github.ref_name }}" bun run build:demo
|
|
|
|
- name: Screenshot it
|
|
working-directory: frontend
|
|
run: |
|
|
bunx vite preview --port 4173 --strictPort &
|
|
timeout 60 bash -c 'until curl -sf http://localhost:4173/ > /dev/null; do sleep 0.5; done'
|
|
python3 ../docs/screenshot_demo.py http://localhost:4173/
|
|
|
|
- name: Commit the new screenshot
|
|
uses: stefanzweifel/git-auto-commit-action@v5
|
|
with:
|
|
commit_message: "Update README screenshot for ${{ github.ref_name }}"
|
|
file_pattern: 'docs/Firegex_Screenshot.png'
|
|
branch: main
|
|
|
|
# `firegex` carries a C extension — the llhttp binding in `fgex-lib/llhttp/` — so it
|
|
# is no longer one universal wheel. A wheel is built per platform and per interpreter
|
|
# here, and the sdist goes up beside them: where no wheel matches, pip compiles from
|
|
# that. Compiling is a thing pip knows how to do; assuming a system library is already
|
|
# installed is not, which is why the parser ships as source rather than as a `ctypes`
|
|
# binding against the distribution's `libllhttp`.
|
|
build_firegex_wheels:
|
|
needs: [run_tests, engine_tests]
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: ubuntu-latest # manylinux + musllinux, x86_64
|
|
- os: ubuntu-24.04-arm # and the same for aarch64
|
|
# arm64 and x86_64 both, from one runner: the Intel macOS image this used to ask
|
|
# for (`macos-13`) has been retired, and a matrix entry naming a runner that no
|
|
# longer exists fails — which, with the publish job needing this one, blocked
|
|
# every PyPI release. The x86_64 wheel is cross-compiled; see CIBW_TEST_SKIP.
|
|
- os: macos-latest
|
|
- os: windows-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Extract tag name
|
|
id: tag
|
|
shell: bash
|
|
run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
|
|
- name: Update version
|
|
shell: bash
|
|
run: >-
|
|
sed -i.bak "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/setup.py;
|
|
sed -i.bak "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/firegex/__init__.py;
|
|
- name: Build wheels
|
|
# 3.x, because 2.x predates CPython 3.14 and quietly built nothing for it: the
|
|
# selector below matched no identifier it knew, which is not an error.
|
|
uses: pypa/cibuildwheel@v3.2.0
|
|
with:
|
|
package-dir: fgex-lib
|
|
output-dir: wheelhouse
|
|
env:
|
|
# The library declares `python_requires>=3.10`, so build what it claims to
|
|
# support and nothing else. PyPy is excluded: the datapath embeds CPython.
|
|
# 3.10 and 3.11 build because the binding declares its 3.12-only module slot
|
|
# only where it exists (`llhttp_module.c`) — unguarded, it failed to compile
|
|
# there, and the whole job with it.
|
|
CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*"
|
|
CIBW_SKIP: "*-musllinux_i686 *_i686 pp*"
|
|
CIBW_ARCHS_MACOS: "arm64 x86_64"
|
|
# An x86_64 wheel built on an arm64 runner cannot be imported there without
|
|
# Rosetta, which the runner is not promised to have; the arm64 wheels, built
|
|
# from the same sources in the same job, are what the smoke test checks.
|
|
CIBW_TEST_SKIP: "*-macosx_x86_64"
|
|
# A wheel that imports is the only kind worth publishing, and the extension is
|
|
# the whole reason this job exists. The checks live in a file rather than in a
|
|
# `-c` string because the one that matters needs a `try`: the parser has to
|
|
# raise its *own* exception on malformed traffic, and this clean environment —
|
|
# no `pyllhttp`, no developer's leftovers — is the only place that is tested.
|
|
CIBW_TEST_COMMAND: python -I {package}/wheel_smoke.py
|
|
- uses: actions/upload-artifact@v4
|
|
with:
|
|
name: wheels-${{ matrix.os }}
|
|
path: wheelhouse/*.whl
|
|
|
|
pypi_publish_firegex:
|
|
needs: build_firegex_wheels
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.x'
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install build
|
|
- name: Extract tag name
|
|
id: tag
|
|
run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
|
|
- name: Update version in setup.py
|
|
run: >-
|
|
sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/setup.py;
|
|
sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/firegex/__init__.py;
|
|
# The sdist is what pip falls back to where no wheel matches, so it has to carry
|
|
# the parser's C sources — `MANIFEST.in` is what puts them in.
|
|
- name: Build the sdist
|
|
run: cd fgex-lib && python -m build --sdist && mv ./dist ../
|
|
- name: Collect the wheels built for every platform
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: wheels-*
|
|
merge-multiple: true
|
|
path: dist
|
|
- name: Publish package
|
|
uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
|
|
with:
|
|
user: __token__
|
|
password: ${{ secrets.PYPI_API_TOKEN }}
|
|
|
|
# After `firegex`, never beside it: this package is nothing but `firegex==<tag>`, so
|
|
# published while a wheel job failed it is an alias for a version that does not exist,
|
|
# and `pip install fgex` fails for everybody until somebody notices.
|
|
pypi_publish_fgex_alias:
|
|
needs: pypi_publish_firegex
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.x'
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install build
|
|
- name: Extract tag name
|
|
id: tag
|
|
run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
|
|
- name: Update version in setup.py
|
|
run: >-
|
|
sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/fgex-pip/setup.py;
|
|
- name: Build package
|
|
run: cd fgex-lib/fgex-pip && python -m build && mv ./dist ../../
|
|
- name: Publish package
|
|
uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
|
|
with:
|
|
user: __token__
|
|
password: ${{ secrets.PYPI_API_TOKEN_FGEX }}
|