firegex/.github/workflows/release.yml

name: Test, build and publish release artifacts

on:
  release:
    types:
      - published

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  # Gate for every publish target below: nothing gets pushed to Docker/PyPI unless this passes.
  run_tests:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, ubuntu-24.04-arm]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Build and run firegex
        run: python3 run.py start -P testpassword

      # No apt-get: the tests' dependencies are pure python. `iperf3`, matplotlib and
      # numpy belong to the benchmarks, which CI does not run — a measurement has no
      # pass or fail to gate a release on.
      #
      # As root, because what needs root **skips** otherwise — and a case that always
      # skips is not a case this gate is checking. Through the script rather than `sudo
      # pytest`, because the script installs the dependencies into the environment it is
      # running in; a root process does not read the invoking user's site-packages.
      - name: Run tests
        run: cd tests && sudo ./run_tests.sh

  # The proxy engine's own suite: HTTP/2, HTTP/3 and QUIC termination, the Python worker's
  # framing and deadlines, the UDP relay — much of it reachable from no integration test,
  # and none of it was run before a release. In the image stage built for it, which is
  # the compiler plus the filter library the worker tests start. `backend/` is mounted
  # rather than baked in because the worker tests launch `../modules/services/pyworker.py`.
  engine_tests:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, ubuntu-24.04-arm]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Build the engine's test image
        run: docker build --target enginetest -t firegex-enginetest .

      - name: Run the engine's tests
        run: >-
          docker run --rm -v "$PWD/backend:/execute" -w /execute/proxysrc
          firegex-enginetest cargo test          

  docker_build:
    needs: [run_tests, engine_tests]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            arch: amd64
          - os: ubuntu-24.04-arm
            arch: arm64
    permissions:
      contents: read
      packages: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Convert repository name to lowercase
        id: lowercase
        run: echo "image_name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@master

      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}

      - name: Extract tag name
        id: tag
        run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT

      - name: Update version in setup.py
        run: >-
          sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" backend/utils/__init__.py;
          sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/setup.py;
          sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/firegex/__init__.py;          

      - name: Build and push Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          builder: ${{ steps.buildx.outputs.name }}
          platforms: linux/${{ matrix.arch }}
          push: true
          tags: |
            ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-${{ matrix.arch }}
            ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-${{ matrix.arch }}            
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha,scope=${{ matrix.arch }}
          cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
          provenance: false
          sbom: false

  docker_manifest:
    needs: docker_build
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    steps:
      - name: Convert repository name to lowercase
        id: lowercase
        run: echo "image_name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT

      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract tag name
        id: tag
        run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT

      - name: Create and push multi-platform manifest
        run: |
          # Create manifest list for specific tag
          docker manifest create ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }} \
            --amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-amd64 \
            --amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-arm64

          # Annotate the manifest with architecture info
          docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }} \
            ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-amd64 \
            --arch amd64 --os linux

          docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }} \
            ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}-arm64 \
            --arch arm64 --os linux

          docker manifest push ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.tag.outputs.TAG_NAME }}

          # Create manifest list for latest tag
          docker manifest create ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest \
            --amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-amd64 \
            --amend ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-arm64

          # Annotate the latest manifest with architecture info
          docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest \
            ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-amd64 \
            --arch amd64 --os linux

          docker manifest annotate ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest \
            ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest-arm64 \
            --arch arm64 --os linux

          docker manifest push ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:latest          

  create-rootfs-assets:
    runs-on: ubuntu-latest
    needs: [docker_manifest]
    permissions:
      contents: write
      packages: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Convert repository name to lowercase
        id: lowercase
        run: echo "image_name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT

      - name: Set up QEMU
        uses: docker/setup-qemu-action@master
        with:
          platforms: all

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@master

      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Get latest release tag
        id: get_tag
        run: |
          LATEST_TAG=$(curl -s https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.tag_name')
          echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT
          echo "Latest release tag: $LATEST_TAG"          

      - name: Export rootfs for amd64
        run: |
          echo "Creating and exporting amd64 container..."
          CONTAINER_ID=$(docker create --platform linux/amd64 ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.get_tag.outputs.tag }})
          docker export $CONTAINER_ID --output="firegex-rootfs-amd64.tar"
          docker rm $CONTAINER_ID
          echo "Compressing amd64 rootfs..."
          gzip firegex-rootfs-amd64.tar
          ls -lh firegex-rootfs-amd64.tar.gz          

      - name: Export rootfs for arm64
        run: |
          echo "Creating and exporting arm64 container..."
          CONTAINER_ID=$(docker create --platform linux/arm64 ${{ env.REGISTRY }}/${{ steps.lowercase.outputs.image_name }}:${{ steps.get_tag.outputs.tag }})
          docker export $CONTAINER_ID --output="firegex-rootfs-arm64.tar"
          docker rm $CONTAINER_ID
          echo "Compressing arm64 rootfs..."
          gzip firegex-rootfs-arm64.tar
          ls -lh firegex-rootfs-arm64.tar.gz          

      - name: Upload rootfs assets to release
        run: |
          echo "Uploading assets to release ${{ steps.get_tag.outputs.tag }}..."
          gh release upload ${{ steps.get_tag.outputs.tag }} \
            firegex-rootfs-amd64.tar.gz \
            firegex-rootfs-arm64.tar.gz \
            --clobber
          echo "Assets uploaded successfully!"          
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  # `firegex` carries a C extension — the llhttp binding in `fgex-lib/llhttp/` — so it
  # is no longer one universal wheel. A wheel is built per platform and per interpreter
  # here, and the sdist goes up beside them: where no wheel matches, pip compiles from
  # that. Compiling is a thing pip knows how to do; assuming a system library is already
  # installed is not, which is why the parser ships as source rather than as a `ctypes`
  # binding against the distribution's `libllhttp`.
  build_firegex_wheels:
    needs: [run_tests, engine_tests]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest        # manylinux + musllinux, x86_64
          - os: ubuntu-24.04-arm     # and the same for aarch64
          # arm64 and x86_64 both, from one runner: the Intel macOS image this used to ask
          # for (`macos-13`) has been retired, and a matrix entry naming a runner that no
          # longer exists fails — which, with the publish job needing this one, blocked
          # every PyPI release. The x86_64 wheel is cross-compiled; see CIBW_TEST_SKIP.
          - os: macos-latest
          - os: windows-latest

    steps:
      - uses: actions/checkout@v4
      - name: Extract tag name
        id: tag
        shell: bash
        run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
      - name: Update version
        shell: bash
        run: >-
          sed -i.bak "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/setup.py;
          sed -i.bak "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/firegex/__init__.py;          
      - name: Build wheels
        # 3.x, because 2.x predates CPython 3.14 and quietly built nothing for it: the
        # selector below matched no identifier it knew, which is not an error.
        uses: pypa/cibuildwheel@v3.2.0
        with:
          package-dir: fgex-lib
          output-dir: wheelhouse
        env:
          # The library declares `python_requires>=3.10`, so build what it claims to
          # support and nothing else. PyPy is excluded: the datapath embeds CPython.
          # 3.10 and 3.11 build because the binding declares its 3.12-only module slot
          # only where it exists (`llhttp_module.c`) — unguarded, it failed to compile
          # there, and the whole job with it.
          CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*"
          CIBW_SKIP: "*-musllinux_i686 *_i686 pp*"
          CIBW_ARCHS_MACOS: "arm64 x86_64"
          # An x86_64 wheel built on an arm64 runner cannot be imported there without
          # Rosetta, which the runner is not promised to have; the arm64 wheels, built
          # from the same sources in the same job, are what the smoke test checks.
          CIBW_TEST_SKIP: "*-macosx_x86_64"
          # A wheel that imports is the only kind worth publishing, and the extension is
          # the whole reason this job exists. The checks live in a file rather than in a
          # `-c` string because the one that matters needs a `try`: the parser has to
          # raise its *own* exception on malformed traffic, and this clean environment —
          # no `pyllhttp`, no developer's leftovers — is the only place that is tested.
          CIBW_TEST_COMMAND: python {package}/wheel_smoke.py
      - uses: actions/upload-artifact@v4
        with:
          name: wheels-${{ matrix.os }}
          path: wheelhouse/*.whl

  pypi_publish_firegex:
    needs: build_firegex_wheels
    runs-on: ubuntu-latest
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build          
      - name: Extract tag name
        id: tag
        run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
      - name: Update version in setup.py
        run: >-
          sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/setup.py;
          sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/firegex/__init__.py;          
      # The sdist is what pip falls back to where no wheel matches, so it has to carry
      # the parser's C sources — `MANIFEST.in` is what puts them in.
      - name: Build the sdist
        run: cd fgex-lib && python -m build --sdist && mv ./dist ../
      - name: Collect the wheels built for every platform
        uses: actions/download-artifact@v4
        with:
          pattern: wheels-*
          merge-multiple: true
          path: dist
      - name: Publish package
        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}

  pypi_publish_fgex_alias:
    needs: [run_tests, engine_tests]
    runs-on: ubuntu-latest
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build          
      - name: Extract tag name
        id: tag
        run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT
      - name: Update version in setup.py
        run: >-
          sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/fgex-pip/setup.py;          
      - name: Build package
        run: cd fgex-lib/fgex-pip && python -m build && mv ./dist ../../
      - name: Publish package
        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN_FGEX }}