import asyncio import contextlib import copy import time import uuid from contextvars import ContextVar from application.coordinator import command_lock from application.snapshots import configuration, public, restore, snapshot from mcp_api.auth import MCPError, require from mcp_api.store import digest request_context = ContextVar("firegex_mcp_request", default=None) def revision(value): return digest(configuration(value)) def diff(before, after, path=""): """Bounded field-level changes; identities make list changes readable to an agent.""" changes = _diff(configuration(before), configuration(after), path) return changes[:100] + ([{"omitted_changes": len(changes) - 100}] if len(changes) > 100 else []) def _diff(before, after, path): if before == after: return [] if isinstance(before, dict) and isinstance(after, dict): result = [] for key in sorted(before.keys() | after.keys()): result.extend(_diff(before.get(key), after.get(key), path + "/" + key)) return result if isinstance(before, list) and isinstance(after, list): identity = {"services": "service_id", "addresses": "address_id", "filters": "filter_id", "regexes": "regex_id"}.get(path.rsplit("/", 1)[-1]) if identity and all(isinstance(v, dict) and identity in v for v in before + after): return _diff({v[identity]: v for v in before}, {v[identity]: v for v in after}, path) return _diff({str(i): v for i, v in enumerate(before)}, {str(i): v for i, v in enumerate(after)}, path) def brief(v): if path.rsplit("/", 1)[-1] in {"tls_key", "tls_cert", "password", "access_token"} and v: return {"redacted": True, "sha256": digest(v)} if isinstance(v, (list, dict)) or isinstance(v, str) and len(v) > 200: return {"sha256": digest(v), "size": len(v)} return v return [{"path": path, "before": brief(before), "after": brief(after)}] class Control: def __init__(self, store, runtime): self.store, self.runtime = store, runtime self.tasks = {} self.reaper = None self.epoch = uuid.uuid4().hex def change(self, change_id): change = self.store.get("change", change_id) if not change: raise MCPError("NOT_FOUND", "Unknown change/job ID") return change def receipt(self, change): return {k: v for k, v in change.items() if k not in {"before", "candidate", "result"}} def replay(self, request): existing = self.store.request_get(request["actor"], request["id"]) if not existing: return None if existing["fingerprint"] != request["fingerprint"]: raise MCPError("REQUEST_ID_CONFLICT", "This request_id was used with different arguments") change = self.change(existing["change_id"]) if change.get("result"): return {**change["result"], "change": self.receipt(change)} return {"ok": True, "data": {"job_id": change["id"]}, "change": self.receipt(change)} async def mutate(self, operation, before, candidate, options, apply, reversible=True): require("operator") current_revision = revision(before) exists = before["scope"] != "service" or before["service"] is not None if exists and options.if_revision != current_revision: raise MCPError("REVISION_CONFLICT", "Read the current configuration and use its revision", current_revision=current_revision) if not exists and options.if_revision is not None and options.if_revision != current_revision: raise MCPError("REVISION_CONFLICT", "The new service has a different initial revision", current_revision=current_revision) if options.dry_run: return {"ok": True, "data": {"dry_run": True, "revision": current_revision, "diff": diff(before, candidate)}} if not reversible and options.confirm_within_seconds: raise MCPError("INVALID_ARGUMENT", "This administrative operation does not support timed rollback") for pending in self.store.pending("awaiting_confirmation"): if pending["status"] != "awaiting_confirmation": continue overlaps = ("instance" in {before["scope"], pending["scope"]} or before["scope"] == pending["scope"] and before.get("service_id") == pending.get("service_id")) if overlaps: raise MCPError("CHANGE_PENDING_CONFIRMATION", "Confirm or roll back the pending change first", change_id=pending["id"]) actor = require() change_id = "chg_" + uuid.uuid4().hex change = {"id": change_id, "operation": operation, "actor": actor["name"], "actor_id": actor["id"], "scope": before["scope"], "service_id": before.get("service_id"), "created_at": time.time(), "status": "queued", "before_revision": current_revision, "before": before, "candidate": candidate, "diff": diff(before, candidate), "rollback_available": reversible, "confirmation_required": bool(options.confirm_within_seconds)} self.store.queue_change(change, request_context.get()) async def execute(): change["status"] = "applying" self.store.put("change", change_id, change) try: data = await apply() actual = snapshot(before["scope"], before.get("service_id")) change["after_revision"] = revision(actual) change["status"] = "awaiting_confirmation" if options.confirm_within_seconds else "applied" change["diff"] = diff(before, actual) if options.confirm_within_seconds: change["confirm_by"] = time.time() + options.confirm_within_seconds result = {"ok": True, "data": {**(data or {}), "revision": change["after_revision"]}, "change": self.receipt(change)} except BaseException as error: if isinstance(error, (KeyboardInterrupt, SystemExit)): raise change["status"] = "rolling_back" self.store.put("change", change_id, change) try: if reversible: await restore(before) change["status"] = "rolled_back" if reversible else "failed" except Exception as rollback_error: change["status"] = "rollback_failed" change["rollback_error"] = str(rollback_error) detail = getattr(error, "data", None) or { "code": "CANCELLED" if isinstance(error, asyncio.CancelledError) else "APPLY_FAILED", "message": str(getattr(error, "detail", error)), } result = {"ok": False, "error": detail, "change": self.receipt(change)} change["finished_at"] = time.time() change["result"] = result self.store.put("change", change_id, change) return result if options.background: async def run(): async with command_lock: # Nothing may change between queuing and actually acquiring the lock. if revision(snapshot(before["scope"], before.get("service_id"))) != current_revision: change["status"] = "conflict" change["result"] = {"ok": False, "error": {"code": "REVISION_CONFLICT", "message": "Configuration changed while the job was queued"}} self.store.put("change", change_id, change) return await execute() task = asyncio.create_task(run()) self.tasks[change_id] = task task.add_done_callback(lambda _: self.tasks.pop(change_id, None)) return {"ok": True, "data": {"job_id": change_id}, "change": self.receipt(change)} return await execute() async def rollback(self, change_id, automatic=False): change = self.change(change_id) if not change["rollback_available"]: raise MCPError("NOT_REVERSIBLE", "This operation does not have a configuration rollback") if change["status"] == "rolled_back": return {"ok": True, "change": self.receipt(change)} if change["status"] not in {"applied", "awaiting_confirmation", "confirmed"}: raise MCPError("INVALID_CHANGE_STATE", "This change cannot be rolled back", status=change["status"]) actual = snapshot(change["scope"], change.get("service_id")) if revision(actual) != change["after_revision"]: if automatic: change["status"] = "recovery_required" change["recovery_reason"] = "A newer configuration exists; automatic rollback would overwrite it" self.store.put("change", change_id, change) raise MCPError("REVISION_CONFLICT", "A newer change exists; rollback would overwrite it", current_revision=revision(actual)) change["status"] = "rolling_back" self.store.put("change", change_id, change) try: await restore(change["before"]) change["status"] = "rolled_back" change["confirmation_required"] = False except Exception as error: change["status"] = "rollback_failed" change["rollback_error"] = str(error) raise finally: self.store.put("change", change_id, change) return {"ok": True, "change": self.receipt(change), "data": {"revision": revision(snapshot(change["scope"], change.get("service_id")))}} async def confirm(self, change_id): change = self.change(change_id) if change["status"] == "confirmed": return {"ok": True, "change": self.receipt(change)} if change["status"] != "awaiting_confirmation": raise MCPError("INVALID_CHANGE_STATE", "This change is not waiting for confirmation") if time.time() >= change["confirm_by"]: await self.rollback(change_id, automatic=True) raise MCPError("CONFIRMATION_EXPIRED", "The confirmation deadline has expired; the change was rolled back") if revision(snapshot(change["scope"], change.get("service_id"))) != change["after_revision"]: raise MCPError("REVISION_CONFLICT", "The configuration changed before confirmation") change.update(status="confirmed", confirmation_required=False) self.store.put("change", change_id, change) return {"ok": True, "change": self.receipt(change)} async def recover(self): for change in self.store.pending("applying", "rolling_back", "queued"): if change["status"] in {"applying", "rolling_back"}: try: if change["rollback_available"]: actual = revision(snapshot(change["scope"], change.get("service_id"))) expected = {change["before_revision"], revision(change["candidate"])} if change.get("after_revision"): expected.add(change["after_revision"]) if actual not in expected: raise RuntimeError("Interrupted change has an unrecognized configuration; review before restoring a snapshot") if actual != change["before_revision"]: await restore(change["before"]) change["status"] = "rolled_back" if change["rollback_available"] else "recovery_required" except Exception as error: change.update(status="recovery_required", recovery_reason=str(error)) change["result"] = {"ok": False, "error": {"code": "INTERRUPTED", "message": "The backend restarted during this operation"}, "change": self.receipt(change)} self.store.put("change", change["id"], change) elif change["status"] == "queued": change.update(status="cancelled", result={"ok": False, "error": { "code": "INTERRUPTED", "message": "The backend restarted before this job began"}}) self.store.put("change", change["id"], change) await self.expire() async def expire(self): for change in self.store.pending("awaiting_confirmation"): if change["status"] == "awaiting_confirmation" and time.time() >= change["confirm_by"]: with contextlib.suppress(Exception): await self.rollback(change["id"], automatic=True) self.store.expire_artifacts() async def start(self): await self.recover() async def reap(): while True: await asyncio.sleep(0.5) async with command_lock: await self.expire() self.reaper = asyncio.create_task(reap()) async def stop(self): if self.reaper: self.reaper.cancel() with contextlib.suppress(asyncio.CancelledError): await self.reaper tasks = list(self.tasks.values()) for task in tasks: task.cancel() await asyncio.gather(*tasks, return_exceptions=True)