"""GPU worker registry.

A *worker* is a remote box that runs the standalone worker agent
(``abstract_hugpy_dev.worker_agent``), exposes an HTTP inference endpoint, and
joins this central node so its GPU(s) can serve one or more models from the
manifest.

This module is the single source of truth for the pool. It owns:

    - persistence of the worker list to a JSON file beside the model manifest
      (so the pool survives restarts),
    - registration / heartbeat / removal,
    - model assignment (which worker may serve which model_key),
    - liveness (a worker is ``online`` only if it has heartbeat-ed recently),
    - selection (pick an online worker that is assigned + ready for a model).

Routing (chat/streaming) and the ``/llm/workers`` routes are dumb consumers of
the functions exported here.
"""
from __future__ import annotations

import os
import json
import time
import uuid
import threading
from contextlib import contextmanager
from typing import Any, Dict, List, Optional

try:
    import fcntl  # POSIX advisory file locks — cross-process coordination.
except ImportError:  # pragma: no cover - non-POSIX fallback
    fcntl = None

from .schemas import settings

import logging
logger = logging.getLogger(__name__)


# ── assignment memory (daylight 4b) ─────────────────────────────────────────
# Operator designations are WORKER-lifetime, not row-lifetime: a console
# dead-worker sweep DELETE (or a registry loss) used to wipe a worker's
# models + per-model spill, so a re-register came back empty (2026-07-03:
# computron lost 4 of 7 designations). Every assign/unassign snapshots the
# worker's designations here, keyed by its persistent worker id; a fresh-row
# re-register with a known id restores them. Deleting a row deliberately does
# NOT delete its memory — that's the point.

def _assign_memory_path() -> str:
    return os.path.join(os.path.dirname(settings.manifest_path),
                        "worker_assignments.json")


def _load_assign_memory() -> Dict[str, Any]:
    try:
        with open(_assign_memory_path(), "r", encoding="utf-8") as fh:
            data = json.load(fh)
        return data if isinstance(data, dict) else {}
    except (OSError, ValueError):
        return {}


def _remember_assignments(worker: Dict[str, Any]) -> None:
    """Snapshot one worker's designations (models + spill) into the memory."""
    wid = worker.get("id")
    if not wid:
        return
    try:
        mem = _load_assign_memory()
        prior = mem.get(wid) if isinstance(mem.get(wid), dict) else {}
        entry = {
            "name": worker.get("name"),
            "models": list(worker.get("models") or []),
            "spill_by_model": dict(worker.get("spill_by_model") or {}),
            "remembered_at": _now(),
        }
        # Durable HARDWARE FACTS ride the same sidecar so they survive even a
        # full registry loss (operator addendum 2026-07-24): carry the last-known
        # totals, advance-only — never overwrite a remembered fact with an absent
        # one, so a re-register during a transient probe miss can't erase them.
        gpu_known = (worker.get(_GPU_TOTAL_DURABLE_KEY)
                     or prior.get(_GPU_TOTAL_DURABLE_KEY))
        ram_known = (worker.get(_RAM_TOTAL_DURABLE_KEY)
                     or prior.get(_RAM_TOTAL_DURABLE_KEY))
        if gpu_known:
            entry[_GPU_TOTAL_DURABLE_KEY] = gpu_known
        if ram_known:
            entry[_RAM_TOTAL_DURABLE_KEY] = ram_known
        mem[wid] = entry
        tmp = _assign_memory_path() + ".tmp"
        with open(tmp, "w", encoding="utf-8") as fh:
            json.dump(mem, fh, indent=1)
        os.replace(tmp, _assign_memory_path())
    except Exception as exc:  # noqa: BLE001 — memory is best-effort, never fatal
        logger.warning("assignment memory write failed for %s: %s", wid, exc)


def forget_assignment_memory(worker_id: str) -> str:
    """SANCTIONED maintenance path to remove one GHOST entry from the
    assignment-memory sidecar (``worker_assignments.json``).

    The design above (deleting a live row does NOT delete its memory) stays
    intact: this refuses to touch any id that still has a live row in
    ``workers.json`` — it can only forget ids that are ALREADY absent from the
    live registry (stray/malformed ids, one-off manual pokes, a worker that
    will never come back under that id). Callers that want to retire a live
    worker's designations should remove/unassign the live row first (or just
    leave the memory — that is the intended durability).

    Returns ``"forgot"`` on success or ``"unknown"`` if ``worker_id`` was not
    present in memory at all. Raises ``ValueError`` if ``worker_id`` is
    currently live (refused).
    """
    if worker_store.get(worker_id) is not None:
        raise ValueError(
            f"worker {worker_id} is LIVE in the registry — assignment memory "
            "may only be forgotten for ids absent from the live registry"
        )
    mem = _load_assign_memory()
    if worker_id not in mem:
        return "unknown"
    del mem[worker_id]
    tmp = _assign_memory_path() + ".tmp"
    with open(tmp, "w", encoding="utf-8") as fh:
        json.dump(mem, fh, indent=1)
    os.replace(tmp, _assign_memory_path())
    return "forgot"


def _default_workers_path() -> str:
    """Sit the worker registry next to the model manifest (…/projects/)."""
    return os.path.join(os.path.dirname(settings.manifest_path), "workers.json")


# A worker that hasn't checked in within this window is considered offline.
HEARTBEAT_TIMEOUT_SECONDS = 45.0


def _provision_stall_seconds() -> float:
    """Forward-progress silence (seconds) after which a provisioning entry stops
    reading as an in-flight pull.

    Same shape + rationale as comms.jobs._stall_seconds (the orphan-job fix):
    read at COMPUTE time so an operator can retune without a restart, and a bad
    value degrades to the default rather than raising into a read path.

    Default 600s. Deliberately MUCH larger than the 45s heartbeat timeout: a
    real pull can legitimately go quiet for minutes (a slow segment, a stalled
    HF mirror, a big file's final flush), and calling a live transfer "dead"
    would strip its eviction guard mid-write — the one truly destructive
    mistake available here. Offline workers are already caught by the cheaper
    liveness gate, so this window only has to cover the ONLINE-but-wedged case.
    """
    raw = (os.environ.get("HUGPY_PROVISION_STALL_SECONDS") or "").strip()
    if not raw:
        return 600.0
    try:
        v = float(raw)
        return v if v > 0 else 600.0
    except ValueError:
        return 600.0


def _live_provisioning(worker: Dict[str, Any]) -> set:
    """The subset of ``worker['provisioning']`` that is a GENUINELY LIVE pull.

    Defect (operator, 2026-07-16): a ``provisioning`` entry was immortal. The
    worker announces the list in its heartbeat and removes an entry in a
    ``finally``; if the process dies mid-pull, that ``finally`` never runs, so
    central reported "provisioning" forever (observed: op offline 2h+, still 4
    entries; ae online with 63 entries and ZERO bytes moving).

    This is the orphan-job defect class — state that ages on writes which STOP
    ARRIVING when the writer dies. The recorded lesson: age on PROGRESS, not on
    presence-in-a-list. So an entry is live only when BOTH hold:

      1. the worker is alive       -> REUSES ``_is_online`` (the single existing
                                      staleness notion; no second rule invented)
      2. its bytes are moving      -> a ``provision_progress`` entry whose
                                      ``done_bytes`` advanced within the stall
                                      window, per the central-stamped
                                      ``progressed_at`` clock (see ``heartbeat``)

    Why (2) needs a central clock rather than ``frac > 0``: op's dead pull is
    frozen at ``frac=0.0722`` with 1.8GB done. A truthy frac only proves bytes
    moved ONCE — never that they are moving NOW. Only elapsed-time-since-advance
    can tell a live 7% from a corpse stuck at 7%.

    QUEUED-NOT-STALLED (why absence of an entry is not evidence of death): the
    worker adds a key to ``_provisioning`` at KICK time but only creates a
    ``_provision_progress`` entry once its download callback fires, and
    ``WORKER_PROVISION_CONCURRENCY`` defaults to 1. So ae's 63 progress-less
    entries are models QUEUED behind the semaphore, not wedged ones — correctly
    NOT in-flight (nothing is transferring), and equally correctly NOT
    eviction-protected (they have no bytes on disk to protect).

    Fail-SAFE toward the live case: if the clock is missing/garbage on an ONLINE
    worker with a progress entry, treat it as live. A false "live" costs a
    delayed console pill; a false "dead" could unprotect a real in-flight write.
    """
    prov = set(worker.get("provisioning") or [])
    if not prov or not _is_online(worker):
        # Offline/stale worker: nothing it last claimed is in flight, because
        # nothing of it is running. This is the op case.
        return set()
    progress = worker.get("provision_progress") or {}
    if not isinstance(progress, dict):
        return set()
    now = _now()
    window = _provision_stall_seconds()
    live = set()
    for mk in prov:
        entry = progress.get(mk)
        if not isinstance(entry, dict):
            continue          # queued behind the concurrency semaphore (ae case)
        ts = entry.get("progressed_at")
        if ts is None:
            live.add(mk)      # fail-safe: pre-clock/legacy entry on a live worker
            continue
        try:
            if (now - float(ts)) <= window:
                live.add(mk)
        except (TypeError, ValueError):
            live.add(mk)      # fail-safe: never unprotect on a garbage clock
    return live


def tracked_pkg_name() -> str:
    """Distribution name workers track + central reports its version of.

    Must match the worker's ``--pkg-name`` (``WORKER_PKG_NAME``). Default is the
    dev distribution.
    """
    return os.environ.get("HUGPY_PKG_NAME", "abstract_hugpy_dev")


def required_pkg_version() -> Optional[str]:
    """The dev package version central wants every worker to be running.

    Advertised back to workers in every register/heartbeat response. Resolution
    order:
      1. ``HUGPY_REQUIRED_PKG_VERSION`` env (explicit pin), then
      2. a ``required_pkg_version`` file beside the manifest, then
      3. **central's own installed version** of the tracked dist.

    (3) is the zero-config path: the existing deploy (`pip install -U
    <dist>` on central) becomes the signal — workers converge to whatever
    version central is itself running. ``None`` (dist not installed, no override)
    means "not managing versions" and workers never self-update.
    """
    env = os.environ.get("HUGPY_REQUIRED_PKG_VERSION")
    if env and env.strip():
        # Explicit operator pin — honored verbatim, INCLUDING a PEP 440 local
        # ("+build") version, for a fleet deliberately set up to install from
        # central's private --pkg-index.
        return env.strip()

    # The file/installed fallbacks must resolve to a PUBLICLY installable version.
    # A local version (contains "+", e.g. "0.1.51+c8b13590d") only exists on
    # central's private index; advertising one to the common PyPI-based worker
    # makes its self-update fail on a version pip can't find (rc=1 every
    # heartbeat) and would force a downgrade off a newer public release. So a
    # local fallback version means "not managing versions" → workers stay put.
    def _public(v: Optional[str]) -> Optional[str]:
        return v if (v and "+" not in v) else None

    path = os.environ.get("HUGPY_REQUIRED_PKG_VERSION_FILE") or \
        os.path.join(os.path.dirname(settings.manifest_path), "required_pkg_version")
    try:
        with open(path, "r", encoding="utf-8") as fh:
            pinned = fh.read().strip()
        if pinned:
            return _public(pinned)
    except OSError:
        pass
    # Do NOT auto-derive a pin from central's own installed version. That dev
    # build is frequently a local "+build" (e.g. 0.1.41+phone5) that PyPI workers
    # can't install, and even a clean value would silently downgrade a worker on
    # a newer public release. Version management is therefore OPT-IN: set a clean
    # public version via HUGPY_REQUIRED_PKG_VERSION or the required_pkg_version
    # file. Otherwise central does not manage worker versions (workers stay put).
    return None


def pkg_index_dir() -> str:
    """Directory of built wheels that central serves as a PEP-503 simple index.

    The ``sync.trigger`` build drops the freshly-built dev wheel here. Override
    with ``HUGPY_PKG_INDEX_DIR``; defaults to a ``pip_index`` dir beside the
    model manifest.
    """
    return os.environ.get("HUGPY_PKG_INDEX_DIR") or \
        os.path.join(os.path.dirname(settings.manifest_path), "pip_index")


def _now() -> float:
    return time.time()


# ── operator model BLOCK (central serving-pool primitive) ────────────────────
# A blocked model is removed from the pool: never a routing candidate, never a
# warm/provision target, never a fallback default. The registry lives in the F4
# settings store (comms.blocklist); these are guarded thin wrappers so a
# blocklist read can NEVER raise into routing (fail-open = not blocked, because
# a routing gate that 500s is worse than a momentarily-unblocked model).
def _model_blocked(model_key: str) -> bool:
    try:
        from abstract_hugpy_dev.comms.blocklist import is_blocked
        return is_blocked(model_key)
    except Exception:  # noqa: BLE001 — never let the block gate break selection
        return False


def _blocked_keys() -> set:
    try:
        from abstract_hugpy_dev.comms.blocklist import blocked_keys
        return blocked_keys()
    except Exception:  # noqa: BLE001
        return set()


def _is_online(worker: Dict[str, Any]) -> bool:
    last = worker.get("last_seen") or 0
    return (_now() - last) <= HEARTBEAT_TIMEOUT_SECONDS


def _public_view(worker: Dict[str, Any]) -> Dict[str, Any]:
    """The shape returned to API callers — derived ``status`` included.

    ``status`` is *liveness* (online/offline from last_seen). ``admission`` is the
    operator gate (pending/approved/blocked) and is independent of liveness. Rows
    written before the admission feature have no ``admission`` key; they are
    grandfathered to ``approved`` here so an existing fleet keeps serving.
    """
    return {
        **worker,
        **_vram_summary(worker),
        **_ram_summary(worker),
        # Derived local-storage view + guarded LRU eviction proposal, recomputed
        # on every read from already-stored fields (same pure-function pattern as
        # the vram/ram summaries above; no daemon, no auto-fire — nothing deletes
        # here). Overwrites the raw ``storage`` heartbeat field with the enriched
        # console-facing shape (over_budget + proposed_evictions[]).
        "storage": storage_proposal(worker),
        # IN-FLIGHT PULLS ONLY (2026-07-16). The raw record keeps whatever the
        # worker last announced; the PUBLIC view reports only pulls that are
        # actually moving, so a dead/stalled entry can never render as an
        # active transfer ("defaults are promises" — a row that says "working
        # on it" when nothing is working is a lie). Derived here, on every read,
        # like status/storage above — no daemon, no sweep. The console's
        # ⏳ pulling pill and its provision_progress % both key off this list,
        # so an assigned-but-absent model correctly falls through to "missing".
        "provisioning": sorted(_live_provisioning(worker)),
        "status": "online" if _is_online(worker) else "offline",
        "admission": worker.get("admission", "approved"),
        # SYSTEM-authored placement grants (Phase 1 item 2) — separate from the
        # operator-designated ``models`` list. Never treat a missing key as
        # absence-of-feature; always surface the (possibly empty) dict so
        # console/tests can see grants land and clear.
        "grants": dict(worker.get("grants") or {}),
    }


def _clamp_limits(limits: Dict[str, Any], caps: Dict[str, Any]) -> Dict[str, Any]:
    """Clamp operator limits to the worker's own configured caps.

    The worker's unit config is authoritative ("central shall be forced to
    view that as its max") — central can set anything LESS, never more."""
    out: Dict[str, Any] = {}
    for k, v in limits.items():
        cap = caps.get(k)
        if cap is not None:
            try:
                v = min(float(v), float(cap))
                if k == "threads":
                    v = int(v)
            except (TypeError, ValueError):
                continue
        out[k] = v
    return out


# Default runtime-environment tier. A worker that doesn't report its env (older
# agent) and a model with no explicit requirement both resolve to this, so a
# pre-feature fleet keeps matching exactly as before the tier gate existed.
DEFAULT_ENV_TIER = "stable"


def _model_env_tiers() -> Dict[str, str]:
    """Operator map of model -> REQUIRED env tier.

    Parsed from ``HUGPY_MODEL_ENV_TIERS`` = ``"key:tier,key2:tier"`` (e.g.
    ``"Qwen3.6-27B-AEON:edge"``). A model not listed requires the default tier,
    so this whole gate is a no-op until the operator maps a model.
    """
    out: Dict[str, str] = {}
    for part in os.environ.get("HUGPY_MODEL_ENV_TIERS", "").split(","):
        part = part.strip()
        if not part or ":" not in part:
            continue
        key, _, tier = part.rpartition(":")
        key, tier = key.strip(), tier.strip().lower()
        if key and tier:
            out[key] = tier
    return out


def env_tier_for_model(model_key: str) -> str:
    """The runtime-env tier ``model_key`` requires (alias-tolerant lookup)."""
    tiers = _model_env_tiers()
    if not tiers:
        return DEFAULT_ENV_TIER
    wanted = _match_keys(model_key)
    for key, tier in tiers.items():
        if key == model_key or (_match_keys(key) & wanted):
            return tier
    return DEFAULT_ENV_TIER


def _worker_env_tier(worker: Dict[str, Any]) -> str:
    """The env tier a worker ADVERTISES (from its own venv, via register/heartbeat).

    Workers that don't report an env (older agents) are treated as serving the
    default tier — the grandfather rule that keeps a pre-feature fleet routing
    unchanged. An edge model therefore only lands on a worker that AFFIRMATIVELY
    advertises the edge env.
    """
    env = worker.get("env")
    tier = env.get("tier") if isinstance(env, dict) else None
    if tier is None:
        return DEFAULT_ENV_TIER
    return str(tier).strip().lower() or DEFAULT_ENV_TIER


def _engine_unusable(worker: Dict[str, Any]) -> bool:
    """True only when a worker EXPLICITLY reports it has no inference engine.

    Workers that don't report engine status (older agents) are assumed capable,
    so this never excludes a pre-feature fleet — it only skips a worker that
    affirmatively says ``engine.installed == False`` (e.g. llama-cpp missing),
    which would otherwise be picked and fail every request.
    """
    eng = worker.get("engine")
    return isinstance(eng, dict) and eng.get("installed") is False


# Tasks whose worker capability is authoritatively gated ELSEWHERE and must NOT be
# re-filtered by the find_spec-derived task_capabilities map:
#   image-text-to-text — vision uses the stricter engine.supports_vision truth
#     (llama.cpp mtmd build) enforced in resolvers.remote; a transformers-VL worker
#     without llama_cpp would advertise this False and be wrongly skipped here.
_TASK_CAP_GATE_EXCLUDE = {"image-text-to-text"}


def _task_capable(worker: Dict[str, Any], task: Optional[str]) -> bool:
    """Whether ``worker`` can run ``task`` per its advertised ``task_capabilities``.

    Capability-honest, exactly like the engine/vision/tier gates: a worker is
    skipped ONLY when it AFFIRMATIVELY advertises the task as unavailable — the
    2026-07-11 request-time-failure class (a canonical venv missing
    sentence-transformers / whisper / keybert). LEGACY workers (no
    ``task_capabilities`` field) and tasks a worker doesn't enumerate are assumed
    capable, so a pre-feature fleet routes exactly as before. A ``None`` task
    (non-ML routing, e.g. video auto-pick) never gates, and vision defers to the
    stricter engine gate (``_TASK_CAP_GATE_EXCLUDE``).
    """
    if not task or task in _TASK_CAP_GATE_EXCLUDE:
        return True
    caps = worker.get("task_capabilities")
    if not isinstance(caps, dict) or task not in caps:
        return True
    return bool(caps.get(task))


def _comfy_id_lock_capable(worker: Dict[str, Any]) -> bool:
    """Whether ``worker``'s ComfyUI can do identity-locked STILLs — the
    IPAdapter node pack is installed (``comfy.id_lock`` advertised True from the
    agent's object_info probe).

    STRICT / affirmative-only, DELIBERATELY UNLIKE ``_task_capable``'s
    legacy-permissive default: an id_lock request must land on a box that PROVABLY
    has the nodes, because silently degrading to a NON-locked image is forbidden
    (WORKER-SETUP §5b / comfy_runner). A worker with no ``comfy`` block, comfy
    unavailable, or ``id_lock`` != True does NOT qualify. There's no legacy fleet
    to preserve here — id_lock is a brand-new capability, so "unknown" means "not
    yet", not "assume yes".
    """
    comfy = worker.get("comfy")
    if not isinstance(comfy, dict) or not comfy.get("available"):
        return False
    return bool(comfy.get("id_lock"))


def _has_usable_gpu(worker: Dict[str, Any]) -> bool:
    """Whether the worker advertises a GPU with free VRAM (for efficiency ranking).

    Capability-honest, like vision routing: a worker whose llama.cpp build
    AFFIRMATIVELY reports it cannot offload (engine.supports_gpu_offload is
    False) ranks as GPU-less no matter what nvidia-smi shows — n_gpu_layers is
    silently ignored by a CPU-only wheel, so its "GPU" would never be used.
    Older agents that don't report the flag keep their GPU credit (no guessing).
    """
    if (worker.get("engine") or {}).get("supports_gpu_offload") is False:
        return False
    return any((g.get("memory_free") or 0) > 0 for g in (worker.get("gpus") or []))


_GIB = 2 ** 30

# Spill keys that constitute a PERSISTED placement intent. If a (worker, model)
# spill carries ANY of these, it is NOT blank — the capability-aware default is
# never consulted and the persisted contract is honored verbatim (an explicit
# alloc_mode ALWAYS wins). Only a spill with none of these is "blank" and gets
# the feasibility-derived default in spill_for. Kept in sync with the mode/
# budget/band key families in managers.alloc_modes + worker_routes.
_PLACEMENT_SPILL_KEYS = frozenset({
    "alloc_mode", "n_gpu_layers", "leniency_pct", "priority", "priority_device",
    "gpu_mem_gib", "cpu_mem_gib", "gpu_mem_gib_deviation_pct",
    "cpu_mem_gib_deviation_pct", "tensor_split", "threads",
})


def _limit_bytes(worker: Dict[str, Any], key: str) -> Optional[int]:
    """A central limit (limits.<key> in GiB) as bytes, or None if unset."""
    v = (worker.get("limits") or {}).get(key)
    if v is None:
        return None
    try:
        return int(float(v) * _GIB)
    except (TypeError, ValueError):
        return None


def _honest_bar(physical_total, central_limit, worker_usage, external_usage):
    """Thin wrapper over spill.budget_bar so both summaries share ONE import of
    the operator's spec math. Degrades to None (legacy caller decides) on any
    import failure — the summary must never 500 the worker list."""
    try:
        from ......managers.spill import budget_bar
        return budget_bar(physical_total, central_limit, worker_usage, external_usage)
    except Exception:  # noqa: BLE001 — never fail _public_view over the bar math
        return None


def _vram_summary(worker: Dict[str, Any]) -> Dict[str, Any]:
    """Flat GPU/VRAM rollup + the honest budget bar (t13/t14).

    ``vram_total``/``vram_free``/``vram_used`` stay the box-wide driver figures
    from ``gpus[]`` (unchanged — the physical truth). ON TOP, when the worker
    reports the pid_registry split (vram_attributed_bytes) this computes the
    operator's honest bar against the central GPU limit (limits.gpu_mem_gib):
    worker_usage = attributed model VRAM, external_usage = box driver-used minus
    attributed (ComfyUI + foreign + non-hugpy). bar_* fields carry bar_used/
    remaining/encroachment/over_limit + the raw figures; ``bar_semantics`` is
    "central" (a limit is set), "physical" (no limit) or "legacy" (a pre-slice
    worker that never reported the attributed split — the UI then labels it
    honestly instead of drawing a mixed-universe bar).
    All counts are bytes; ``None`` where unknown (never fabricated).
    """
    gpus = [g for g in (worker.get("gpus") or []) if isinstance(g, dict)]
    if not gpus:
        return {"gpu": None, "gpu_count": 0, "vram_total": None,
                "vram_free": None, "vram_used": None,
                "vram_bar_semantics": "legacy", "bar_semantics": "legacy"}
    name   = next((g.get("name") for g in gpus if g.get("name")), None)
    totals = [g.get("memory_total") for g in gpus if g.get("memory_total")]
    frees  = [g.get("memory_free")  for g in gpus if g.get("memory_free") is not None]
    vram_total = sum(totals) if totals else None
    vram_free  = sum(frees)  if frees  else None
    vram_used  = (vram_total - vram_free) if (vram_total is not None and vram_free is not None) else None
    out = {
        "gpu": name,
        "gpu_count": len(gpus),
        "vram_total": vram_total,
        "vram_free": vram_free,
        "vram_used": vram_used,
    }
    attributed = worker.get("vram_attributed_bytes")
    if attributed is None:
        # Pre-slice worker: no honest split available. Leave the driver figures
        # as-is and flag legacy so the UI labels the bar honestly.
        out["vram_bar_semantics"] = "legacy"
        return out
    limit = _limit_bytes(worker, "gpu_mem_gib")
    # external = whatever the driver shows used beyond hugpy's attributed models
    # (ComfyUI, foreign squatters, other apps, CUDA-context slack).
    external = (max(0, vram_used - attributed)
                if vram_used is not None else worker.get("vram_unattributed_bytes"))
    bar = _honest_bar(vram_total, limit, attributed, external)
    out.update(_bar_public_fields(bar, prefix="vram_"))
    return out


def _ram_summary(worker: Dict[str, Any]) -> Dict[str, Any]:
    """Flat RAM rollup + the honest budget bar (t13/t14) — the CPU-tier mirror.

    ``ram_total`` is the box's RAW installed memory. The old ``ram_used`` =
    ``ram_total − free_ram`` was an ARTIFACT: free_ram is ceiling-clamped, so on
    an under-budget box it algebraically collapsed to physical − central_limit
    (ae's phantom "28.9 GB used" = 124.9 − 96). The honest bar replaces it: when
    the worker reports ram_worker_bytes/ram_external_bytes this computes the
    operator's spec against the RAM ceiling (limits.ram_max_gib), and ``ram_used``
    becomes the SPEC bar_used (the fill the chip draws). bar_* fields + raw
    figures ride alongside; ``bar_semantics`` is central/physical/legacy.
    Pre-slice workers (no ram_worker_bytes) keep the OLD ram_used and are flagged
    legacy so the UI can say so rather than draw a mixed-universe bar.
    None where unknown (never fabricated); clamped ≥0.
    """
    ram_total = worker.get("ram_total")
    worker_usage = worker.get("ram_worker_bytes")
    if worker_usage is None:
        # Pre-slice worker: keep the historical (acknowledged-imperfect) figure,
        # flagged legacy. free_ram is the clamped field, matching old behavior.
        free_ram = worker.get("free_ram")
        ram_used = (max(0, ram_total - free_ram)
                    if (ram_total is not None and free_ram is not None) else None)
        return {"ram_total": ram_total, "ram_used": ram_used,
                "ram_bar_semantics": "legacy", "bar_semantics": "legacy"}
    external = worker.get("ram_external_bytes")
    limit = _limit_bytes(worker, "ram_max_gib")
    bar = _honest_bar(ram_total, limit, worker_usage, external)
    out = {"ram_total": ram_total}
    fields = _bar_public_fields(bar, prefix="ram_")
    out.update(fields)
    # ram_used IS the bar fill the chip draws (spec bar_used), so the existing
    # chip prop keeps working while gaining honest semantics.
    out["ram_used"] = fields.get("bar_used")
    return out


def _bar_public_fields(bar: Optional[Dict[str, Any]],
                       prefix: str = "") -> Dict[str, Any]:
    """Project spill.budget_bar's result onto the flat fields the console reads,
    PREFIXED so RAM and VRAM (both spread onto the same record in _public_view)
    never collide: prefix="ram_" -> ram_bar_semantics/ram_bar_used/…; prefix=
    "vram_" -> vram_bar_*. The un-prefixed generic ``bar_*`` keys are ALSO
    written for wire-compat with any caller that reads the shared names (the
    LAST summary spread wins those — RAM, applied second in _public_view). A None
    bar degrades to bar_semantics="legacy" with no numbers so the UI labels
    honestly."""
    def _keyed(d):
        # emit both the prefixed and the generic keys
        out = {}
        for k, v in d.items():
            out[f"{prefix}{k}"] = v
            out[k] = v
        return out
    if not bar:
        return _keyed({"bar_semantics": "legacy"})
    return _keyed({
        "bar_semantics": bar.get("semantics"),
        "bar_used": bar.get("bar_used"),
        "bar_total": bar.get("total"),
        "bar_remaining": bar.get("remaining"),
        "bar_raw_used": bar.get("raw_used"),
        "bar_encroachment": bar.get("encroachment"),
        "bar_over_limit": bool(bar.get("over_limit")),
        "bar_over_by": bar.get("over_by") or 0,
        "bar_worker_usage": bar.get("worker_usage"),
        "bar_external_usage": bar.get("external_usage"),
        "bar_external_headroom": bar.get("external_headroom"),
    })


def _disk_reserve_bytes() -> int:
    """Free-space reserve (bytes) kept on a worker's MODEL-ROOT volume.

    Below this reserve a worker is "over budget" and its COLD local models become
    eviction candidates. Sized to comfortably exceed the largest single model
    pull (~45 GiB per the model_cache header note) so a provision can always land
    after one eviction. Override with ``HUGPY_WORKER_DISK_RESERVE_GIB`` (default
    50). This is DISTINCT from ``HUGPY_MODEL_CACHE_MAX_GIB`` (=450, the separate
    SSD hot-cache bound in managers/serve/model_cache.py) — do not conflate: this
    reserve is on the model-root disk, not the SSD cache.
    """
    try:
        gib = float(os.environ.get("HUGPY_WORKER_DISK_RESERVE_GIB", "50"))
    except (TypeError, ValueError):
        gib = 50.0
    if gib < 0:
        gib = 0.0
    return int(gib * (1 << 30))


def _model_size_bytes(model_key: str) -> Optional[int]:
    """One ASSIGNED model's size per central's manifest, or None if unknowable.

    The same source ``worker_agent/provision.central_total_bytes`` resolves for a
    single pull, read LOCALLY here (central owns the manifest and the model dirs,
    so this needs no HTTP — see allocated_totals for why that matters).

    GGUF honesty: a GGUF dir holds SEVERAL quants, so its directory sum is NOT
    what serving costs. ``effective_bytes`` (gguf_variants_detail) is the quant
    that actually serves — the same number the Models tab shows. Falls back to
    the mtime-cached directory footprint for transformers/comfy.

    None is a FIRST-CLASS answer meaning "central cannot say" (not in the
    manifest / not on disk / sizing raised). Callers MUST count it as unknown and
    report it — never coerce it to 0, which would make an over-subscribed
    assignment set read as comfortably fitting (the exact dishonesty this
    feature exists to remove).
    """
    if not model_key:
        return None
    try:
        # Import depths differ and are NOT interchangeable: this module sits at
        # flask_app/app/functions/imports/utils/, so `routes` is 4 up while the
        # TOP-LEVEL `imports` package (abstract_hugpy_dev.imports — a different
        # tree from this one's own `imports` parent) is 6. Getting this wrong
        # raises ModuleNotFoundError, which an over-broad except would swallow
        # into a permanent "size unknown" — every model silently unsized, an
        # over-subscribed set reading as empty. Logged loudly for that reason.
        from ....routes.llm_storage_routes import _annotate_gguf_size, _annotate_size
        from ......imports.config.models.models_config import get_models_dict
    except Exception as exc:  # noqa: BLE001 — sizing must never break a read
        logger.warning("allocation sizing unavailable (%s) — assigned-set totals "
                       "will report as unknown", exc)
        return None
    try:
        manifest = get_models_dict(dict_return=True) or {}
        entry = manifest.get(model_key)
        if not isinstance(entry, dict):
            return None
        # Work on a COPY: the annotators mutate the dict they are handed, and the
        # registry entry is the cached, shared MODEL_REGISTRY_DICT row.
        model = dict(entry)
        _annotate_gguf_size(model, model_key)   # -> effective_bytes (GGUF quant)
        _annotate_size(model, model_key)        # -> size_bytes (eff or dir sum)
        size = model.get("size_bytes")
        return int(size) if size else None
    except Exception:  # noqa: BLE001 — unknown size is a valid answer here
        return None


def _model_moe_gpu_bytes(model_key: str) -> Optional[int]:
    """The GPU-side need of a detected-MoE GGUF under the expert split (its
    non-expert bytes + mmproj), or None for dense/non-GGUF/unresolvable.

    Same source as _model_size_bytes: the registry row annotated by
    _annotate_gguf_size, whose ``moe`` field rides gguf_variants_detail exactly
    like effective_bytes (spill.gguf_moe_detail caches the header parse per
    file, so this costs the enrichment read it already does — never a
    per-request re-parse). Feasibility uses it so a MoE the split makes
    serveable is never eliminated against its full file size."""
    if not model_key:
        return None
    try:
        from ....routes.llm_storage_routes import _annotate_gguf_size
        from ......imports.config.models.models_config import get_models_dict
        manifest = get_models_dict(dict_return=True) or {}
        entry = manifest.get(model_key)
        if not isinstance(entry, dict):
            return None
        model = dict(entry)                     # copy: annotators mutate
        _annotate_gguf_size(model, model_key)
        moe = model.get("moe") or {}
        nexp = moe.get("non_expert_bytes")
        if not nexp:
            return None
        return int(nexp) + int(model.get("mmproj_bytes") or 0)
    except Exception:  # noqa: BLE001 — MoE sizing is additive; unknown is fine
        return None


def _model_engine(model_key: str) -> Optional[str]:
    """One model's engine/framework from central's registry, lowercased, or None
    when unresolvable. Used to pick the FEASIBLE blank default (GGUF is always
    max-gpu; only a non-GGUF model can default to ram-only). None -> the caller
    degrades to max-gpu (never guess an engine)."""
    if not model_key:
        return None
    try:
        from ......imports.config.main import get_model_config
        cfg = get_model_config(model_key)
        fw = getattr(cfg, "framework", None)
        return str(fw).lower() if fw else None
    except Exception:  # noqa: BLE001 — unknown engine: caller treats as unknown
        return None


# Durable hardware-total fields (operator addendum 2026-07-24): a box's GPU and
# RAM CAPACITY are physical FACTS, not per-session state — they don't change
# between a worker's restarts. Central persists the last KNOWN-GOOD reading per
# worker identity in these fields (mirrors the auto_reap/wildcard durable-field
# pattern: a plain persisted key that register/heartbeat leave intact). They are
# ADVANCE-ONLY from a real reading — a transient empty/partial gpu probe (driver
# not ready right after boot; detect_gpus() returning []) NEVER wipes them — so
# feasibility has totals for every worker central has ever met, and the
# fail-open collapses to genuinely first-contact-only. _remember_hw_totals
# updates them; the *_total_bytes readers prefer the live figure and fall back
# to the durable one.
_GPU_TOTAL_DURABLE_KEY = "gpu_total_bytes_known"
_RAM_TOTAL_DURABLE_KEY = "ram_total_bytes_known"


def _live_gpu_total_bytes(worker: Dict[str, Any]) -> Optional[int]:
    """GPU total from the CURRENT gpus[] reading only (no durable fallback)."""
    gpus = [g for g in (worker.get("gpus") or []) if isinstance(g, dict)]
    totals = [g.get("memory_total") for g in gpus if g.get("memory_total")]
    if not totals:
        return None
    try:
        return int(sum(totals))
    except (TypeError, ValueError):
        return None


def _remember_hw_totals(worker: Dict[str, Any]) -> None:
    """Persist the worker's last KNOWN-GOOD GPU/RAM totals as durable facts.
    ADVANCE-ONLY: called after register/heartbeat merges the new reading, it
    copies a real (non-None, >0) live total into the durable key and NEVER
    overwrites a good durable value with an absent/zero one — so a transient
    probe miss can't erase a fact central already learned. Idempotent + cheap;
    safe to call on every register/heartbeat inside the transaction."""
    live_gpu = _live_gpu_total_bytes(worker)
    if live_gpu:
        worker[_GPU_TOTAL_DURABLE_KEY] = live_gpu
    ram = worker.get("ram_total")
    try:
        ram = int(ram) if ram else None
    except (TypeError, ValueError):
        ram = None
    if ram:
        worker[_RAM_TOTAL_DURABLE_KEY] = ram


def _worker_gpu_total_bytes(worker: Dict[str, Any]) -> Optional[int]:
    """Box-wide TOTAL GPU capacity (bytes) = sum of gpus[].memory_total, the same
    physical-truth source _vram_summary reads — falling back to the DURABLE
    last-known total (_GPU_TOTAL_DURABLE_KEY) when the current reading is missing
    (a re-register window, or a transient empty probe). None only when central
    has NEVER seen a total for this worker (true first contact -> caller degrades
    to max-gpu, never derives ram-only blind)."""
    live = _live_gpu_total_bytes(worker)
    if live:
        return live
    durable = worker.get(_GPU_TOTAL_DURABLE_KEY)
    try:
        return int(durable) if durable else None
    except (TypeError, ValueError):
        return None


def _worker_ram_total_bytes(worker: Dict[str, Any]) -> Optional[int]:
    """The box's RAW installed memory (bytes) — the ``ram_total`` field
    _ram_summary reads, falling back to the DURABLE last-known RAM total when the
    current field is absent (re-register window / older beat). None only when
    central has never seen a RAM total for this worker."""
    for key in ("ram_total", _RAM_TOTAL_DURABLE_KEY):
        v = worker.get(key)
        if v is None:
            continue
        try:
            iv = int(v)
        except (TypeError, ValueError):
            continue
        if iv:
            return iv
    return None


def derived_default_mode(worker: Dict[str, Any], model_key: str) -> str:
    """The FEASIBLE blank default alloc mode for one (worker, model) — engine +
    box-totals aware (operator ruling 2026-07-24). Pure glue over the stdlib
    ``feasible_default_mode``: resolves the engine, effective size, GPU total,
    RAM total from central's authoritative sources and asks the shared math.
    ANY lookup miss degrades to 'max-gpu' (today's blank behavior) — never a
    500, never a guess. This ONLY supplies the default when NOTHING is persisted
    for the model; an explicit alloc_mode always wins upstream."""
    try:
        from ......managers.alloc_modes import feasible_default_mode
        return feasible_default_mode(
            _model_engine(model_key),
            _model_size_bytes(model_key),
            _worker_gpu_total_bytes(worker),
            _worker_ram_total_bytes(worker))
    except Exception:  # noqa: BLE001 — a derivation must never break a read/relay
        return "max-gpu"


def allocated_totals(worker: Dict[str, Any]) -> Dict[str, Any]:
    """Size the worker's ASSIGNMENT SET against its budget — the STRUCTURAL view.

    OPERATOR (2026-07-16): "it should also show how much is needed based on the
    total size of all models allocated". The per-pull refusal ("this 23.5 GiB
    pull won't fit") answers a different, smaller question. This answers: can the
    ASSIGNED SET fit AT ALL? A worker assigned 12 models totalling 180 GiB
    against a 50 GiB budget is over-subscribed BY CONSTRUCTION — no eviction
    order rescues it, and it will wedge on some future call no matter which model
    is unlucky enough to be the one that asks.

    Domain = ``worker['models']`` — the OPERATOR DESIGNATION set written by
    assign_model/unassign_model. NOT the on-disk inventory (lazy-download means
    an assigned model routinely has no files yet — sizing only what landed would
    UNDER-report an over-subscribed set, hiding the very thing this shows) and
    NOT ``grants`` (system-authored, freely evictable, never operator intent).

    Returns::

        {"allocated_total_bytes": int,      # sum of the KNOWN-size models
         "allocated_count": int,            # models in the assignment set
         "allocated_unknown_count": int,    # sizes central couldn't resolve
         "allocated_over_budget_bytes": int}  # total - budget, 0 when it fits

    ``allocated_total_bytes`` is a FLOOR when allocated_unknown_count > 0: the
    unknowns are counted and surfaced, never silently zeroed, so a reader can see
    the number is incomplete rather than trust a comfortable-looking lie.
    """
    models = [m for m in (worker.get("models") or []) if m]
    total = 0
    unknown = 0
    for mk in models:
        size = _model_size_bytes(mk)
        if size:
            total += size
        else:
            unknown += 1
    return {"allocated_total_bytes": total,
            "allocated_count": len(models),
            "allocated_unknown_count": unknown}


def storage_proposal(worker: Dict[str, Any]) -> Dict[str, Any]:
    """Derive a worker's local-STORAGE view + a guarded LRU eviction PROPOSAL.

    A PURE read-time computation over already-stored heartbeat fields — no
    daemon, no background loop, no persistent toggle, and THIS FUNCTION never
    deletes anything (it returns a proposal; a caller must act on it). It is
    spread into every worker read by ``_public_view`` (the always-on storage
    monitoring depiction) and re-run by the ``/reap-approve`` route as its
    central second guard, so the console preview and the approval share one
    source of truth.

    NOTE (2026-07-16): "no auto-fire" describes THIS central preview and the
    operator-gated bulk reaper it feeds — it is NOT a fleet-wide claim. The
    worker's ``worker_agent/budget.py`` auto-evicts on the PROVISION path
    (call-driven only) to seat a model being pulled. That path deliberately
    reuses THIS function's ordering + guard semantics (unprotected candidates,
    ascending last_picked, largest-first among equally-cold) so the console's
    preview and an auto-evict can never disagree about what would go.

    Inputs (all raw worker-record fields):
      - ``worker['storage']``   the worker-reported survey
            ``{cache_used_bytes, disk_free, models:[{model_key, bytes, pinned,
            loaded, loading, provisioning, assigned, protected, why}]}``.
            ABSENT on a pre-feature agent -> a monitoring-only view with an empty
            proposal (the worker must ship this field for the proposal to have a
            per-model inventory).
      - ``worker['disk']``      ``{free_bytes, total_bytes}`` of the model root.
      - ``worker['model_last_picked']`` central LRU signal ``{model_key: epoch}``
            stamped in ``pick_for_model``. A missing entry defaults to 0 (coldest
            -> proposed for eviction first — exactly right for never-served
            test-churn leftovers).
      - ``worker['limits']['disk_cache_gib']`` optional explicit per-worker cap;
            WINS over the free-disk reserve when set.
      - ``worker['loaded_models']`` / ``['loading']`` / ``['provisioning']``
            central slot-merged live truth — the redundant central guard that
            closes the worker reaper's in-process-only loaded gap.
      - ``worker['config']['residency']`` / ``['pinned']`` static/pin attribution.

    Budget (two modes, cheap; explicit cap wins):
      * explicit cap  -> over_budget ``cache_used > cap``;  need ``cache_used-cap``
      * else reserve  -> over_budget ``disk_free < reserve``; need ``reserve-disk_free``

    Proposal (mirrors ``model_cache.evict_for``): domain = RECLAIMABLE candidates
    only (unprotected), sorted ASCENDING by ``last_picked`` (LRU oldest-first),
    greedily accumulating bytes until ``need`` is covered — that subset (possibly
    several models) is ``proposed_evictions``. The console renders it; it computes
    nothing.

    📌 PIN + ALLOCATION HAVE NO BEARING ON EVICTION (operator ruling,
    2026-07-17, verbatim): "the pins only should designate that the model
    allocation survives restarts. the allocation only stipulates the routing for
    that model (to that worker). neither of those should have any bearing on the
    pull or eviction, unless its to do with priority, then a pinned model should
    take higher precidence than unpinned, but even that is trivial".
      * 📌 pin = the model's ALLOCATION survives restarts (and unassign — the
        409). Nothing else. Allocation = ROUTING (which worker answers).
      * A pinned or assigned model's FILES are a normal LRU eviction candidate
        here — ``proposed_evictions`` MAY include pinned files. Evicting them
        leaves pin + allocation untouched; the bytes re-pull on next call.
      * Pin's only eviction role is the trivial FIFO tiebreak below (unpinned
        proposed first at an exact last_picked tie).
      * The ``pinned``/``why`` fields stay HONEST as ATTRIBUTION info (a row can
        read pinned:true / why:"pinned" while protected:false). 🔒static is the
        ONLY durable local-presence guard; loaded/loading/provisioning are
        live-use guards. This removed the day-one tripwire (attribution/routing
        masquerading as a disk shield). unassign-409 is UNTOUCHED — that IS pin.
    """
    storage = worker.get("storage")
    reported = isinstance(storage, dict)
    disk = worker.get("disk") or {}

    def _as_int(v):
        try:
            return int(v)
        except (TypeError, ValueError):
            return None

    disk_free = None
    if reported and storage.get("disk_free") is not None:
        disk_free = _as_int(storage.get("disk_free"))
    if disk_free is None and disk.get("free_bytes") is not None:
        disk_free = _as_int(disk.get("free_bytes"))
    disk_total = _as_int(disk.get("total_bytes"))

    cache_used = _as_int(storage.get("cache_used_bytes")) if reported else None
    # ORPHANED (unattributed-on-disk) residue reported by the worker (release-
    # bound field). Passed through verbatim: model dirs / stalled .part sets on
    # disk that match NO current assignment (computron's 5.7G Qwen2.5-VL-3B
    # .part junk). Absent on a pre-2026-07-17 agent -> zeros (feature-off).
    orphaned_bytes = (_as_int(storage.get("orphaned_bytes")) or 0) if reported else 0
    orphaned_count = (_as_int(storage.get("orphaned_count")) or 0) if reported else 0
    orphaned_items = (storage.get("orphaned_items") or []) if reported else []
    last_picked_map = worker.get("model_last_picked") or {}
    limits = worker.get("limits") or {}
    cfg = worker.get("config") or {}
    residency = cfg.get("residency") or {}
    pinned_cfg = cfg.get("pinned") or {}
    # Central slot-merged live truth — closes the reaper's in-process-only
    # loaded_model_keys() gap (it misses slot occupants / answering models).
    loaded_now = set(worker.get("loaded_models") or [])
    loading_now = set(worker.get("loading") or [])
    # LIVE pulls only — a stale/dead-owner entry is neither reported as
    # in-flight nor granted eviction protection. See _live_provisioning.
    provisioning_now = _live_provisioning(worker)

    reserve = _disk_reserve_bytes()

    # ── budget: explicit per-worker cap wins over the free-disk reserve ──────
    cap_gib = limits.get("disk_cache_gib")
    budget_basis = "reserve"
    budget = None
    over_budget = False
    need_bytes = 0
    if cap_gib not in (None, ""):
        cap_bytes = None
        try:
            cap_bytes = int(float(cap_gib) * (1 << 30))
        except (TypeError, ValueError):
            cap_bytes = None
        if cap_bytes is not None:
            budget_basis = "cap"
            budget = cap_bytes
            if cache_used is not None and cache_used > cap_bytes:
                over_budget = True
                need_bytes = cache_used - cap_bytes
    if budget_basis == "reserve":
        # Express the cache-ceiling budget so the console bar (cache_used vs
        # budget) is consistent with the flag: over_budget <=> cache_used > budget
        # <=> disk_free < reserve. need is the free-disk shortfall.
        if disk_free is not None and cache_used is not None:
            budget = cache_used + disk_free - reserve
        if disk_free is not None and disk_free < reserve:
            over_budget = True
            need_bytes = reserve - disk_free

    # ── per-model view + reclaimable candidate domain ───────────────────────
    def _lp(mk):
        v = last_picked_map.get(mk)
        try:
            return float(v) if v is not None else 0.0
        except (TypeError, ValueError):
            return 0.0

    grants_now = worker.get("grants") or {}
    models_out: List[Dict[str, Any]] = []
    candidates: List[tuple] = []   # (last_picked, bytes, model_key)
    raw_models = storage.get("models") if reported else None
    for m in (raw_models or []):
        if not isinstance(m, dict) or not m.get("model_key"):
            continue
        mk = m["model_key"]
        b = _as_int(m.get("bytes")) or 0
        lp = _lp(mk)
        # Central-final protection = worker's own flag OR any redundant central
        # guard (slot-merged loaded/loading, provisioning, static, pin). A model
        # is a candidate ONLY if unprotected on BOTH sides.
        #
        # PROVISIONING is the ONE guard in this chain that is liveness-gated
        # (2026-07-16). The worker's own per-row ``m['provisioning']`` flag is
        # NOT consulted here, unlike loaded/loading: it comes from the same dead
        # heartbeat snapshot as the stale list, so honouring it would re-admit
        # exactly the phantom protection this fix removes (op's dead pull still
        # flags its row). Central instead trusts only _live_provisioning.
        #
        # This LOSES no protection for a real pull: an ONLINE worker with a
        # moving pull is live by construction. And the worker keeps its OWN
        # authoritative guard locally (worker_agent/budget.py's
        # _PROTECTED_REASONS) — a box never deletes under its own live write on
        # central's say-so. What it REMOVES is permanent phantom protection: a
        # dead entry used to make a real, cold, reclaimable file un-evictable
        # forever, silently shrinking the reclaimable pool on a full disk.
        #
        # NOTE (Phase 1 item 2, grant markers): a SYSTEM grant is DELIBERATELY
        # ABSENT from this chain — the opposite of an operator "assigned"
        # designation. A model that is ONLY granted (not assigned/static/loaded)
        # gets no protection here and remains a normal LRU eviction candidate;
        # grants are reclaimable by construction, never a residency guarantee.
        # If a model happens to be BOTH granted and assigned/static/etc., that
        # FILE-PROTECTING designation still protects it as before — the grant
        # itself contributes nothing. (📌pin is NOT file-protecting as of
        # 2026-07-17, so granted+pinned is a candidate — see below.)
        why = m.get("why") or ""
        protected = bool(m.get("protected"))
        is_pinned = bool(m.get("pinned") or pinned_cfg.get(mk))
        # Worker-reported protection is trusted ONLY for reasons that are not
        # pure attribution ("shared/central storage — never reaped", "model
        # store not marked reapable", live-use guards). A released worker
        # (<=0.1.183) still stamps protected/why="pinned" or "assigned" from the
        # old doctrine — strip those two here and let the chain below recompute,
        # or the fleet's stale flags keep the day-one tripwire alive centrally
        # until the next release (keeper, 2026-07-17: ae reported 21/21 pinned
        # rows protected with zero live-use flags — reclaimable pool read as
        # empty on a 700G/429G box).
        if protected and why in ("pinned", "assigned"):
            # Clear the stale label too: if a live-use guard re-protects below,
            # the chain stamps the HONEST reason (loaded/loading/…) instead of
            # leaving attribution vocabulary on a protection flag.
            protected, why = False, ""
        # 📌 pin does NOT protect files (operator ruling, 2026-07-17): "the pins
        # only should designate that the model allocation survives restarts. the
        # allocation only stipulates the routing... neither of those should have
        # any bearing on the pull or eviction". So pin is DELIBERATELY absent
        # from this protection chain — a pinned model's files are a normal LRU
        # candidate; the pin + allocation survive the eviction and the bytes
        # re-pull on next call. `pinned` is still reported below as ATTRIBUTION
        # (m['pinned']) but never sets `protected`/`why`. 🔒static is the ONLY
        # durable local-presence guard; loaded/loading/provisioning are live-use
        # guards. This removes the day-one tripwire that let attribution/routing
        # masquerade as a disk shield.
        if not protected:
            if str(residency.get(mk) or "").lower() == "static":
                protected, why = True, why or "static"
            # NO `assigned` branch (operator ruling 2026-07-17): "the allocation
            # only stipulates the routing for that model... neither of those
            # should have any bearing on the pull or eviction." Assignment is
            # attribution, same as pin — worker-side budget._is_protected has
            # said so since f1894b2; this chain protecting `assigned` was the
            # central half of the same day-one tripwire (on a box whose on-disk
            # models are all assigned — ae, op — the reclaimable pool read as
            # permanently empty).
            elif mk in loaded_now or m.get("loaded"):
                protected, why = True, why or "loaded"
            elif mk in loading_now or m.get("loading"):
                protected, why = True, why or "loading"
            elif mk in provisioning_now:
                protected, why = True, why or "provisioning"
            elif is_pinned and not why:
                # ATTRIBUTION-only annotation: honest `why` for a pinned model
                # that has no other protecting flag, while `protected` stays
                # False (it remains a candidate). A bare pinned row therefore
                # shows why="pinned" but IS eligible for the proposal below.
                why = "pinned"
        models_out.append({
            "model_key": mk,
            "bytes": b,
            "last_picked": lp or None,     # None = never served through central
            "protected": protected,
            "why": why,
            "granted": mk in grants_now,   # SYSTEM marker only — confers no protection
            "pinned": is_pinned,           # ATTRIBUTION only — confers no eviction protection (2026-07-17)
            "loaded": bool(m.get("loaded") or mk in loaded_now),
            "loading": bool(m.get("loading") or mk in loading_now),
            # LIVE pulls only (not the worker's stale per-row flag) — this is
            # what the console renders as "⏳ pulling"; a dead pull must read
            # as missing, never as an active transfer.
            "provisioning": mk in provisioning_now,
            "assigned": bool(m.get("assigned")),
        })
        if not protected:
            candidates.append((lp, is_pinned, b, mk))

    # ── reclaimable-count trace (operator ask, 2026-07-17): "yell out the
    # course of the reclaimable count". The worker half is traced by
    # scan_keys_considered → scan_skip_reasons → scan_rows; this logs the
    # CENTRAL half — rows in vs candidates out of the final-guard chain — and
    # shouts when the chain zeroes a non-empty report (the collapse signature
    # that cost a day to localize by hand).
    _rows_in = len([m for m in (raw_models or []) if isinstance(m, dict) and m.get("model_key")])
    if _rows_in and not candidates:
        logger.warning(
            "reclaimable collapse on %s: worker reported %d storage rows but 0 "
            "survived central's guard chain (protected breakdown: %s)",
            worker.get("name") or worker.get("id", "?")[:8], _rows_in,
            {w: sum(1 for m in models_out if m.get("why") == w)
             for w in {m.get("why") for m in models_out if m.get("protected")}})

    proposed: List[Dict[str, Any]] = []
    proposed_free = 0
    if over_budget and need_bytes > 0 and candidates:
        # LRU oldest-first (ascending last_picked). Then a TRIVIAL 📌pin tiebreak
        # (operator called it "trivial and likely unnecessary", 2026-07-17):
        # among equally-stale candidates, propose UNPINNED before PINNED (False
        # sorts first) — a pinned model gets a hair of extra precedence only at
        # an exact last_picked tie. Then largest-first so the budget clears in
        # the fewest deletes; stable key tiebreak. IDENTICAL to budget.fit_plan,
        # so central's preview and the worker's auto-evict agree on what goes.
        candidates.sort(key=lambda c: (c[0], c[1], -c[2], c[3]))
        for lp, _pin, b, mk in candidates:
            if proposed_free >= need_bytes:
                break
            proposed.append({"model_key": mk, "bytes": b,
                             "last_picked": lp or None})
            proposed_free += b

    # ── ALLOCATION-LEVEL view (operator, 2026-07-16) ────────────────────────
    # Structural, and TRUE EVEN WHEN NO PULL IS HAPPENING: if the assigned set
    # itself exceeds the budget, the worker is over-subscribed now — the console
    # can surface that BEFORE some unlucky call wedges. Computed on every read
    # (like the vram/ram summaries) and cheap: sizes come from the cached
    # registry + mtime-cached dir walks, not per-model HTTP.
    alloc = allocated_totals(worker)
    alloc_over = 0
    if budget is not None and alloc["allocated_total_bytes"] > budget:
        alloc_over = alloc["allocated_total_bytes"] - budget
    alloc["allocated_over_budget_bytes"] = alloc_over

    # ── ATTRIBUTED vs RESIDENT (2026-07-17) ─────────────────────────────────
    # The operator scare: assignment/pin ATTRIBUTES a model to a worker without
    # putting bytes on disk (lazy download, 7f0e6e8/2a3baeb). The fleet gauge
    # read cache_used/budget, and an over-subscribed ATTRIBUTION set made a box
    # with nothing transferring look like a runaway download storm. Split the two
    # so attribution can NEVER masquerade as disk pressure:
    #   * attributed = the assignment/pin SET's effective size (may exceed disk;
    #     "assigned but not on disk" is a CORRECT resting state, not pressure).
    #   * resident   = bytes ACTUALLY on disk. The worker's measured cache_used is
    #     the authority; the per-model on-disk sum is the fallback/cross-check.
    # The disk-pressure GAUGE is derived from RESIDENT only.
    resident_from_models = sum(int(m.get("bytes") or 0) for m in models_out)
    resident_bytes = cache_used if cache_used is not None else (
        resident_from_models if reported else None)
    attributed = {
        "attributed_total_bytes": alloc["allocated_total_bytes"],
        "attributed_count": alloc["allocated_count"],
        "attributed_unknown_count": alloc["allocated_unknown_count"],
        "attributed_over_budget_bytes": alloc_over,
    }
    resident = {
        # bytes on disk NOW. `resident_bytes` is the number the gauge must use.
        "resident_bytes": resident_bytes,
        "resident_model_bytes": resident_from_models,
        # measured vs summed can disagree (heartbeat lag / non-model files); both
        # surfaced so the console shows the truth instead of averaging a lie.
        "resident_source": ("measured" if cache_used is not None
                            else ("summed" if reported else "unknown")),
        # ORPHANED = on disk but attributed to NO model (leftover dirs + stalled
        # .part sets). A THIRD class distinct from attributed and
        # resident-attributed: junk eating the drive that the allocation ledger
        # never showed. UI label: "unattributed on disk".
        "orphaned_bytes": orphaned_bytes,
        "orphaned_count": orphaned_count,
        "orphaned_items": orphaned_items,
    }
    # The disk-pressure gauge: RESIDENT over budget. Attribution is deliberately
    # excluded — an over-subscribed assignment set is surfaced via
    # attributed_over_budget_bytes (structural), never as a full-disk reading.
    gauge = {
        "gauge_used_bytes": resident_bytes,   # <-- what the UI bar fills to
        "gauge_budget_bytes": budget,
        "gauge_basis": "resident",
        "gauge_over_budget": over_budget,     # already computed from cache_used/disk_free
    }

    return {
        **alloc,
        **attributed,
        **resident,
        **gauge,
        "reported": reported,
        "cache_used_bytes": cache_used,
        "disk_free": disk_free,
        "disk_total": disk_total,
        "reserve": reserve,
        "budget": budget,
        "budget_basis": budget_basis,
        "over_budget": over_budget,
        "need_bytes": need_bytes if over_budget else 0,
        "proposed_free_bytes": proposed_free,
        "proposed_evictions": proposed,
        "models": models_out,
        # Storage REFUSALS reported by the worker: {model_key: {state:"refused",
        # reason, needs_bytes, budget_bytes, reclaimable_bytes, blocked, ...}}.
        # Models whose pull was refused BEFORE it started because even a full
        # FIFO of the reclaimable models couldn't seat them. Passed through
        # VERBATIM — this is the worker's own verdict about its own disk, and
        # central has no better information to second-guess it with. They have
        # no files on disk, so they are deliberately absent from `models` and
        # never appear in a proposal; the console renders them as MISSING with
        # the reason on hover.
        "refused": (storage.get("refused") or {}) if reported else {},
        # SCAN DIAGNOSTICS (slice 3, B) — passed through VERBATIM from the worker
        # survey so a broken/degraded reap scan can never masquerade as a clean
        # empty store (the ae 2026-07-17 defect: rows:0 while 65 models were on
        # disk). The console can surface scan_error / considered≫rows. Absent on a
        # pre-slice-3 worker -> falsy defaults (feature simply off).
        "scan_error": (storage.get("scan_error") or "") if reported else "",
        "scan_keys_considered": (_as_int(storage.get("scan_keys_considered")) or 0) if reported else 0,
        "scan_rows": (_as_int(storage.get("scan_rows")) or 0) if reported else 0,
        "scan_row_errors": (_as_int(storage.get("scan_row_errors")) or 0) if reported else 0,
        # SKIP-REASON HISTOGRAM (slice 5) — passed through VERBATIM so the console
        # can name the ae failure class (not_local/no_config/comfy). Absent on a
        # pre-slice-5 worker -> {} (feature off).
        "scan_skip_reasons": (storage.get("scan_skip_reasons") or {}) if reported else {},
        # REGISTRY SOURCES (slice 6) — per-origin config counts, passed through
        # VERBATIM so the console shows a dead source (discovered==0) in one beat.
        # Absent on a pre-slice-6 worker -> {} (feature off).
        "registry_sources": (storage.get("registry_sources") or {}) if reported else {},
        # EFFECTIVE BUDGET (slice 4, min-wins) — the worker's own resolved
        # min(central disk_cache_gib, worker same-drive declarations) + the source
        # map, passed through VERBATIM so the console can show WHY a number
        # governs (e.g. central 400 wins over worker hot 1500). Absent on a
        # pre-slice-4 worker -> None/{}/False (feature simply off). This is the
        # WORKER's own resolution; central's `budget`/`over_budget` above are its
        # own view and unchanged.
        "budget_effective_bytes": (_as_int(storage.get("budget_effective_bytes"))) if reported else None,
        "budget_sources": (storage.get("budget_sources") or {}) if reported else {},
        "budget_cap_not_applicable": bool(storage.get("budget_cap_not_applicable")) if reported else False,
        # AUTO-REAP MODE (slice 8, Part B) — the per-worker opt-in flag + the last
        # time an auto-fire ran, so the console can show the mode ("auto" vs the
        # default hand-approve) and when it last acted. Read from the WORKER record
        # (central-owned policy), not the worker-reported storage. Default false /
        # None — a worker never opted in reads as hand-approve, today's posture.
        "auto_reap": bool(worker.get("auto_reap")),
        "last_auto_reap_at": worker.get("last_auto_reap_at"),
    }


def _match_keys(model_key: str) -> set:
    """Normalized aliases a model might be named by, for tolerant matching.

    A model can be referenced as its registry key, its hub_id (owner/name), or
    just the trailing name — and with different case. We compare on the set of
    these forms so an assignment made via one spelling still routes a chat that
    uses another. Example: "Qwen/Qwen2.5-Coder-3B-Instruct-GGUF",
    "Qwen2.5-Coder-3B-Instruct-GGUF" and the lowercased variants all match.

    "~"-TAIL UNIFICATION (key-match unification, operator doctrine 2026-07-23).
    Registry keys qualify a base name with its owner via "~" ("Qwen~X",
    "unsloth~X") while workers routinely serve/report the BARE base name ("X").
    Before this, the two spellings never intersected — the k30 class of
    invisible mismatches: a designation carries the ~-qualified key, the
    worker's loaded/models list carries the bare one, and routing looked at a
    box actually holding the model and said "not serveable here". Operator
    doctrine: a specific call may try any same-base sibling — so the "~"-tail
    is an alias exactly like the "/"-tail. "Qwen~X" -> {"Qwen~X", "qwen~x",
    "X", "x"}; bare "X" -> {"X", "x"}; qualified and bare now intersect in BOTH
    directions. (Two different owners of the SAME base intersect too — the
    blocked-sibling guard in _serveable_match is what keeps that from serving a
    BLOCKED sibling under an unblocked name.) Raw forms stay first-class; the
    tails are additions, never replacements.
    """
    if not model_key:
        return set()
    raw = str(model_key).strip()
    forms = {raw, raw.lower()}
    tail = raw.split("/")[-1]
    forms.add(tail)
    forms.add(tail.lower())
    if "~" in raw:
        base = raw.split("~", 1)[1]
        if base:
            forms.add(base)
            forms.add(base.lower())
    return forms


def _serveable_match(model_key: str, wanted: set, serveable) -> bool:
    """True iff one of ``serveable``'s advertised keys names ``model_key``.

    The single membership predicate for every routing match site (the
    eligibility gate, the engine-skip say-why counter, explain_no_worker's
    designation walk). It replaces the old
    ``model_key in serveable or wanted & {alias-union}`` idiom with a
    PER-ADVERTISED-KEY decision, which is what makes the BLOCKED-SIBLING GUARD
    possible: the "~"-tail unification in _match_keys lets a request for an
    unblocked ``A~X`` tail-match an advertised ``B~X`` (different owner, same
    base). If that advertised sibling is BLOCKED, counting the match would
    effectively serve a blocked model under an unblocked name — so an
    alias-only match against a blocked advertised key is skipped. The
    REQUESTED key's own block gate is enforced upstream (workers_for_model
    returns [] before any per-worker work); the blocklist is consulted here
    ONLY for alias-matched (never literal-matched) advertised keys, keeping
    the hot path cheap.
    """
    for m in serveable:
        if m == model_key:
            return True
        if wanted & _match_keys(m):
            if _model_blocked(m):
                continue     # blocked sibling — an alias never launders a block
            return True
    return False


def _wildcard_map() -> Dict[str, bool]:
    """The per-worker WILDCARD ("take all comers") opt-in map {worker_id: True}.

    Operator doctrine 2026-07-23: worker designations are a HARD routing scope;
    an undesignated model "gets in where it fits in" ONLY on workers that
    explicitly opted in as wildcard. Stored in models_config
    (worker_wildcard.json — same store family as the boot_prewarm star); absent
    key = False, so a fleet with no flags set routes exactly as before the
    feature existed (defaults are promises). Fully guarded: a store miss must
    never break the heartbeat/selection hot path — {} (nobody is a wildcard) is
    the safe degradation.
    """
    try:
        from abstract_hugpy_dev.imports.config.models.models_config import (
            worker_wildcard_state)
        return worker_wildcard_state()
    except Exception:  # noqa: BLE001 — never let the flag store break selection
        return {}


def _star_map() -> Dict[str, Any]:
    """The per-worker ⭐ BOOT-LOAD STAR map {worker_id: model_key} (or {}).

    The star's ONLY routing effect (operator RULING 2026-07-23, post-incident:
    "it shouldn't effect anything but priority for ambiguous model calls") is a
    tie-break in worker ranking: when nothing is warm, prefer the worker whose
    boot star == the requested model — that box would boot-load it anyway, so a
    no-warm call lands where the model is (or will soon be) resident. The star is
    NOT keep-warm and has NO reconcile/eviction effect (see agent boot-once +
    worker_routes._reconcile_warm_set). Stored in models_config
    (worker_boot_prewarm.json — same store family as the wildcard flag). Read ONCE
    per pick call (never per candidate). Fully guarded: a store miss must never
    break selection — {} (no star anywhere) is the safe degradation and leaves
    ranking exactly as it was before the star key existed.
    """
    try:
        from abstract_hugpy_dev.imports.config.models.models_config import (
            worker_boot_prewarm_state)
        return worker_boot_prewarm_state()
    except Exception:  # noqa: BLE001 — never let the star store break selection
        return {}


class WorkerStore:
    """Disk-authoritative, multi-process-safe registry of GPU workers.

    Under gunicorn/uwsgi the API runs as several processes, so an in-memory
    dict would split-brain: a worker registered in process A would be invisible
    to a heartbeat or chat request handled by process B (the classic symptom is
    "registers + shows in the UI, but heartbeats 410 and chats never offload").

    To avoid that, ``workers.json`` is the single source of truth: every read
    re-loads it, and every mutation takes an exclusive ``fcntl`` lock, reloads,
    mutates, and writes back atomically. A short-lived in-process RLock just
    keeps threads within one process from racing the same fd.
    """

    # Read-cache TTL: the console polls /llm/workers every ~10s; without this
    # every poll does an open+flock+read of workers.json, which BLOCKS on a
    # degraded mount and stalls the API. Reads serve from cache within the TTL;
    # writes always go to disk and refresh the cache, so liveness stays correct.
    _READ_TTL = 3.0

    def __init__(self, path: Optional[str] = None) -> None:
        self._path = path or _default_workers_path()
        self._lock = threading.RLock()
        self._cache: Optional[Dict[str, Dict[str, Any]]] = None
        self._cache_at = 0.0
        self._ensure_parent()

    # -- persistence (disk-authoritative) ----------------------------------
    def _ensure_parent(self) -> None:
        parent = os.path.dirname(self._path)
        if parent:
            try:
                os.makedirs(parent, exist_ok=True)
            except OSError:
                pass

    def _read_unlocked(self, fh=None) -> Dict[str, Dict[str, Any]]:
        """Parse the workers map from an open fh, or from disk if none given.

        A non-empty file that fails to parse is treated as CORRUPTION, not as an
        empty registry: we log and re-raise rather than return {}. Otherwise a
        torn write (this unit restarts often) would be silently 'healed' into an
        empty fleet, and the next write would persist that empty set — wiping
        every worker. Absent/empty files still return {} (normal cold start).
        """
        try:
            if fh is not None:
                fh.seek(0)
                raw = fh.read()
            elif os.path.exists(self._path):
                with open(self._path, "r", encoding="utf-8") as f:
                    raw = f.read()
            else:
                return {}
        except OSError:
            return {}
        if not raw.strip():
            return {}
        try:
            data = json.loads(raw)
            if not isinstance(data, dict):
                raise ValueError("workers registry root is not a JSON object")
            return {w["id"]: w for w in data.get("workers", []) if w.get("id")}
        except (ValueError, KeyError) as exc:
            import logging as _logging
            _logging.getLogger(__name__).error(
                "workers registry %s is unparseable (%d bytes) — refusing to treat "
                "as empty; leaving the file intact for recovery (%s)",
                self._path, len(raw), exc,
            )
            raise

    def _write_unlocked(self, fh, workers: Dict[str, Dict[str, Any]]) -> None:
        """Overwrite the open, locked fh with the workers map."""
        payload = json.dumps({"workers": list(workers.values())}, indent=2)
        fh.seek(0)
        fh.truncate()
        fh.write(payload)
        fh.flush()
        try:
            os.fsync(fh.fileno())
        except OSError:
            pass

    def _load(self) -> Dict[str, Dict[str, Any]]:
        """Read-only snapshot of the registry, cached for a few seconds.

        Polls (list/get/pick) hit this; the cache keeps a hung/slow mount from
        blocking every request. Writes refresh the cache, so freshly-registered
        or reassigned workers are visible immediately to the writing process.
        """
        now = time.time()
        with self._lock:
            if self._cache is not None and (now - self._cache_at) < self._READ_TTL:
                return self._cache
            try:
                data = self._read_unlocked()
            except (ValueError, KeyError):
                # Corrupt on-disk file: don't crash polls — serve the last good
                # snapshot if we have one (the error is already logged).
                if self._cache is not None:
                    return self._cache
                raise
            self._cache = data
            self._cache_at = now
            return data

    @contextmanager
    def _transaction(self):
        """Yield the on-disk workers map under an exclusive cross-process lock.

        Reload -> mutate (caller) -> persist. The yielded dict is written back
        when the block exits without raising. Falls back to a plain in-process
        critical section when ``fcntl`` is unavailable.
        """
        with self._lock:
            self._ensure_parent()
            # Open r+ (create if missing) so we hold one fd for lock+read+write.
            fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o644)
            fh = os.fdopen(fd, "r+", encoding="utf-8")
            try:
                if fcntl is not None:
                    fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
                workers = self._read_unlocked(fh)
                yield workers
                self._write_unlocked(fh, workers)
                # Refresh the read-cache so this process sees its own write
                # immediately (and other processes within the TTL).
                self._cache = workers
                self._cache_at = time.time()
            finally:
                try:
                    if fcntl is not None:
                        fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
                finally:
                    fh.close()

    # -- registration / lifecycle ------------------------------------------
    def register(
        self,
        *,
        name: str,
        url: str,
        gpus: Optional[List[Dict[str, Any]]] = None,
        role: str = "worker",
        models: Optional[List[str]] = None,
        worker_id: Optional[str] = None,
        pkg_version: Optional[str] = None,
        rpc_endpoint: Optional[str] = None,
        free_ram: Optional[int] = None,
        ram_total: Optional[int] = None,
        engine: Optional[Dict[str, Any]] = None,
        pool: Optional[str] = None,
        caps: Optional[Dict[str, Any]] = None,
        env: Optional[Dict[str, Any]] = None,
        serving_limits: Optional[Dict[str, Any]] = None,
        slot_capable: Optional[bool] = None,
        slot_incapable_reason: Optional[str] = None,
        task_capabilities: Optional[Dict[str, bool]] = None,
    ) -> Dict[str, Any]:
        """Add a worker (or re-register an existing one by id/url).

        Re-registration is keyed first on the supplied ``worker_id``, then on
        ``url`` — so an agent that restarts and advertises the same URL keeps
        its assignments instead of creating a duplicate row.
        """
        url = (url or "").rstrip("/")
        with self._transaction() as workers:
            existing = None
            if worker_id and worker_id in workers:
                existing = workers[worker_id]
            else:
                for w in workers.values():
                    if w.get("url") == url:
                        existing = w
                        break

            if existing is not None:
                # Grandfather pre-feature rows to approved; never silently revive a
                # blocked worker (the route refuses it, but don't let a re-register
                # flip it back to serving).
                existing.setdefault("admission", "approved")
                existing.update(
                    name=name or existing.get("name"),
                    url=url or existing.get("url"),
                    gpus=gpus if gpus is not None else existing.get("gpus", []),
                    role=role or existing.get("role", "worker"),
                    last_seen=_now(),
                )
                if models is not None:
                    existing["models"] = sorted(set(models))
                if pkg_version is not None:
                    existing["pkg_version"] = pkg_version
                if rpc_endpoint is not None:
                    existing["rpc_endpoint"] = rpc_endpoint
                if free_ram is not None:
                    existing["free_ram"] = free_ram
                if ram_total is not None:
                    existing["ram_total"] = ram_total
                if engine is not None:
                    existing["engine"] = engine
                if caps is not None:
                    existing["caps"] = caps
                if env is not None:
                    existing["env"] = env
                # Concurrency-hardening capability (2026-07-11). Stored verbatim;
                # _public_view spreads them onto /llm/workers rows. A None from an
                # older agent leaves the field untouched (legacy-safe).
                if serving_limits is not None:
                    existing["serving_limits"] = serving_limits
                if slot_capable is not None:
                    existing["slot_capable"] = slot_capable
                    existing["slot_incapable_reason"] = slot_incapable_reason
                # Per-task capability honesty (2026-07-11) — stored verbatim, same
                # legacy-safe idiom: a None from an older agent leaves any prior
                # value untouched. Central's workers_for_model gate reads it.
                if task_capabilities is not None:
                    existing["task_capabilities"] = task_capabilities
                # Only a NON-EMPTY declared pool re-asserts on re-register, so an
                # operator-set pool isn't wiped by a worker that doesn't declare
                # WORKER_POOL (which sends ""). Declaring workers still win.
                if pool and pool.strip():
                    existing["pool"] = pool.strip()
                # 4b organic backfill: every re-register refreshes the
                # assignment memory, so designations that predate the memory
                # feature become durable without an explicit assign.
                if existing.get("models"):
                    _remember_assignments(existing)
                # Durable hardware facts: advance last-known GPU/RAM totals from
                # this reading (never wiped by a transient empty probe) so a
                # re-register that momentarily lacks totals still has them.
                _remember_hw_totals(existing)
                return _public_view(existing)

            wid = worker_id or uuid.uuid4().hex
            # 4b: a fresh row for a KNOWN worker id (its old row was swept /
            # the registry was lost) restores the operator's designations from
            # the assignment memory — designations are worker-lifetime.
            remembered = _load_assign_memory().get(wid) if worker_id else None
            restored_models: List[str] = []
            restored_spill: Dict[str, Any] = {}
            restored_gpu_known = None
            restored_ram_known = None
            if remembered:
                restored_models = list(remembered.get("models") or [])
                restored_spill = dict(remembered.get("spill_by_model") or {})
                restored_gpu_known = remembered.get(_GPU_TOTAL_DURABLE_KEY)
                restored_ram_known = remembered.get(_RAM_TOTAL_DURABLE_KEY)
                if restored_models:
                    logger.warning(
                        "register: restoring %d remembered designation(s) for "
                        "returning worker %s (%s): %s", len(restored_models),
                        name or wid, wid, restored_models)
            worker = {
                "id": wid,
                "name": name or wid,
                "url": url,
                "role": role or "worker",
                "gpus": gpus or [],
                "models": sorted(set(models or []) | set(restored_models)),
                "spill_by_model": restored_spill,
                "pkg_version": pkg_version,
                "rpc_endpoint": rpc_endpoint,
                "free_ram": free_ram,
                "ram_total": ram_total,
                "engine": engine,
                "caps": caps,
                # Concurrency-hardening capability (2026-07-11): safe in-process
                # concurrency + whether the box can seat a native crash-isolated
                # slot. None on a pre-feature agent -> central assumes cap 1 and
                # shows no slot badge. See remote._advertised_cap / _public_view.
                "serving_limits": serving_limits,
                "slot_capable": slot_capable,
                "slot_incapable_reason": slot_incapable_reason,
                # Per-task capability honesty (2026-07-11): {task: bool} of the /ml
                # tasks this box can actually run (find_spec probe + a real whisper
                # import). None on a pre-feature agent -> central assumes capable so
                # a legacy fleet routes unchanged. See workers_for_model / _task_capable.
                "task_capabilities": task_capabilities,
                # Runtime-env capability: {"tier": "stable"|"edge"|..., versions}.
                # Read from the worker's own venv, so it's truth not config claim.
                "env": env,
                # Dedicated-pool label. "" = general pool. A pooled worker serves
                # ONLY requests tagged for its pool (reserved capacity); general
                # traffic never lands on it. See workers_for_model.
                "pool": (pool or "").strip(),
                # New workers land pending: they appear in the console but do not
                # serve traffic until an operator admits them (approval-required).
                "admission": "pending",
                "created_at": _now(),
                "last_seen": _now(),
            }
            # Inherit durable hardware facts remembered for this id (a returning
            # worker whose live row was lost keeps its totals immediately), then
            # advance from THIS register's reading if it carried them.
            if restored_gpu_known:
                worker[_GPU_TOTAL_DURABLE_KEY] = restored_gpu_known
            if restored_ram_known:
                worker[_RAM_TOTAL_DURABLE_KEY] = restored_ram_known
            _remember_hw_totals(worker)
            workers[wid] = worker
            return _public_view(worker)

    def heartbeat(
        self,
        worker_id: str,
        *,
        gpus: Optional[List[Dict[str, Any]]] = None,
        loaded_models: Optional[List[str]] = None,
        loading: Optional[List[str]] = None,
        models_local: Optional[List[str]] = None,
        provisioning: Optional[List[str]] = None,
        provision_progress: Optional[Dict[str, Any]] = None,
        spill: Optional[Dict[str, Any]] = None,
        url: Optional[str] = None,
        pkg_version: Optional[str] = None,
        role: Optional[str] = None,
        rpc_endpoint: Optional[str] = None,
        free_ram: Optional[int] = None,
        ram_total: Optional[int] = None,
        free_ram_raw: Optional[int] = None,
        ram_worker_bytes: Optional[int] = None,
        ram_external_bytes: Optional[int] = None,
        vram_attributed_bytes: Optional[int] = None,
        vram_unattributed_bytes: Optional[int] = None,
        disk: Optional[Dict[str, Any]] = None,
        engine: Optional[Dict[str, Any]] = None,
        pool: Optional[str] = None,
        caps: Optional[Dict[str, Any]] = None,
        env: Optional[Dict[str, Any]] = None,
        config: Optional[Dict[str, Any]] = None,
        comfy: Optional[Dict[str, Any]] = None,
        loaded_detail: Optional[Dict[str, Any]] = None,
        slots: Optional[List[Dict[str, Any]]] = None,
        allocations: Optional[List[Dict[str, Any]]] = None,
        pid_registry: Optional[Dict[str, Any]] = None,
        storage: Optional[Dict[str, Any]] = None,
        install: Optional[Dict[str, Any]] = None,
        serving_limits: Optional[Dict[str, Any]] = None,
        slot_capable: Optional[bool] = None,
        slot_incapable_reason: Optional[str] = None,
        task_capabilities: Optional[Dict[str, bool]] = None,
        vram_evictions: Optional[Dict[str, Any]] = None,
    ) -> Optional[Dict[str, Any]]:
        """Mark a worker alive and refresh its live GPU / loaded-model stats."""
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            worker["last_seen"] = _now()
            if url:
                worker["url"] = url.rstrip("/")
            if gpus is not None:
                worker["gpus"] = gpus
            if loaded_models is not None:
                # TRUTHFUL residency: the agent's loaded_models only covers its
                # in-process dispatch cache — a model resident in a SLOT child
                # (llama_cpp.server / llama-server it spawned or adopted) is
                # invisible to it, so the console showed "Serving, nothing
                # loaded" while GBs sat on the GPU, and the warm reconcile
                # would re-probe already-warm models. Union in what the slots
                # report about themselves.
                merged = list(loaded_models)
                for s in (slots if slots is not None
                          else worker.get("slots") or []):
                    mk = (s or {}).get("model_key")
                    if mk and s.get("healthy") and mk not in merged:
                        merged.append(mk)
                worker["loaded_models"] = merged
            if loading is not None:
                worker["loading"] = loading   # weights load in flight ("heating")
            if models_local is not None:
                worker["models_local"] = models_local   # disk-truth (UTIL-08)
            if provisioning is not None:
                worker["provisioning"] = provisioning
            if provision_progress is not None:
                # PROGRESS CLOCK (orphan-job lesson: age on PROGRESS, not on any
                # write). The worker re-sends its whole progress map every
                # heartbeat, so the ARRIVAL of this field proves only that the
                # agent is alive — not that any pull is moving. A dead pull's
                # last snapshot keeps replaying verbatim (op sat frozen at
                # frac=0.0722 for 2h+). Carry a central ``progressed_at`` per
                # model, bumped ONLY when done_bytes actually ADVANCES, so
                # _live_provisioning can tell a live 7% from a corpse at 7%.
                prev = worker.get("provision_progress") or {}
                stamped: Dict[str, Any] = {}
                for mk, entry in (provision_progress or {}).items():
                    if not isinstance(entry, dict):
                        stamped[mk] = entry
                        continue
                    entry = dict(entry)
                    old = prev.get(mk) if isinstance(prev, dict) else None
                    old = old if isinstance(old, dict) else {}

                    def _done(e):
                        try:
                            return float(e.get("done_bytes") or 0)
                        except (TypeError, ValueError):
                            return 0.0

                    advanced = _done(entry) > _done(old)
                    carried = old.get("progressed_at")
                    if advanced or carried is None:
                        # First sighting counts as progress: a pull that just
                        # started has moved no bytes yet and must not be born
                        # already-stale.
                        entry["progressed_at"] = _now()
                    else:
                        entry["progressed_at"] = carried
                    stamped[mk] = entry
                worker["provision_progress"] = stamped
            if spill is not None:
                worker["spill"] = spill
            if pkg_version is not None:
                worker["pkg_version"] = pkg_version
            if role is not None:
                worker["role"] = role
            if rpc_endpoint is not None:
                worker["rpc_endpoint"] = rpc_endpoint
            if free_ram is not None:
                worker["free_ram"] = free_ram
            if ram_total is not None:
                worker["ram_total"] = ram_total
            # Honest budget-bar inputs (t13/t14) — stored verbatim; _ram_summary/
            # _vram_summary compute the spec bar from them. Absent -> the summary
            # flags bar_semantics="legacy" and shows today's numbers.
            if free_ram_raw is not None:
                worker["free_ram_raw"] = free_ram_raw
            if ram_worker_bytes is not None:
                worker["ram_worker_bytes"] = ram_worker_bytes
            if ram_external_bytes is not None:
                worker["ram_external_bytes"] = ram_external_bytes
            if vram_attributed_bytes is not None:
                worker["vram_attributed_bytes"] = vram_attributed_bytes
            if vram_unattributed_bytes is not None:
                worker["vram_unattributed_bytes"] = vram_unattributed_bytes
            if disk is not None:
                worker["disk"] = disk   # model-root volume free/total (preflight)
            if engine is not None:
                worker["engine"] = engine
            if env is not None:
                worker["env"] = env
            if config is not None:
                worker["config"] = config   # effective serving-config + source
            if comfy is not None:
                worker["comfy"] = comfy     # ComfyUI presence (slice A)
            if loaded_detail is not None:
                worker["loaded_detail"] = loaded_detail
            if slots is not None:
                worker["slots"] = slots
            if allocations is not None:
                # Unified engine-agnostic allocation view (slot-seated + in-RAM
                # residents). Stored verbatim; _public_view spreads it through.
                worker["allocations"] = allocations
            if pid_registry is not None:
                # Precision model->PID log (2026-07-14): per-model pid/host_mode/
                # vram + unattributed foreign squatters. Stored verbatim;
                # _public_view spreads it so the console renders it per worker.
                worker["pid_registry"] = pid_registry
            if vram_evictions is not None:
                # VRAM eviction churn (slice 10): stored verbatim so the console
                # can surface GPU evict-to-fit churn beside the disk reaps.
                worker["vram_evictions"] = vram_evictions
            if storage is not None:
                # Worker-reported local-storage survey (per-model on-disk bytes +
                # protection flags + cache_used_bytes). Stored verbatim; the
                # over_budget flag + LRU eviction proposal are derived centrally
                # in _public_view via storage_proposal (which overlays the fields
                # the worker can't know: last_picked + the budget).
                worker["storage"] = storage
            if install is not None:
                # Install-shape (uniform-install drift detection): {unit,
                # via_systemd, venv, python, canonical}. Stored verbatim and
                # spread through _public_view (via **worker); the console badges
                # a non-canonical install off it.
                worker["install"] = install
            if caps is not None:
                worker["caps"] = caps
                # Worker-side config is the hard ceiling: if its caps tightened
                # below an operator limit, re-clamp the stored limit now.
                if worker.get("limits"):
                    worker["limits"] = _clamp_limits(worker["limits"], caps)
            # Concurrency-hardening capability (2026-07-11) — refreshed every beat
            # so the console/gate see live truth (a worker that installs the engine
            # binary flips slot_capable within one heartbeat). Legacy-safe: a None
            # from an older agent leaves the fields absent (central assumes cap 1).
            if serving_limits is not None:
                worker["serving_limits"] = serving_limits
            if slot_capable is not None:
                worker["slot_capable"] = slot_capable
                worker["slot_incapable_reason"] = slot_incapable_reason
            # Per-task capability honesty (2026-07-11) — refreshed every beat so an
            # /ops/pip that adds a missing dep flips the task True within one beat.
            # Legacy-safe: a None from an older agent leaves the field absent.
            if task_capabilities is not None:
                worker["task_capabilities"] = task_capabilities
            if pool and pool.strip():   # non-empty only — see register() note
                worker["pool"] = pool.strip()
            # Advance the durable hardware facts from this beat's totals (a beat
            # carries gpus[] w/ memory_total + ram_total). Advance-only, so a
            # beat that transiently omits them keeps the last-known fact.
            _remember_hw_totals(worker)
            return _public_view(worker)

    def remove(self, worker_id: str) -> bool:
        with self._transaction() as workers:
            return workers.pop(worker_id, None) is not None

    # Operator-settable per-worker resource limits. Central may only TIGHTEN:
    # a worker's own configured caps (reported in its heartbeat as ``caps``)
    # are the hard ceiling, so every write is clamped against them.
    #
    # ``disk_cache_gib`` is the OPTIONAL explicit per-worker storage cap (GiB):
    # when set it drives the over-budget flag off cache_used vs the cap (WINS over
    # the free-disk reserve default in storage_proposal), and — unlike the others
    # — has no worker-reported cap, so _clamp_limits passes it through unclamped.
    # More robust than the free-disk reserve against non-model disk pressure.
    _LIMIT_KEYS = ("ram_max_gib", "gpu_mem_gib", "threads", "disk_cache_gib")

    def set_limits(self, worker_id: str,
                   limits: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
        """Set (or clear, with None/{}) central's resource limits for a worker.

        Values are clamped to the worker's self-reported caps — the box's own
        config always wins. Unknown keys are dropped; non-numeric values raise.
        """
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            if not limits:
                worker.pop("limits", None)
                return _public_view(worker)
            clean: Dict[str, Any] = {}
            for k in self._LIMIT_KEYS:
                if k not in limits or limits[k] in (None, ""):
                    continue
                try:
                    clean[k] = float(limits[k]) if k != "threads" else int(limits[k])
                except (TypeError, ValueError):
                    raise ValueError(f"limit {k} must be numeric")
            worker["limits"] = _clamp_limits(clean, worker.get("caps") or {})
            return _public_view(worker)

    def set_pool(self, worker_id: str, pool: str) -> Optional[Dict[str, Any]]:
        """Operator override of a worker's dedicated pool ("" clears). Survives
        heartbeats from workers that don't declare WORKER_POOL (they send "",
        which the register/heartbeat guards ignore)."""
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            worker["pool"] = (pool or "").strip()
            return _public_view(worker)

    def set_auto_reap(self, worker_id: str, enabled: bool) -> Optional[Dict[str, Any]]:
        """Opt this worker into AUTO-REAP (slice 8, Part B; operator ask
        2026-07-17 "there needs to be a way to auto approve this").

        DEFAULT FALSE — the hand-approve flow stays the default posture (defaults
        are promises: central does not self-approve deletions unless the operator
        turned it on for THIS worker). When true, central's heartbeat ingest fires
        EXACTLY the operator reap-approve flow (recompute → intersect → audit →
        guarded relay) once per cooldown when the worker is over budget with a
        non-empty proposal. Persisted on the worker record beside limits/pool, so
        it survives heartbeats and is set through the same operator-gated route
        family. Never widens the blast radius: an auto-fire reclaims at most the
        proposal's need, exactly like a hand-approved one."""
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            worker["auto_reap"] = bool(enabled)
            return _public_view(worker)

    def record_auto_reap(self, worker_id: str, when: float) -> None:
        """Stamp last_auto_reap_at (epoch) so the per-worker cooldown can gate the
        next auto-fire and the console can show when auto-reap last acted. Plain
        stamp — the eviction itself goes through the guarded relay, not here."""
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is not None:
                worker["last_auto_reap_at"] = float(when)

    _ADMISSION_STATES = ("pending", "approved", "blocked")

    def set_admission(self, worker_id: str, state: str) -> Optional[Dict[str, Any]]:
        """Set a worker's admission gate (pending/approved/blocked).

        ``approved`` lets it serve; ``pending`` parks it (visible, idle);
        ``blocked`` evicts it — the register/heartbeat routes refuse a blocked
        worker so its agent stops instead of respawning. Persisted, so the gate
        survives the worker's next heartbeat (unlike ``remove``, which a heartbeat
        would undo).
        """
        if state not in self._ADMISSION_STATES:
            raise ValueError(f"admission must be one of {self._ADMISSION_STATES}")
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            worker["admission"] = state
            return _public_view(worker)

    # -- model assignment ---------------------------------------------------
    def assign_model(
        self,
        worker_id: str,
        model_key: str,
        spill: Optional[Dict[str, Any]] = None,
    ) -> Optional[Dict[str, Any]]:
        """Assign a model to a worker, with optional per-assignment spill config.

        ``spill`` is an opaque dict of GPU/CPU knobs (e.g. n_gpu_layers,
        gpu_mem_gib, cpu_mem_gib) the worker applies when it loads the model.
        Omitted / None means "use the worker's autofit default."
        """
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            models = set(worker.get("models", []))
            models.add(model_key)
            worker["models"] = sorted(models)
            if spill is not None:
                by_model = worker.setdefault("spill_by_model", {})
                # An empty dict clears any override back to autofit.
                if spill:
                    by_model[model_key] = spill
                else:
                    by_model.pop(model_key, None)
            _remember_assignments(worker)   # 4b: designations survive row loss
            return _public_view(worker)

    def unassign_model(self, worker_id: str, model_key: str) -> Optional[Dict[str, Any]]:
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            worker["models"] = sorted(set(worker.get("models", [])) - {model_key})
            worker.get("spill_by_model", {}).pop(model_key, None)
            # Hygiene: drop the per-model LRU stamp too, so the model_last_picked
            # map doesn't grow unbounded with unassigned models. Harmless for the
            # eviction proposal — a missing entry defaults to 0 (coldest), which
            # is correct for a now-unassigned leftover.
            worker.get("model_last_picked", {}).pop(model_key, None)
            _remember_assignments(worker)   # 4b: an explicit unassign IS forgotten
            return _public_view(worker)

    # -- placement grants (Phase 1 item 2) -----------------------------------
    # A GRANT is a SYSTEM-authored designation — born from a future
    # capacity-aware placement decision, NOT an operator assign/pin. Stored
    # separately from ``worker["models"]`` so it can never masquerade as
    # operator intent: assign/unassign, storage protection's "assigned" branch,
    # and the assignment-memory snapshot all stay blind to it. A grant is
    # freely LRU-evictable (see storage_proposal) and dies with the live
    # worker row — it is deliberately NOT written to the assign-memory file
    # (_remember_assignments), so a row-loss restore never resurrects it. This
    # method only touches ``worker["grants"]``; ``worker["models"]`` is
    # untouched (orthogonal to assign_model/unassign_model).
    def grant_model(self, worker_id: str, model_key: str,
                    job_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            grants = worker.setdefault("grants", {})
            grants[model_key] = {
                "ts": _now(),
                "job_id": job_id,
                "origin": "system",
            }
            return _public_view(worker)

    def ungrant_model(self, worker_id: str, model_key: str) -> Optional[Dict[str, Any]]:
        """Remove one grant. Idempotent — a missing key is a no-op, not an error."""
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            worker.get("grants", {}).pop(model_key, None)
            return _public_view(worker)

    def spill_for(self, worker_id: str, model_key: str) -> Dict[str, Any]:
        """Per-assignment spill override for (worker, model), or {} for
        max-gpu (autofit). THE version-gated emission seam (k37): a spill
        carrying the NEW allocation-mode keys (alloc_mode/leniency_pct/
        priority_device — max-ram/explicit) is only emitted to a worker whose
        pkg_version honors them; an older worker gets {} (max-gpu autofit) for
        the request and the downgrade is logged honestly — a selected mode
        must never be a silent dead knob. The PERSISTED contract is untouched
        (it applies the moment the worker updates)."""
        worker = self._load().get(worker_id)
        if worker is None:
            return {}
        spill = dict(worker.get("spill_by_model", {}).get(model_key, {}))
        # CAPABILITY-AWARE BLANK DEFAULT (operator ruling 2026-07-24): when
        # NOTHING placement-affecting is persisted for this (worker, model), the
        # blank default is derived by FEASIBILITY instead of the flat max-gpu.
        # A transformers model far too big for this box's GPU but that fits RAM
        # defaults to ram-only ON THIS WORKER, so its blank default SERVES
        # (defaults-are-promises) instead of a doomed all-GPU attempt. GGUF and
        # any fitting/unresolvable case keep max-gpu ({} — unchanged). This only
        # ever touches the BLANK case; an explicit persisted contract is left
        # exactly as-is. The emitted encoding is the LEGACY n_gpu_layers key —
        # no version gate needed (every worker version honors it).
        if not (set(spill) & _PLACEMENT_SPILL_KEYS):
            try:
                mode = derived_default_mode(worker, model_key)
            except Exception:  # noqa: BLE001 — never break the relay over a derive
                mode = "max-gpu"
            if mode == "ram-only":
                logger.info("blank default for %s on %s derived to ram-only "
                            "(feasibility: oversized for GPU, fits RAM)",
                            model_key, worker.get("name") or worker_id)
                return {"n_gpu_layers": "off"}
            return {}                            # max-gpu: today's blank ({})
        try:
            from ......managers.alloc_modes import gate_spill_for_worker
            gated, note = gate_spill_for_worker(
                spill, worker.get("pkg_version"),
                worker.get("name") or worker_id)
            if note:
                logger.warning("alloc-mode downgrade for %s on %s: %s",
                               model_key, worker.get("name") or worker_id, note)
            return gated
        except Exception:  # noqa: BLE001 — the gate must never break relaying
            return spill

    def set_load_report(self, worker_id: str, model_key: str,
                        report: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
        """Record the outcome of a warm/probe attempt for (worker, model).

        ``report`` is the worker's /probe response ({ok, fit, vram_used, error, …})
        plus a ``ts`` stamp, or a synthesized {ok: False, error} when the probe
        HTTP call itself failed. ``None`` clears the entry (e.g. on unassign).
        Stored under ``load_reports[model_key]`` on the worker record so the
        console can say WHY a model stayed cold instead of showing a silent
        no-op activate."""
        with self._transaction() as workers:
            worker = workers.get(worker_id)
            if worker is None:
                return None
            reports = worker.setdefault("load_reports", {})
            if report is None:
                reports.pop(model_key, None)
            else:
                reports[model_key] = report
            return _public_view(worker)

    # -- queries ------------------------------------------------------------
    def get(self, worker_id: str) -> Optional[Dict[str, Any]]:
        worker = self._load().get(worker_id)
        return _public_view(worker) if worker else None

    def all(self) -> List[Dict[str, Any]]:
        return [_public_view(w) for w in self._load().values()]

    def storage_view(self, worker_id: str) -> Optional[Dict[str, Any]]:
        """The derived storage view + LRU eviction proposal for one worker,
        computed from its RAW record (NOT the _public_view output, whose
        ``storage`` key is already the derived shape). This is the /reap-approve
        route's second-guard recompute: it must read the raw worker-reported
        ``storage`` survey to re-derive the CURRENT proposal at approve time.
        ``None`` if the worker is unknown."""
        worker = self._load().get(worker_id)
        return storage_proposal(worker) if worker else None

    def workers_for_model(self, model_key: str, *, online_only: bool = True,
                          pool: Optional[str] = None,
                          task: Optional[str] = None,
                          require_comfy_id_lock: bool = False) -> List[Dict[str, Any]]:
        # Operator BLOCK gate (central pool primitive): a blocked model is
        # removed from the pool entirely — no worker is EVER a candidate for it,
        # regardless of assignment/pin. Block outranks pin (pin is routing
        # persistence; block is an operator override), so a pinned+blocked model
        # simply yields no candidates here while its designation row stays
        # recorded (inert). One honest log line, same say-why spirit as the gates
        # below; returns before any per-worker work.
        if _model_blocked(model_key):
            import logging as _logging
            _logging.getLogger(__name__).info(
                "model %s is BLOCKED from the serving pool by the operator — "
                "no worker is a routing candidate (unblock to route again)",
                model_key)
            return []
        wanted = _match_keys(model_key)
        want_pool = (pool or "").strip()
        need_tier = env_tier_for_model(model_key)
        # WILDCARD map, read ONCE per call (not per worker-loop iteration) —
        # the per-worker lookup below is then a dict hit. Guarded inside
        # _wildcard_map: a store miss degrades to "nobody is a wildcard".
        wildcards = _wildcard_map()
        tier_skipped = 0
        task_skipped = 0
        id_lock_skipped = 0
        engine_skipped = 0
        wildcard_engine_skipped = 0
        out = []
        for w in self.all():
            # Only admitted workers serve. Pending (awaiting operator approval) and
            # blocked workers are never picked for inference, even if assigned.
            if w.get("admission") != "approved":
                continue
            # Capability guard: skip a worker that reports no inference engine —
            # it would accept the dispatch and fail, wasting a hop before the
            # local fallback. (Workers not reporting engine status are kept.)
            if _engine_unusable(w):
                # Say-why parity with the tier/task/id_lock gates below: count a
                # worker only when it is otherwise ASSIGNED to this model, so an
                # empty result's log names the real cause (a DESIGNATED worker
                # whose engine can't serve — the "assigned+pinned but 500s"
                # mystery) instead of every engine-broken box on the fleet.
                _serveable = (list(w.get("models", [])) + list(w.get("loaded_models", []))
                              + list(w.get("grants", {}).keys()))
                if _serveable_match(model_key, wanted, _serveable):
                    engine_skipped += 1
                elif wildcards.get(w.get("id") or ""):
                    # A WILDCARD CATCH lost to the engine gate. Counted apart
                    # from the designated skips so the say-why log below stays
                    # truthful ("assigned" vs "wildcard") without new machinery.
                    wildcard_engine_skipped += 1
                continue
            # Dedicated-pool reservation: a request for pool P uses ONLY pool-P
            # workers; a general request (no pool) uses ONLY un-pooled workers.
            # So dedicated capacity is reserved for its app and never consumed by
            # general traffic — and a pool request that finds no pool worker
            # falls back to local (caller's None handling), not to the shared pool.
            if (w.get("pool") or "").strip() != want_pool:
                continue
            # Candidates = models this worker is ASSIGNED **or currently reports
            # LOADED**. A worker holding the model warm (loaded via probe, or
            # left resident after an unassign) is the best possible server —
            # ignoring it sent the request to a cold local fallback while a
            # GPU sat there with the weights already up. Loaded-ness is
            # heartbeat-fresh; if it evicts between beats the relay fails
            # pre-token and the caller falls back as always.
            # Grants (SYSTEM-authored placement, Phase 1 item 2) are serveable
            # exactly like an operator assignment or a live-loaded model —
            # once a granted model is actually held by the worker it must
            # route, or the grant is pointless. Grants confer NO eviction
            # protection (see storage_proposal) — this is purely "can serve",
            # not "may not be reclaimed".
            serveable = (list(w.get("models", [])) + list(w.get("loaded_models", []))
                         + list(w.get("grants", {}).keys()))
            # Match on the raw key OR any normalized alias (hub_id vs key vs
            # case vs "~"-qualification), so an assignment made via one form
            # still routes a chat that names the model a slightly different way.
            # _serveable_match also carries the blocked-sibling guard: an
            # alias-only match against a BLOCKED advertised sibling never
            # counts. RESIDENT = DE FACTO DESIGNATION: ``loaded_models`` (and
            # grants) ride in ``serveable``, so a box currently holding the
            # model is ALWAYS a "home" match here — never route-refused —
            # wildcard flag or not.
            home = _serveable_match(model_key, wanted, serveable)
            if not home:
                # WILDCARD PLACEMENT (operator doctrine 2026-07-23):
                # designations are a HARD routing scope — an unmatched worker
                # is out UNLESS it explicitly opted in as a wildcard ("a worker
                # can be designated to take all comers ... or it can not be
                # selected as a wildcard and adhere only to its own allocated
                # models"). A wildcard catch relaxes ONLY this
                # designation-membership gate: every hard gate around it still
                # applies — admission/engine/pool above, liveness/env-tier/
                # task-capability/id-lock below, and the requested key's BLOCK
                # gate already returned [] before the loop. Default False for
                # every worker (absent key = not a wildcard), so a fleet with
                # no flags set routes exactly as before this feature existed.
                if not wildcards.get(w.get("id") or ""):
                    continue
            if online_only and w["status"] != "online":
                continue
            # Runtime-env tier gate: the model runs ONLY on a worker whose venv
            # tier matches (strict both ways — an edge env can regress stable
            # models just as a stable env can't load edge architectures). Both
            # sides default to "stable", so an unmapped model on an unreporting
            # fleet routes exactly as before this gate existed.
            if _worker_env_tier(w) != need_tier:
                tier_skipped += 1
                continue
            # Per-task capability gate (2026-07-11): skip a worker that
            # AFFIRMATIVELY advertises it can't run this task (a canonical venv
            # missing an optional ML dep — sentence-transformers / whisper /
            # keybert). Legacy/unknown = capable, so a pre-feature fleet is
            # untouched; a None task never gates. Same say-why idiom as the tier
            # gate below.
            if not _task_capable(w, task):
                task_skipped += 1
                continue
            # ID-LOCK routing gate (identity-locked STILLs): an id_lock image
            # request must land on a box whose ComfyUI PROVABLY has the IPAdapter
            # nodes (comfy.id_lock). Affirmative-only — never route id_lock to a
            # comfy-less / nodeless worker where it would fail at request time (or
            # worse, tempt a silent non-locked fallback). Off (False) for every
            # other request, so ordinary routing is untouched.
            if require_comfy_id_lock and not _comfy_id_lock_capable(w):
                id_lock_skipped += 1
                continue
            if not home:
                # Transient RESPONSE-COPY marker: ``w`` is a _public_view copy
                # (self.all() rebuilds it from the store on every read), so this
                # can never leak into the persisted record — same transient
                # semantics as the derived status/storage fields. Ranking sorts
                # home matches ABOVE wildcard catches (pick_for_model /
                # candidates_for_model), which IS the overflow mechanism; and
                # say-why readers can tell a candidate is here by wildcard, not
                # designation.
                w["_wildcard_catch"] = True
            out.append(w)
        if not out and engine_skipped:
            # The model HAS designated servers — every one was excluded because it
            # AFFIRMATIVELY reports its inference engine is unusable (llama-cpp not
            # loadable AND no native llama-server binary; engine.installed=False).
            # Name the cause so the operator repairs the box (`hugpy install-engine`
            # / reinstall llama-cpp-python) instead of seeing only the downstream
            # "no worker available / local serving disabled" 500 with no reason.
            import logging as _logging
            _logging.getLogger(__name__).warning(
                "model %s: %d assigned worker(s) skipped — inference engine "
                "unusable (llama-cpp not loadable AND no native llama-server "
                "binary). Repair the engine on those boxes or assign the model to "
                "a healthy worker.", model_key, engine_skipped)
        if not out and wildcard_engine_skipped:
            # WILDCARD catches lost to the engine gate — kept apart from the
            # designated-worker warning above so "assigned" never overcounts.
            # Matters when a model's ONLY possible servers were wildcard boxes:
            # without this line an empty result would look like "no designation"
            # instead of "the all-comers boxes are engine-broken".
            import logging as _logging
            _logging.getLogger(__name__).warning(
                "model %s: %d wildcard (all-comers) worker(s) skipped — "
                "inference engine unusable", model_key, wildcard_engine_skipped)
        if not out and tier_skipped:
            # The model HAS servers — they were excluded on env tier alone. Say
            # so, or the operator sees only the downstream "no worker / local
            # fallback disabled" error with no cause.
            import logging as _logging
            _logging.getLogger(__name__).warning(
                "model %s requires env tier %r; %d otherwise-eligible worker(s) "
                "skipped (none advertise that tier)",
                model_key, need_tier, tier_skipped)
        if not out and task_skipped:
            # The model HAS servers — they were excluded on task capability alone
            # (they advertise they can't run this task). Name the reason, or the
            # operator sees only the downstream no-worker error with no cause.
            import logging as _logging
            _logging.getLogger(__name__).warning(
                "model %s task %r: %d otherwise-eligible worker(s) skipped "
                "(task unavailable — missing optional ML dependency on those boxes)",
                model_key, task, task_skipped)
        if not out and id_lock_skipped:
            # The model HAS servers — every one was excluded because its ComfyUI
            # lacks the IPAdapter node pack (comfy.id_lock False/absent). Name the
            # cause so the operator installs it (WORKER-SETUP §5b) instead of
            # seeing only the downstream no-worker error.
            import logging as _logging
            _logging.getLogger(__name__).warning(
                "model %s id_lock: %d otherwise-eligible worker(s) skipped — no "
                "box advertises comfy.id_lock (install ComfyUI_IPAdapter_plus + "
                "weights per WORKER-SETUP §5b)", model_key, id_lock_skipped)
        return out

    def pick_for_model(self, model_key: str, pool: Optional[str] = None,
                       task: Optional[str] = None,
                       require_comfy_id_lock: bool = False) -> Optional[Dict[str, Any]]:
        """Choose an online worker to serve ``model_key`` (optionally within a
        dedicated ``pool``, and — when set — one that can run ``task``).

        ``require_comfy_id_lock`` (set for identity-locked STILL requests) further
        restricts to boxes whose ComfyUI advertises the IPAdapter nodes.

        Preference order:
            1. HOME workers (designated / resident / granted) before wildcard
               catches — this ordering IS the overflow mechanism (operator
               doctrine 2026-07-23): a designated model tries its home workers
               first and spills onto wildcard ("take all comers") boxes only
               when no home worker survives the gates; it busts only when
               neither can serve. No separate overflow machinery exists.
            2. workers that already report the model as loaded (warm),
            3. otherwise the least-recently-picked online assignee.

        Returns ``None`` when no online worker (in the requested pool) is assigned
        to the model, which signals the caller to fall back to local execution.
        """
        candidates = self.workers_for_model(
            model_key, online_only=True, pool=pool, task=task,
            require_comfy_id_lock=require_comfy_id_lock)
        if not candidates:
            # Fall back to assigned workers even with a stale heartbeat. Heartbeat
            # (worker->central) can time out when central is briefly slow, while
            # offload (central->worker) still works — so an assigned worker that
            # looks "offline" is often still serviceable. The stream proxy fails
            # fast to local if the worker is genuinely unreachable.
            candidates = self.workers_for_model(
                model_key, online_only=False, pool=pool, task=task,
                require_comfy_id_lock=require_comfy_id_lock)
        if not candidates and (pool or "").strip():
            # PHANTOM-POOL RESCUE: a pool restriction only means something when the
            # pool exists. If NO registered worker carries this pool tag at all
            # (e.g. a client still sending the old default pool="ml" on a fleet
            # that never tagged one), honoring it would silently strand the request
            # on central-local even though a general worker serves the model. That
            # is the exact bug the un-pooled client default fixed — cover stale
            # clients here too. A pool with members but none available keeps the
            # reservation semantics: no crossover, local fallback.
            want_pool = pool.strip()
            pool_exists = any((w.get("pool") or "").strip() == want_pool
                              for w in self.all())
            if not pool_exists:
                import logging as _logging
                _logging.getLogger(__name__).info(
                    "pool %r has no registered workers; treating request "
                    "for %s as general (un-pooled)", want_pool, model_key)
                return self.pick_for_model(
                    model_key, pool=None, task=task,
                    require_comfy_id_lock=require_comfy_id_lock)
        if not candidates:
            return None

        # Version gate (soft): prefer workers running central's required package
        # version, so a chat doesn't land on a worker mid-rollout that's still on
        # old code. Soft — if NONE have converged yet, we still serve from the
        # (stale-but-working) assignees rather than forcing a local-only outage
        # during the ~heartbeat-long update window.
        required = required_pkg_version()
        if required:
            matched = [w for w in candidates if w.get("pkg_version") == required]
            if matched:
                candidates = matched

        # Efficiency-aware ranking (capability already filtered above). Prefer,
        # in order: a HOME match (designated/resident/granted) over a wildcard
        # catch (the overflow-by-ordering mechanism — see the docstring), then
        # a worker that already has the model warm (avoids a multi-GB reload),
        # then a worker whose ⭐ boot star == this model (the ambiguity tie-break:
        # nothing warm → prefer the box that boot-loads it anyway), then one with
        # a usable GPU over CPU-only, then the least-recently-picked (spreads
        # load). Stable id tiebreak so the order never wobbles. (Full
        # need-vs-capacity placement is the allocator's job; this is the
        # lightweight default pick.)
        # Read the star store ONCE per pick (never per candidate), alias-tolerant
        # via _match_keys (same unification Slice A uses): a star recorded under a
        # ~-qualified key still matches a bare-key request and vice versa.
        star_map = _star_map()
        wanted_forms = _match_keys(model_key)

        def _starred(w: Dict[str, Any]) -> bool:
            s = star_map.get(w.get("id"))
            return bool(s) and bool(wanted_forms & _match_keys(str(s)))

        def _rank(w: Dict[str, Any]):
            warm = model_key in (w.get("loaded_models") or [])
            return (1 if w.get("_wildcard_catch") else 0,
                    0 if warm else 1,
                    0 if _starred(w) else 1,
                    0 if _has_usable_gpu(w) else 1,
                    w.get("last_picked", 0),
                    w.get("id", ""))
        candidates.sort(key=_rank)
        chosen = candidates[0]

        # Persist the pick so round-robin survives across processes.
        with self._transaction() as workers:
            stored = workers.get(chosen["id"])
            if stored is not None:
                now = _now()
                stored["last_picked"] = now
                # Per-(worker,model) LRU signal for the storage eviction proposal.
                # ``last_picked`` above is a SINGLE per-WORKER round-robin scalar,
                # stamped on EVERY pick regardless of model — it spreads load, it
                # can't key an LRU-per-model eviction. This map records the last
                # time THIS model was routed to THIS worker (the authoritative
                # "central served (worker, model)" event), so storage_proposal can
                # sort candidates oldest-first; a model never served through
                # central has no entry -> defaults to 0 -> proposed first.
                stored.setdefault("model_last_picked", {})[model_key] = now
                chosen = stored
        return _public_view(chosen)

    def candidates_for_model(self, model_key: str,
                             pool: Optional[str] = None,
                             task: Optional[str] = None) -> List[Dict[str, Any]]:
        """Ranked ONLINE workers that can serve ``model_key`` — the cap-aware
        relay router's alternatives list (concurrency hardening 2026-07-11).

        Same eligibility + ranking as ``pick_for_model`` (home before wildcard
        catch — the relay's fallback walk must also prefer home, so overflow
        stays overflow — then warm, then ⭐ star, then GPU, then
        least-recently-picked), but WITHOUT the ``last_picked`` write: central's
        in-flight gate iterates this to reroute around a worker that is at its
        advertised in-process concurrency cap, and re-stamping every candidate on
        each probe would corrupt the round-robin. Online only — a reroute target
        must be live right now (the stale-heartbeat fallback pick_for_model does
        is for last-resort primary selection, not for spreading concurrent load).
        """
        candidates = self.workers_for_model(model_key, online_only=True, pool=pool, task=task)
        if not candidates:
            return []
        required = required_pkg_version()
        if required:
            matched = [w for w in candidates if w.get("pkg_version") == required]
            if matched:
                candidates = matched

        # Star store read ONCE (never per candidate), alias-tolerant — see
        # pick_for_model's _rank for the rationale (ambiguity tie-break only).
        star_map = _star_map()
        wanted_forms = _match_keys(model_key)

        def _starred(w: Dict[str, Any]) -> bool:
            s = star_map.get(w.get("id"))
            return bool(s) and bool(wanted_forms & _match_keys(str(s)))

        def _rank(w: Dict[str, Any]):
            warm = model_key in (w.get("loaded_models") or [])
            return (1 if w.get("_wildcard_catch") else 0,
                    0 if warm else 1,
                    0 if _starred(w) else 1,
                    0 if _has_usable_gpu(w) else 1,
                    w.get("last_picked", 0),
                    w.get("id", ""))

        return sorted(candidates, key=_rank)


worker_store = WorkerStore()


# Module-level convenience wrappers (mirrors the manifest.py / peers.py style of
# exposing plain functions for routes to import).
def register_worker(**kwargs) -> Dict[str, Any]:
    return worker_store.register(**kwargs)


def heartbeat_worker(worker_id: str, **kwargs) -> Optional[Dict[str, Any]]:
    # kwargs: gpus, loaded_models, spill — all optional, passed straight through.
    return worker_store.heartbeat(worker_id, **kwargs)


def remove_worker(worker_id: str) -> bool:
    return worker_store.remove(worker_id)


def set_worker_admission(worker_id: str, state: str) -> Optional[Dict[str, Any]]:
    return worker_store.set_admission(worker_id, state)


def set_worker_pool(worker_id: str, pool: str) -> Optional[Dict[str, Any]]:
    return worker_store.set_pool(worker_id, pool)


def set_worker_limits(worker_id: str, limits) -> Optional[Dict[str, Any]]:
    return worker_store.set_limits(worker_id, limits)


def set_worker_auto_reap(worker_id: str, enabled: bool) -> Optional[Dict[str, Any]]:
    """Opt a worker into auto-reap (slice 8, Part B). Operator-gated route only."""
    return worker_store.set_auto_reap(worker_id, enabled)


def record_worker_auto_reap(worker_id: str, when: float) -> None:
    """Stamp when an auto-reap last fired (cooldown gate + console)."""
    worker_store.record_auto_reap(worker_id, when)


def enroll_required() -> bool:
    """Whether a valid enrollment token is mandatory to register/heartbeat.

    Default OFF (gradual rollout): tokenless workers may still register, but land
    ``pending`` like everyone else. Flip ``HUGPY_WORKER_ENROLL_REQUIRED`` truthy
    once the fleet is re-enrolled to refuse tokenless / revoked workers outright.
    """
    return os.environ.get("HUGPY_WORKER_ENROLL_REQUIRED", "").strip().lower() in (
        "1", "true", "yes", "on",
    )


def assign_model(worker_id: str, model_key: str,
                 spill: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
    return worker_store.assign_model(worker_id, model_key, spill=spill)


def unassign_model(worker_id: str, model_key: str) -> Optional[Dict[str, Any]]:
    worker_store.set_load_report(worker_id, model_key, None)
    return worker_store.unassign_model(worker_id, model_key)


def grant_model(worker_id: str, model_key: str,
                job_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
    """SYSTEM-authored placement grant (Phase 1 item 2) — see WorkerStore.grant_model."""
    return worker_store.grant_model(worker_id, model_key, job_id=job_id)


def ungrant_model(worker_id: str, model_key: str) -> Optional[Dict[str, Any]]:
    return worker_store.ungrant_model(worker_id, model_key)


def set_load_report(worker_id: str, model_key: str,
                    report: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
    return worker_store.set_load_report(worker_id, model_key, report)


def spill_for(worker_id: str, model_key: str) -> Dict[str, Any]:
    return worker_store.spill_for(worker_id, model_key)


def derived_default_for(worker_id: str, model_key: str) -> Optional[str]:
    """The feasibility-derived BLANK default mode for (worker, model) from the
    RAW worker record, or None if the worker is unknown. Degrades to 'max-gpu'
    on any miss (via derived_default_mode). Read-only — used to SURFACE the
    derived default the UI distinguishes from a stored one."""
    worker = worker_store._load().get(worker_id)
    if worker is None:
        return None
    return derived_default_mode(worker, model_key)


_FEAS_FAILOPEN_SEEN: set = set()


def _warn_feasibility_failopen(worker: Dict[str, Any], model_key: str,
                               size, gpu_total, ram_total) -> None:
    """Self-policing drift signal (operator refinement 2026-07-24): feasibility
    only falls OPEN (all modes selectable) when central couldn't resolve the
    model size or the box totals — which should be a RARE, TRANSIENT state (a
    fresh worker before its first heartbeat; a registry row before enrichment).
    Log it ONCE per (model, worker, missing-set) at WARNING so a transient never
    spams but a PERSISTENT fail-open (real drift the keeper should see) surfaces.
    Only fires when something is actually missing."""
    missing = []
    if not size:
        missing.append("no model size")
    if not gpu_total or ram_total is None:
        missing.append("no gpu/ram totals")
    if not missing:
        return
    wid = worker.get("id") or worker.get("name") or "?"
    key = (wid, model_key, ",".join(missing))
    if key in _FEAS_FAILOPEN_SEEN:
        return
    _FEAS_FAILOPEN_SEEN.add(key)
    logger.warning("feasibility fail-open for %s on %s: %s — all alloc modes "
                   "offered (transient before enrichment is fine; a persistent "
                   "one is a drift signal)",
                   model_key, worker.get("name") or wid, "; ".join(missing))


def feasible_modes_for(worker_id: str, model_key: str) -> Optional[tuple]:
    """The feasible allocation modes for (worker, model) from the RAW record —
    engine + box-totals aware. None if the worker is unknown; degrades to the
    full mode set on any lookup miss (never eliminate on missing data). Used to
    SURFACE the selectable set and to ENFORCE it at /assign."""
    worker = worker_store._load().get(worker_id)
    if worker is None:
        return None
    try:
        from ......managers.alloc_modes import feasible_modes
        size = _model_size_bytes(model_key)
        gpu_total = _worker_gpu_total_bytes(worker)
        ram_total = _worker_ram_total_bytes(worker)
        _warn_feasibility_failopen(worker, model_key, size, gpu_total, ram_total)
        return feasible_modes(_model_engine(model_key), size, gpu_total, ram_total,
                              moe_split_gpu_bytes=_model_moe_gpu_bytes(model_key))
    except Exception:  # noqa: BLE001 — a derivation must never break a read/relay
        from ......managers.alloc_modes import ALLOC_MODES
        return ALLOC_MODES


def feasibility_context(worker_id: str, model_key: str) -> Dict[str, Any]:
    """The raw numbers behind a feasibility decision for (worker, model), for an
    honest 409 reason and the UI. All bytes; None where central can't resolve.
    Empty dict if the worker is unknown."""
    worker = worker_store._load().get(worker_id)
    if worker is None:
        return {}
    return {
        "engine": _model_engine(model_key),
        "model_bytes": _model_size_bytes(model_key),
        "gpu_total_bytes": _worker_gpu_total_bytes(worker),
        "ram_total_bytes": _worker_ram_total_bytes(worker),
        # MoE (2026-07-24): the expert-split GPU need (non-expert + mmproj) a
        # feasibility decision priced GPU-fit with; None for dense models.
        "moe_split_gpu_bytes": _model_moe_gpu_bytes(model_key),
    }


def list_workers() -> List[Dict[str, Any]]:
    return worker_store.all()


def get_worker(worker_id: str) -> Optional[Dict[str, Any]]:
    return worker_store.get(worker_id)


def worker_storage_view(worker_id: str) -> Optional[Dict[str, Any]]:
    """Freshly-recomputed storage view + eviction proposal for a worker (from its
    RAW record). The /reap-approve route's central second guard. None if unknown."""
    return worker_store.storage_view(worker_id)


def pick_worker_for_model(model_key: str, pool: Optional[str] = None,
                          task: Optional[str] = None,
                          require_comfy_id_lock: bool = False) -> Optional[Dict[str, Any]]:
    return worker_store.pick_for_model(
        model_key, pool=pool, task=task,
        require_comfy_id_lock=require_comfy_id_lock)


def candidates_for_model(model_key: str, pool: Optional[str] = None,
                         task: Optional[str] = None) -> List[Dict[str, Any]]:
    """Ranked online workers holding ``model_key`` — the relay gate's reroute
    list (see WorkerStore.candidates_for_model). No routing side effects."""
    return worker_store.candidates_for_model(model_key, pool=pool, task=task)


def load_state_for_model(model_key: str, worker_id: str,
                         since_ts: float = 0.0) -> Optional[Dict[str, Any]]:
    """The cold-load HOLD's view (t36) of ``model_key`` on ``worker_id``.

    Reads the worker's LIVE heartbeat state — no worker-side change — and returns
    a compact status the core hold loop (resolvers.remote) consults:

      {"healthy": bool,       # resident/loaded now (ready to serve)
       "in_progress": bool,    # weights loading OR still downloading now
       "progress": float|None, # download fraction when provisioning
       "message": str|None,    # human progress line (from provision_progress)
       "error": str|None}      # a FRESH (ts>=since_ts) honest load failure

    ``error`` is only the worker's own last_load_error (load_reports, ok False)
    that is NEWER than ``since_ts`` — a stale error from a prior request never
    fails a fresh hold. It is returned VERBATIM; the core classifies transient vs
    honest so this stays a dumb reader. Returns None on any failure / unknown
    worker (the hold then degrades to a blind bounded retry)."""
    try:
        w = worker_store.get(worker_id)
        if not w:
            return None
        wanted = _match_keys(model_key)

        def _member(coll) -> Optional[str]:
            for m in (coll or []):
                if m == model_key or (_match_keys(m) & wanted):
                    return m
            return None

        loaded = _member(w.get("loaded_models"))
        in_prog = bool(_member(w.get("loading")) or _member(w.get("provisioning")))

        progress = None
        message = None
        pp = w.get("provision_progress") or {}
        if isinstance(pp, dict):
            for k, v in pp.items():
                if (k == model_key or (_match_keys(k) & wanted)) and isinstance(v, dict):
                    progress = v.get("progress")
                    message = v.get("message")
                    in_prog = True
                    break

        error = None
        reports = w.get("load_reports") or {}
        if isinstance(reports, dict):
            for k, v in reports.items():
                if not isinstance(v, dict):
                    continue
                if not (k == model_key or (_match_keys(k) & wanted)):
                    continue
                try:
                    fresh = float(v.get("ts") or 0) >= float(since_ts or 0)
                except (TypeError, ValueError):
                    fresh = False
                if v.get("ok") is False and fresh:
                    error = str(v.get("error") or "load failed")
                break

        return {
            "healthy": bool(loaded),
            "in_progress": in_prog,
            "progress": progress,
            "message": message,
            "error": error,
        }
    except Exception:  # noqa: BLE001 — advisory only, never break the hold
        return None


def explain_no_worker(model_key: str, pool: Optional[str] = None,
                      task: Optional[str] = None) -> str:
    """Human reason no worker took a request for ``model_key`` — the ``detail`` the
    refused-local error (HUGPY_NO_LOCAL_SERVING) surfaces so a DESIGNATED-but-idle
    model's failure is actionable instead of an opaque "no worker available".

    Walks the workers ASSIGNED to (or holding) the model and names why each was
    excluded from selection by a HARD static gate (admission, engine usability,
    dedicated-pool reservation, env tier, task capability) — the same gates
    ``workers_for_model`` applies. Returns "" when the model has no assigned worker
    at all (the caller's generic message already covers "assign it somewhere"),
    when every assigned worker actually passed the static gates (so the miss was
    transient — a stale beat or momentary cap, not a designation problem), or on
    ANY error: this is advisory and must never raise into a request.
    """
    try:
        # Operator BLOCK is the FIRST, most-specific reason — surfaced even when
        # the model has no assigned worker (unlike the static-gate walk below,
        # which only names DESIGNATED-but-excluded boxes). This is what turns the
        # refused-local error into a distinct "blocked from the serving pool by
        # the operator" line instead of a generic "no worker available".
        try:
            from abstract_hugpy_dev.comms.blocklist import block_reason as _br
            _blk = _br(model_key)
        except Exception:  # noqa: BLE001 — advisory; never raise into a request
            _blk = None
        if _blk:
            return _blk
        wanted = _match_keys(model_key)
        want_pool = (pool or "").strip()
        need_tier = env_tier_for_model(model_key)
        wildcards = _wildcard_map()   # read once; guarded (miss -> {})
        reasons: List[str] = []
        for w in worker_store.all():
            serveable = list(w.get("models", [])) + list(w.get("loaded_models", []))
            # Same membership predicate as workers_for_model (alias-tolerant,
            # blocked-sibling-guarded) so this diagnostic never disagrees with
            # the selector it explains. A WILDCARD worker is a potential server
            # for ANY model, so one excluded by a hard gate belongs in the
            # honest answer too — tagged "(wildcard)" so the operator can tell a
            # designation failure from lost all-comers overflow capacity.
            home = _serveable_match(model_key, wanted, serveable)
            wildcard = (not home) and bool(wildcards.get(w.get("id") or ""))
            if not (home or wildcard):
                continue                          # not designated for this model
            name = w.get("name") or w.get("id") or "worker"
            if wildcard:
                name = f"{name} (wildcard)"
            if w.get("admission") != "approved":
                reasons.append(f"{name}: not approved (admission={w.get('admission')!r})")
                continue
            if _engine_unusable(w):
                eng = w.get("engine") or {}
                sr = w.get("slot_incapable_reason")
                err = str(eng.get("error") or "").strip()
                if w.get("slot_capable") is False and sr:
                    why = str(sr)
                elif err:
                    why = f"llama-cpp not loadable: {err}"
                else:
                    why = "inference engine reports installed=False"
                reasons.append(f"{name}: engine unusable ({why[:400]})")
                continue
            if (w.get("pool") or "").strip() != want_pool:
                reasons.append(f"{name}: reserved for pool {w.get('pool')!r} "
                               f"(request pool {want_pool!r})")
                continue
            if _worker_env_tier(w) != need_tier:
                reasons.append(f"{name}: env tier {_worker_env_tier(w)!r} != "
                               f"required {need_tier!r}")
                continue
            if not _task_capable(w, task):
                reasons.append(f"{name}: cannot run task {task!r} "
                               f"(missing optional dependency)")
                continue
            # Passed every HARD static gate — its miss was runtime/transient, not a
            # designation problem; don't manufacture a reason for it.
        if not reasons:
            return ""
        return (f"{model_key} is assigned but no worker could serve it — "
                + "; ".join(reasons[:4])
                + ". Repair the worker (e.g. `hugpy install-engine` / reinstall "
                  "llama-cpp-python) or assign the model to a healthy worker.")
    except Exception:  # noqa: BLE001 — advisory only; never raise into a request
        return ""


def _reserved_vram_bytes(worker_id: str) -> int:
    """VRAM bytes CURRENTLY reserved on a worker by in-flight heavy video runs
    (p6). Lazily imported + fully guarded so placement never depends on the
    reservation layer being importable/healthy — 0 means 'nothing reserved'."""
    try:
        from ......video_intel.reservation.registry import reserved_bytes
        return int(reserved_bytes(worker_id) or 0)
    except Exception:  # noqa: BLE001 — admission-respect must never break placement
        return 0


def fleet_snapshot() -> list:
    """The deterministic allocator's view of the fleet, from the live registry.

    Each worker → a Node with summed free VRAM (across its GPUs), free RAM,
    rpc_endpoint, and online flag. This snapshot + a task's Need is all the
    allocator looks at, so the same registry state yields the same placement.
    """
    from ......managers.resolvers.allocator import Node
    nodes = []
    for w in worker_store.all():
        gpus = w.get("gpus") or []
        free_vram = sum(int(g.get("memory_free") or 0) for g in gpus)
        # p6 admission-respect: a card reserved by an in-flight heavy video run is
        # NOT free for another placement. Subtract the run's claimed peak so the
        # allocator never shards/places a model onto VRAM a Wan/Hunyuan render is
        # about to occupy. Best-effort (0 on any error) — a reservation-store
        # hiccup must never wrongly starve LLM placement.
        free_vram = max(0, free_vram - _reserved_vram_bytes(w["id"]))
        nodes.append(Node(
            id=w["id"],
            free_vram=free_vram,
            free_ram=int(w.get("free_ram") or 0),
            rpc_endpoint=w.get("rpc_endpoint"),
            can_lead=(w.get("role") != "rpc"),   # rpc nodes are backends, not leads
            online=(w.get("status") == "online"),
            env_tier=_worker_env_tier(w),
        ))
    return nodes


def plan_placement(bytes_needed: int, *, cpu_ok: bool = False, headroom: float = 1.15,
                   env_tier: Optional[str] = None):
    """Deterministically place a task needing ``bytes_needed`` on the live fleet.

    Returns the allocator's Placement (whole / shard / cpu / none). For a 'shard'
    result, ``placement.rpc_servers`` + ``placement.tensor_split`` are what the
    lead is handed as a spill override. ``env_tier`` (when set) restricts the
    snapshot to workers serving that runtime-env tier — the allocator stays
    env-agnostic; we filter its input.
    """
    from ......managers.resolvers.allocator import Need, allocate
    nodes = fleet_snapshot()
    if env_tier:
        nodes = [n for n in nodes if n.env_tier == env_tier]
    return allocate(
        Need(bytes_needed=int(bytes_needed), cpu_ok=cpu_ok, headroom=headroom),
        nodes,
    )


def _shard_eligible() -> Dict[str, int]:
    """Models the operator allows to shard, with a VRAM byte estimate.

    Parsed from ``HUGPY_SHARD_MODELS`` = ``"key:bytes,key2:bytes"`` (bytes may use
    a ``g``/``gb`` suffix, e.g. ``BigModel:140gb``). A model NOT listed never
    shards — so this whole path is a no-op until the operator opts a model in.
    """
    out: Dict[str, int] = {}
    for part in os.environ.get("HUGPY_SHARD_MODELS", "").split(","):
        part = part.strip()
        if not part or ":" not in part:
            continue
        key, _, raw = part.rpartition(":")
        raw = raw.strip().lower()
        mult = 2**30 if raw.endswith(("g", "gb")) else 1
        num = raw.rstrip("gb").strip()
        try:
            out[key.strip()] = int(float(num) * mult)
        except ValueError:
            continue
    return out


def placement_for_model(model_key: str) -> Optional[Dict[str, Any]]:
    """Allocator-driven shard placement for the remote.py seam.

    Returns ``{"worker": <lead dict>, "spill": {...}}`` only when ``model_key`` is
    shard-eligible AND the allocator decides it must shard across the pool; else
    ``None`` so the caller uses ordinary whole-model routing. The spill carries
    ``rpc_servers`` + a VRAM-proportional ``tensor_split`` + ``n_gpu_layers=-1``.
    """
    elig = _shard_eligible()
    need = elig.get(model_key) or elig.get(str(model_key).split("/")[-1])
    if not need:
        return None
    placement = plan_placement(need, cpu_ok=False,
                               env_tier=env_tier_for_model(model_key))
    if placement.kind != "shard":
        return None
    lead = get_worker(placement.lead_id)
    if not lead or not lead.get("url"):
        return None
    return {
        "worker": lead,
        "spill": {
            "rpc_servers": ",".join(placement.rpc_servers),
            "tensor_split": list(placement.tensor_split),
            "n_gpu_layers": -1,
        },
    }


# Register this pool's selector with the core router (web -> core — the correct
# dependency direction). resolve() consults it to offload a (model, task) to a
# live GPU worker, falling back to local. This module is imported at web-app
# startup; the standalone worker agent never imports it, so the core router
# simply runs everything local there (and delegated requests carry _force_local).
try:
    from ......managers.resolvers import (
        set_worker_provider as _set_worker_provider,
        set_placement_provider as _set_placement_provider,
    )
    _set_worker_provider(pick_worker_for_model, spill_for)
    # Allocator-driven sharding. No-op until a model is opted in via
    # HUGPY_SHARD_MODELS, so it never affects ordinary routing by default.
    _set_placement_provider(placement_for_model)
    # Cap-aware relay reroute (concurrency hardening 2026-07-11): the core gate
    # asks this for alternative online workers when the primary is at its
    # advertised in-process concurrency cap. Optional in older cores — guarded.
    try:
        from ......managers.resolvers.remote import set_worker_candidates_provider
        set_worker_candidates_provider(candidates_for_model)
    except Exception as _exc2:  # older core without the seam — gate degrades to primary-only
        import logging as _logging
        _logging.getLogger(__name__).info(
            "candidates provider not registered (older core): %s", _exc2)
    # No-worker diagnostic (2026-07-15): when selection yields no worker and this
    # box refuses local serving, the refused-local error names the DESIGNATED-but-
    # excluded worker(s) + reason (broken engine / no llama-server binary), turning
    # the opaque "assigned+pinned but 500s" mystery into an actionable message.
    # Optional in older cores — guarded; unset ⇒ the message is byte-identical.
    try:
        from ......managers.resolvers.remote import set_no_worker_diagnostic
        set_no_worker_diagnostic(explain_no_worker)
    except Exception as _exc3:  # older core without the seam — message unchanged
        import logging as _logging
        _logging.getLogger(__name__).info(
            "no-worker diagnostic not registered (older core): %s", _exc3)
    # Cold-load HOLD load-state (t36): lets the core hold a FEASIBLE-but-COLD
    # call as a presumed success, surfacing the worker's live load progress and
    # honest last_load_error. Central-only (reads the heartbeat); optional in
    # older cores — guarded; unset ⇒ the hold degrades to a blind bounded retry.
    try:
        from ......managers.resolvers.remote import set_load_state_provider
        set_load_state_provider(load_state_for_model)
    except Exception as _exc4:  # older core without the seam — hold degrades
        import logging as _logging
        _logging.getLogger(__name__).info(
            "load-state provider not registered (older core): %s", _exc4)
except Exception as _exc:  # never let registration break importing the pool
    import logging as _logging
    _logging.getLogger(__name__).warning("worker provider registration failed: %s", _exc)
