Upload folder using huggingface_hub

Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install uv
+RUN pip install --no-cache-dir uv
+
+# Create non-root user (HF Spaces runs as UID 1000)
+RUN useradd -m -u 1000 user
+
+# Copy workspace definition and lockfile first for layer caching
+COPY --chown=user pyproject.toml uv.lock ./
+COPY --chown=user packages/eval-entity-resolver/pyproject.toml packages/eval-entity-resolver/
+
+# Copy source
+COPY --chown=user packages/eval-entity-resolver/src packages/eval-entity-resolver/src
+COPY --chown=user src src
+
+# Install all workspace packages
+RUN uv sync --no-dev
+
+USER user
+ENV PATH="/app/.venv/bin:$PATH"
+ENV HF_HOME=/tmp/hf_cache
+ENV LOCAL_MODE=false
+ENV READ_ONLY=true
+
+EXPOSE 7860
+
+CMD ["uvicorn", "eval_card_registry.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,11 +1,87 @@
 ---
-title: Entity Registry
-emoji: 🦀
-colorFrom: red
-colorTo: indigo
+title: eval-card-registry
+emoji: 🗂️
+colorFrom: blue
+colorTo: green
 sdk: docker
+app_port: 7860
 pinned: false
-short_description: WIP registry to help resolve benchmarks, models, and metrics
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# eval-card-registry
+
+Query-only disambiguation API for AI evaluation entity names. Resolves raw benchmark / model / metric / harness strings (e.g. `"MATH Level 5"`) to stable canonical IDs (`math`).
+
+This Space runs in **read-only mode** — it serves lookups against pre-built entity data. Write operations (entity creation, alias edits) happen in a separate pipeline.
+
+## Base URL
+
+```
+https://evaleval-entity-registry.hf.space/api/v1
+```
+
+## Resolve
+
+```bash
+curl -X POST https://evaleval-entity-registry.hf.space/api/v1/resolve \
+  -H 'Content-Type: application/json' \
+  -d '{"raw_value": "MATH Level 5", "entity_type": "benchmark"}'
+```
+
+Response:
+
+```json
+{
+  "canonical_id": "math-level-5",
+  "strategy": "exact",
+  "confidence": 1.0,
+  "created_new": false,
+  "review_status": "reviewed"
+}
+```
+
+If nothing matches, `canonical_id` is `null` and `strategy` is `"no_match"`. In read-only mode, no draft entity is created.
+
+`entity_type` is one of: `benchmark`, `model`, `metric`, `harness`. Optional `source_config` scopes the lookup to a specific source.
+
+**Batch resolve:**
+
+```bash
+curl -X POST https://evaleval-entity-registry.hf.space/api/v1/resolve/batch \
+  -H 'Content-Type: application/json' \
+  -d '[
+    {"raw_value": "MATH Level 5", "entity_type": "benchmark"},
+    {"raw_value": "meta-llama/Llama-3.1-8B", "entity_type": "model"}
+  ]'
+```
+
+## Browse entities
+
+```
+GET /api/v1/benchmarks?search=math
+GET /api/v1/benchmarks/{id}
+GET /api/v1/models
+GET /api/v1/metrics
+GET /api/v1/harnesses
+GET /api/v1/aliases?status=uncertain&entity_type=benchmark
+```
+
+## Health
+
+```
+GET /api/v1/health
+GET /api/v1/stats
+```
+
+## Write endpoints
+
+Disabled in this Space. `POST`/`PATCH` on entities and aliases return `405 Method Not Allowed`. Mutations happen in the data pipeline (separate from this Space).
+
+## Interactive docs
+
+OpenAPI docs at `/docs`.
+
+## Data sources
+
+- Entity data: HF Dataset repo `evaleval/entity-registry-data` (read at startup)
+- Resolve logs: HF Storage Bucket `evaleval/entity-registry-storage` (written asynchronously for resolver improvement)

packages/eval-entity-resolver/pyproject.toml

@@ -0,0 +1,16 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "eval-entity-resolver"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "pandas>=2.2.0",
+    "pyarrow>=16.0.0",
+    "huggingface-hub>=0.23.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/eval_entity_resolver"]

packages/eval-entity-resolver/src/eval_entity_resolver/__init__.py

@@ -0,0 +1,13 @@
+from eval_entity_resolver.alias_store import AliasStore
+from eval_entity_resolver.eee import clean_eval_name, extract_metric
+from eval_entity_resolver.models import ResolutionResult, ResolverConfig
+from eval_entity_resolver.resolver import Resolver
+
+__all__ = [
+    "AliasStore",
+    "Resolver",
+    "ResolverConfig",
+    "ResolutionResult",
+    "clean_eval_name",
+    "extract_metric",
+]

packages/eval-entity-resolver/src/eval_entity_resolver/alias_store.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+
+
+_SCHEMA = {
+    "id": pd.StringDtype(),
+    "raw_value": pd.StringDtype(),
+    "entity_type": pd.StringDtype(),
+    "canonical_id": pd.StringDtype(),
+    "source_config": pd.StringDtype(),
+    "source_field": pd.StringDtype(),
+    "status": pd.StringDtype(),
+    "strategy": pd.StringDtype(),
+    "confidence": "float64",
+    "notes": pd.StringDtype(),
+    "created_at": pd.StringDtype(),
+    "updated_at": pd.StringDtype(),
+}
+
+
+def _empty_df() -> pd.DataFrame:
+    return pd.DataFrame({col: pd.Series(dtype=dtype) for col, dtype in _SCHEMA.items()})
+
+
+class AliasStore:
+    """Wraps the aliases table. Loaded into memory; writes are in-memory only."""
+
+    def __init__(self, df: pd.DataFrame, read_only: bool = False) -> None:
+        self._df = df.copy()
+        self.read_only = read_only
+        # Per-entity_type caches — built lazily on first access
+        # Normalized lookup key: (entity_type, source_config or None)
+        self._normalized_cache: dict[tuple[str, Optional[str]], dict[str, str]] = {}
+        self._candidates_cache: dict[tuple[str, Optional[str]], list[tuple[str, str]]] = {}
+        self._lookup_index: dict[tuple[str, str, Optional[str]], str] | None = None
+
+    def _ensure_lookup_index(self) -> None:
+        """Build a dict index for O(1) exact lookups."""
+        if self._lookup_index is not None:
+            return
+        self._lookup_index = {}
+        df = self._df[self._df["status"] != "rejected"]
+        for _, row in df.iterrows():
+            key = (row["raw_value"], row["entity_type"], row.get("source_config"))
+            self._lookup_index[key] = row["canonical_id"]
+
+    def _invalidate_caches(self) -> None:
+        self._normalized_cache.clear()
+        self._candidates_cache.clear()
+        self._lookup_index = None
+
+    # ------------------------------------------------------------------
+    # Constructors
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def from_parquet(cls, path: str | Path, read_only: bool = False) -> "AliasStore":
+        p = Path(path) / "aliases.parquet"
+        if p.exists():
+            df = pd.read_parquet(p)
+        else:
+            df = _empty_df()
+        return cls(df, read_only=read_only)
+
+    @classmethod
+    def from_hf(cls, repo_id: str, read_only: bool = False) -> "AliasStore":
+        from huggingface_hub import hf_hub_download
+
+        try:
+            local = hf_hub_download(
+                repo_id=repo_id,
+                filename="aliases/part-0.parquet",
+                repo_type="dataset",
+            )
+            df = pd.read_parquet(local)
+        except Exception:
+            df = _empty_df()
+        return cls(df, read_only=read_only)
+
+    # ------------------------------------------------------------------
+    # Lookup
+    # ------------------------------------------------------------------
+
+    def lookup(
+        self,
+        raw_value: str,
+        entity_type: str,
+        source_config: Optional[str],
+    ) -> Optional[str]:
+        """Return canonical_id for first non-rejected match. Config-scoped before global."""
+        self._ensure_lookup_index()
+        # Config-scoped
+        if source_config:
+            result = self._lookup_index.get((raw_value, entity_type, source_config))
+            if result is not None:
+                return result
+        # Global
+        return self._lookup_index.get((raw_value, entity_type, None))
+
+    # ------------------------------------------------------------------
+    # Writes (in-memory only; caller is responsible for persistence)
+    # ------------------------------------------------------------------
+
+    def add_alias(
+        self,
+        raw_value: str,
+        entity_type: str,
+        canonical_id: str,
+        source_config: Optional[str],
+        source_field: Optional[str],
+        status: str,
+        strategy: str,
+        confidence: float,
+    ) -> None:
+        if self.read_only:
+            raise RuntimeError("AliasStore is read-only")
+        now = datetime.now(timezone.utc).isoformat()
+        row = {
+            "id": str(uuid.uuid4()),
+            "raw_value": raw_value,
+            "entity_type": entity_type,
+            "canonical_id": canonical_id,
+            "source_config": source_config,
+            "source_field": source_field,
+            "status": status,
+            "strategy": strategy,
+            "confidence": confidence,
+            "notes": None,
+            "created_at": now,
+            "updated_at": now,
+        }
+        self._df = pd.concat([self._df, pd.DataFrame([row])], ignore_index=True)
+        self._invalidate_caches()
+
+    def update_alias(
+        self,
+        raw_value: str,
+        entity_type: str,
+        source_config: Optional[str],
+        canonical_id: str,
+        status: str,
+        strategy: str,
+        confidence: float,
+    ) -> None:
+        """Upsert: update existing alias row or add new one."""
+        if self.read_only:
+            raise RuntimeError("AliasStore is read-only")
+        df = self._df
+        mask = (df["raw_value"] == raw_value) & (df["entity_type"] == entity_type)
+        if source_config:
+            mask = mask & (df["source_config"] == source_config)
+        else:
+            mask = mask & df["source_config"].isna()
+        if mask.any():
+            now = datetime.now(timezone.utc).isoformat()
+            self._df.loc[mask, "canonical_id"] = canonical_id
+            self._df.loc[mask, "status"] = status
+            self._df.loc[mask, "strategy"] = strategy
+            self._df.loc[mask, "confidence"] = confidence
+            self._df.loc[mask, "updated_at"] = now
+            self._invalidate_caches()
+        else:
+            self.add_alias(raw_value, entity_type, canonical_id, source_config, None, status, strategy, confidence)
+
+    # ------------------------------------------------------------------
+    # Export
+    # ------------------------------------------------------------------
+
+    def to_dataframe(self) -> pd.DataFrame:
+        return self._df.copy()
+
+    def get_normalized_lookup(
+        self, entity_type: str, source_config: Optional[str] = None
+    ) -> dict[str, str]:
+        """Return {normalized_raw_value: canonical_id} for use by strategies.
+
+        When ``source_config`` is given, the returned map merges config-scoped
+        aliases on top of global (source_config IS NULL) aliases, so scoped
+        matches win over global for the same normalized form. When
+        ``source_config`` is None, only global aliases are included — scoped
+        aliases do NOT leak into unrelated lookups.
+        """
+        key = (entity_type, source_config)
+        if key in self._normalized_cache:
+            return self._normalized_cache[key]
+
+        from eval_entity_resolver.normalization import normalize
+
+        base = self._df[(self._df["entity_type"] == entity_type) & (self._df["status"] != "rejected")]
+        # Start from global aliases.
+        global_df = base[base["source_config"].isna()]
+        result: dict[str, str] = {}
+        for _, row in global_df.iterrows():
+            result[normalize(row["raw_value"])] = row["canonical_id"]
+        # Overlay scoped aliases for the requested source_config.
+        if source_config:
+            scoped_df = base[base["source_config"] == source_config]
+            for _, row in scoped_df.iterrows():
+                result[normalize(row["raw_value"])] = row["canonical_id"]
+        self._normalized_cache[key] = result
+        return result
+
+    def get_all_for_type(
+        self, entity_type: str, source_config: Optional[str] = None
+    ) -> list[tuple[str, str]]:
+        """Return [(raw_value, canonical_id)] for non-rejected aliases of ``entity_type``.
+
+        Filtering matches ``get_normalized_lookup`` — when ``source_config`` is
+        given, includes global + that config's scoped aliases; otherwise global
+        only. Cached per (entity_type, source_config).
+        """
+        key = (entity_type, source_config)
+        if key in self._candidates_cache:
+            return self._candidates_cache[key]
+
+        base = self._df[(self._df["entity_type"] == entity_type) & (self._df["status"] != "rejected")]
+        if source_config:
+            mask = base["source_config"].isna() | (base["source_config"] == source_config)
+            df = base[mask]
+        else:
+            df = base[base["source_config"].isna()]
+        result = list(zip(df["raw_value"].tolist(), df["canonical_id"].tolist()))
+        self._candidates_cache[key] = result
+        return result

packages/eval-entity-resolver/src/eval_entity_resolver/eee.py

@@ -0,0 +1,198 @@
+"""
+EEE-specific preprocessing for entity resolution.
+
+Raw strings from the EEE datastore often encode multiple entity types in a
+single field (e.g. ``evaluation_name`` contains both benchmark and metric).
+These helpers extract clean, resolvable strings before passing them to the
+resolver.
+
+Usage::
+
+    from eval_entity_resolver.eee import extract_metric, clean_eval_name
+
+    metric_raw = extract_metric("Accuracy on IFEval")       # → "Accuracy"
+    bench_raw  = clean_eval_name("bfcl.live.live_accuracy")  # → "bfcl live"
+"""
+from __future__ import annotations
+
+import re
+
+
+# ------------------------------------------------------------------
+# Metric extraction
+# ------------------------------------------------------------------
+
+def extract_metric(metric_desc: str) -> str:
+    """Extract a reusable metric name from an EEE evaluation description.
+
+    EEE configs rarely provide a structured metric_id.  Instead the metric
+    lives inside ``evaluation_description`` in one of several formats:
+
+    * **"X on Y"** — ``"Accuracy on IFEval"`` → ``"Accuracy"``
+    * **Dot notation** — ``"bfcl.live.live_accuracy"`` → ``"accuracy"``
+    * **Verbose description** — ``"Chat accuracy - includes easy subsets"``
+      → ``"accuracy"`` (keyword extraction)
+    * **No keyword** — ``"Global MMLU Lite - Arabic"`` → ``"score"``
+      (generic fallback)
+
+    The returned string is passed to the resolver, which maps it to a
+    canonical metric entity via alias lookup / normalized match.
+    """
+    text = metric_desc.strip()
+    if not text:
+        return text
+
+    from_dot = False
+
+    # 1. Dot notation: "bfcl.live.live_accuracy" → last segment → "live accuracy"
+    if "." in text and " " not in text:
+        text = text.rsplit(".", 1)[1].replace("_", " ").strip()
+        from_dot = True
+
+    # 2. "X on Y" pattern: "Accuracy on IFEval" → "Accuracy"
+    if not from_dot:
+        m = re.match(r"^(.+?)\s+on\s+\S+", text, re.IGNORECASE)
+        if m:
+            text = m.group(1).strip()
+
+    # 3. Try keyword extraction on any multi-word text or dot-notation segment.
+    #    Single bare words ("Accuracy", "F1", "EM") pass straight to the resolver.
+    word_count = len(text.split())
+    needs_extraction = from_dot or word_count > 1
+
+    if needs_extraction:
+        canonical = _keyword_extract(text)
+        if canonical:
+            return canonical
+        # No keyword found — verbose descriptions (4+ words) → generic fallback.
+        # Short phrases (2-3 words) pass through so the resolver can still
+        # match them via alias (e.g. "Equivalent (CoT)" → cot-correct).
+        if not from_dot and word_count > 3:
+            return "score"
+
+    return text
+
+
+# Ordered from most-specific to most-generic.  When multiple patterns
+# match, the earliest *position* in the input text wins (see
+# _keyword_extract).
+_METRIC_KEYWORDS: list[tuple[str, str]] = [
+    # Multi-word / compound patterns
+    (r"pass@8",                          "Pass@8"),
+    (r"pass@1",                          "Pass@1"),
+    (r"mean[\s_-]*win[\s_-]*rate",       "Mean Win Rate"),
+    (r"win[\s_-]*rate",                  "Win Rate"),
+    (r"mean[\s_-]*response[\s_-]*time",  "Mean Response Time"),
+    (r"mean[\s_-]*score",                "Mean Score"),
+    (r"exact[\s_-]*match",               "Exact Match"),
+    (r"bleu[\s_-]*4",                    "BLEU-4"),
+    (r"cot[\s_-]*correct",              "COT correct"),
+    (r"wb[\s_-]*score",                  "WB Score"),
+    (r"avg[\s_-]*attempts",              "Average Attempts"),
+    (r"latency[\s_-]*mean",              "mean-latency"),
+    (r"latency.*(?:p95|95th)",            "p95-latency"),
+    (r"latency.*(?:std|standard)",        "latency-stddev"),
+    (r"max[\s_-]*delta",                 "max-delta"),
+    (r"benchmark\s+evaluation",          "score"),
+    (r"outperform",                      "rank"),
+    # Compound accuracy types (before generic accuracy)
+    # Patterns sourced from metric_names in evaleval/card_backend eval-list.
+    (r"ast[\s_-]*accuracy",              "AST Accuracy"),
+    (r"overall[\s_-]*accuracy",          "Accuracy"),
+    (r"(?:ir)?relevance[\s_-]*detection[\s_-]*accuracy", "Accuracy"),
+    (r"no[\s_-]*snippet[\s_-]*accuracy", "Accuracy"),
+    (r"long[\s_-]*context[\s_-]*accuracy", "Accuracy"),
+    (r"kv[\s_-]*accuracy",               "Accuracy"),
+    (r"vector[\s_-]*accuracy",           "Accuracy"),
+    (r"recursive[\s_-]*summarization[\s_-]*accuracy", "Accuracy"),
+    (r"total[\s_-]*cost",                "cost"),
+    (r"cost[\s_-]*per[\s_-]*task",       "cost-per-task"),
+    # Single-word patterns (generic, checked last by position)
+    (r"\baccuracy\b",                    "Accuracy"),
+    (r"\bacc\b",                         "Accuracy"),
+    (r"\bscores?\b",                     "score"),
+    (r"\bf1\b",                          "F1"),
+    (r"\bem\b",                          "Exact Match"),
+    (r"\belo\b",                         "Elo Rating"),
+    (r"\branks?\b",                      "rank"),
+    (r"\bcosts?\b",                      "cost"),
+    (r"\bharmlessness\b",                "harmlessness"),
+    (r"\bstddev\b",                      "stddev"),
+]
+
+
+def _keyword_extract(text: str) -> str | None:
+    """Return the canonical metric name for the first keyword found in *text*."""
+    lower = text.lower()
+    best: str | None = None
+    best_pos = len(lower) + 1
+    for pattern, canonical in _METRIC_KEYWORDS:
+        m = re.search(pattern, lower)
+        if m and m.start() < best_pos:
+            best_pos = m.start()
+            best = canonical
+    return best
+
+
+# ------------------------------------------------------------------
+# Benchmark-name cleaning
+# ------------------------------------------------------------------
+
+# Trailing metric patterns for space-separated names (e.g.
+# "Gaming Score" → "Gaming").  Checked with ``re.search`` against
+# the lowered name; the first match wins.
+_TRAILING_METRIC_RE: list[str] = [
+    r"mean\s+win\s+rate$",
+    r"mean\s+response\s+time$",
+    r"mean\s+score$",
+    r"win\s+rate$",
+    r"avg\s+attempts$",
+    r"avg\s+latency\s+ms$",
+    r"cost\s+per\s+\d+\s+calls\s+usd$",
+    r"cost\s+per\s+task$",
+    r"pass@\d+$",
+    r"\b(?:score|accuracy|acc|elo|rank|f1|em)$",
+]
+
+
+def clean_eval_name(eval_name: str) -> str:
+    """Strip embedded metric information from an ``evaluation_name``.
+
+    EEE configs often encode both benchmark *and* metric in a single
+    ``evaluation_name`` string.  This function extracts the benchmark
+    portion so that the metric lives only in ``metric_id``.
+
+    Patterns handled:
+
+    * **Dot notation** — ``"bfcl.live.live_accuracy"`` → ``"bfcl live"``
+      (last segment is the metric, everything before is the benchmark)
+    * **Underscore suffix** — ``"fibble1_arena_win_rate"`` → ``"fibble1 arena"``
+    * **Trailing words** — ``"Gaming Score"`` → ``"Gaming"``
+    """
+    name = eval_name.strip()
+    if not name:
+        return name
+
+    # --- 1. Dot notation: split on last dot ------------------------------
+    # The last segment is the metric; everything before is the benchmark.
+    # e.g. "bfcl.live.live_simple_ast_accuracy" → "bfcl live"
+    if "." in name and " " not in name:
+        parts = name.rsplit(".", 1)[0].split(".")
+        return " ".join(p.replace("_", " ") for p in parts)
+
+    # --- 2. Underscore/space names: strip trailing metric keywords -------
+    # Normalise underscores to spaces so "fibble1_arena_win_rate" and
+    # "Gaming Score" use the same codepath.
+    has_underscores = "_" in name and " " not in name
+    normalized = name.replace("_", " ") if has_underscores else name
+
+    lower = normalized.lower()
+    for pattern in _TRAILING_METRIC_RE:
+        m = re.search(pattern, lower)
+        if m:
+            prefix = normalized[: m.start()].strip()
+            if prefix:
+                return prefix
+            break  # matched but prefix is empty — fall through
+
+    return name

packages/eval-entity-resolver/src/eval_entity_resolver/models.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass, field
+from typing import Literal, Optional
+
+
+ResolutionStrategy = Literal["exact", "normalized", "fuzzy", "no_match"]
+EntityType = Literal["model", "benchmark", "metric", "harness"]
+
+
+@dataclass
+class ResolutionResult:
+    raw_value: str
+    entity_type: EntityType
+    source_config: Optional[str]
+    canonical_id: Optional[str]
+    strategy: ResolutionStrategy
+    confidence: float
+
+
+@dataclass
+class ResolverConfig:
+    threshold: float = 0.85

packages/eval-entity-resolver/src/eval_entity_resolver/normalization.py

@@ -0,0 +1,23 @@
+import re
+
+
+def normalize(value: str) -> str:
+    """Lowercase, strip, collapse all separators (space/_/-/slash) to one space.
+
+    Collapsing ``/`` with the other separators means ``tau-bench-2/airline``
+    and ``tau-bench-2_airline`` normalize identically — critical for generalized
+    ingestion where the same benchmark may appear with slash, underscore, or
+    hyphen separators across configs. False merges across distinct canonical
+    IDs are prevented by fuzzy's suffix-stripping being the *only* stem rewrite
+    we apply (no generic similarity).
+
+    Dots between digits are converted to spaces first so that version
+    numbers like ``4.5`` and ``4-5`` normalize identically (both → ``4 5``).
+    """
+    value = value.lower()
+    value = value.strip()
+    # Convert dots between digits to spaces (e.g. "4.5" → "4 5")
+    value = re.sub(r"(?<=\d)\.(?=\d)", " ", value)
+    value = re.sub(r"[^\w\s\-/]", "", value)         # remove punctuation first
+    value = re.sub(r"[\s_\-/]+", " ", value).strip() # collapse separators
+    return value

packages/eval-entity-resolver/src/eval_entity_resolver/resolver.py

@@ -0,0 +1,69 @@
+from typing import Optional
+
+from eval_entity_resolver.alias_store import AliasStore
+from eval_entity_resolver.models import ResolutionResult, ResolverConfig
+from eval_entity_resolver.strategies.exact import exact_match
+from eval_entity_resolver.strategies.normalized import normalized_match
+from eval_entity_resolver.strategies.fuzzy import fuzzy_match
+
+
+class Resolver:
+    def __init__(self, store: AliasStore, config: Optional[ResolverConfig] = None) -> None:
+        self.store = store
+        self.config = config or ResolverConfig()
+
+    def resolve(
+        self,
+        raw_value: str,
+        entity_type: str,
+        source_config: Optional[str] = None,
+    ) -> ResolutionResult:
+        # 1. Exact
+        canonical_id = exact_match(raw_value, entity_type, source_config, self.store)
+        if canonical_id is not None:
+            return ResolutionResult(
+                raw_value=raw_value,
+                entity_type=entity_type,
+                source_config=source_config,
+                canonical_id=canonical_id,
+                strategy="exact",
+                confidence=1.0,
+            )
+
+        # 2. Normalized (confidence 0.95 — only return if above threshold)
+        _NORMALIZED_CONFIDENCE = 0.95
+        if _NORMALIZED_CONFIDENCE >= self.config.threshold:
+            canonical_id = normalized_match(raw_value, entity_type, self.store, source_config)
+            if canonical_id is not None:
+                return ResolutionResult(
+                    raw_value=raw_value,
+                    entity_type=entity_type,
+                    source_config=source_config,
+                    canonical_id=canonical_id,
+                    strategy="normalized",
+                    confidence=_NORMALIZED_CONFIDENCE,
+                )
+
+        # 3. Fuzzy
+        canonical_id, confidence = fuzzy_match(
+            raw_value, entity_type, self.config.threshold, self.store, source_config
+        )
+        if canonical_id is not None:
+            return ResolutionResult(
+                raw_value=raw_value,
+                entity_type=entity_type,
+                source_config=source_config,
+                canonical_id=canonical_id,
+                strategy="fuzzy",
+                confidence=confidence,
+            )
+
+        # 4. No match
+        return ResolutionResult(
+            raw_value=raw_value,
+            entity_type=entity_type,
+            source_config=source_config,
+            canonical_id=None,
+            strategy="no_match",
+            confidence=0.0,
+        )

packages/eval-entity-resolver/src/eval_entity_resolver/strategies/exact.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+
+def exact_match(
+    raw_value: str,
+    entity_type: str,
+    source_config: Optional[str],
+    alias_store,
+) -> Optional[str]:
+    """Direct lookup in alias store — config-scoped then global."""
+    return alias_store.lookup(raw_value, entity_type, source_config)

packages/eval-entity-resolver/src/eval_entity_resolver/strategies/fuzzy.py

@@ -0,0 +1,148 @@
+"""
+Fuzzy matching strategy.
+
+Instead of generic string similarity (which falsely merges distinct versions,
+sizes, and variants), this uses two targeted approaches:
+
+1. **Stem matching** — strip known non-semantic suffixes (evaluation mode,
+   hosting provider, effort level) and check if the stem has an exact or
+   normalized match.  This catches real duplicates like
+   ``model-name-fc`` → ``model-name`` without collapsing ``gpt-5-mini`` into
+   ``gpt-5``.
+
+2. **Org normalization** — handle cases where the org prefix differs between
+   configs (``deepseek-ai/model`` vs ``deepseek/model``).
+
+If neither approach produces a match the strategy returns None so the resolver
+can fall through to auto-draft.
+"""
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+from eval_entity_resolver.normalization import normalize
+
+
+# Suffixes stripped only from the *end* of the raw value.  Order matters:
+# longer suffixes first to avoid partial stripping.
+#
+# Many of these patterns were identified from raw_model_ids in the
+# evaleval/card_backend dataset, where a single canonical model family
+# (e.g. ``anthropic/claude-opus-4-5``) has variants like
+# ``claude-opus-4-5-20251101-thinking-16k``, ``-fc``, ``-prompt``, etc.
+_STRIP_SUFFIXES = [
+    # Evaluation-mode suffixes (BFCL, etc.)
+    "-fc",
+    "-prompt",
+    # Hosting-provider suffixes
+    "-together",
+    "-bedrock",
+    # Reasoning-effort suffixes
+    "-high",
+    "-medium",
+    "-low",
+    "-minimal",
+    # Thinking-style suffixes
+    "-nothink",
+    "-thinking-none",
+]
+
+# Regex-based suffix patterns applied after the literal suffixes.
+# These capture variants with numeric parameters (thinking budgets, dates).
+# Each pattern must anchor with $ and match only the tail of the string.
+_STRIP_SUFFIX_PATTERNS: list[re.Pattern[str]] = [
+    # Thinking-budget suffix: "-thinking-8k", "-thinking-16k", "-thinking-64k"
+    re.compile(r"-thinking-\d+k$", re.IGNORECASE),
+    # Date version suffix (YYYYMMDD): "-20251101", "-20240315"
+    # Only strip dates (8 consecutive digits) to avoid touching version numbers.
+    re.compile(r"-\d{8}$"),
+]
+
+# Known org aliases: {variant_prefix: canonical_prefix}
+# Convention: simplify HF org names (e.g. "deepseek-ai" → "deepseek") to the
+# shorter form used as canonical in this registry.
+_ORG_ALIASES: dict[str, str] = {
+    "deepseek-ai": "deepseek",
+    "cohereforai": "cohere",
+    "cohere-labs": "cohere",
+    "tii-uae": "tiiuae",
+    "meta-llama": "meta",
+    "mistral-ai": "mistralai",
+    "nvidia-nemo": "nvidia",
+}
+
+# Confidence assigned to stem-match results.  Below 1.0 (exact) and 0.95
+# (normalized) so the provenance is clear in the resolution log.
+_STEM_CONFIDENCE = 0.90
+
+
+def _strip_suffix(value: str) -> str | None:
+    """Strip a single known suffix.  Returns the stem or None if no suffix matched."""
+    lower = value.lower()
+    for suffix in _STRIP_SUFFIXES:
+        if lower.endswith(suffix):
+            return value[: len(value) - len(suffix)]
+    for pattern in _STRIP_SUFFIX_PATTERNS:
+        m = pattern.search(value)
+        if m:
+            return value[: m.start()]
+    return None
+
+
+def _normalize_org(value: str) -> str | None:
+    """Replace a known org-alias prefix.  Returns the rewritten string or None."""
+    if "/" not in value:
+        return None
+    org, rest = value.split("/", 1)
+    canonical_org = _ORG_ALIASES.get(org.lower())
+    if canonical_org is None:
+        return None
+    return f"{canonical_org}/{rest}"
+
+
+def fuzzy_match(
+    raw_value: str,
+    entity_type: str,
+    threshold: float,  # kept for API compat; not used by stem matching
+    alias_store,
+    source_config: Optional[str] = None,
+) -> tuple[Optional[str], float]:
+    """
+    Attempt targeted fuzzy resolution.
+
+    Returns ``(canonical_id, confidence)``; canonical_id is None on no match.
+    """
+    candidates_to_try: list[str] = []
+
+    # 1. Suffix stripping (may produce multiple stems: strip one, strip two, etc.)
+    stripped = _strip_suffix(raw_value)
+    if stripped:
+        candidates_to_try.append(stripped)
+        # Try double-strip (e.g. "model-fc-together" — unlikely but cheap)
+        double = _strip_suffix(stripped)
+        if double:
+            candidates_to_try.append(double)
+
+    # 2. Org normalization — on both original and stripped forms
+    for val in [raw_value] + candidates_to_try[:]:
+        rewritten = _normalize_org(val)
+        if rewritten:
+            candidates_to_try.append(rewritten)
+
+    # 3. Check each candidate against exact then normalized lookups.
+    # Scoped-aware: config-scoped aliases for ``source_config`` count as
+    # candidates; unrelated scoped aliases are excluded.
+    norm_lookup = alias_store.get_normalized_lookup(entity_type, source_config)
+
+    for candidate in candidates_to_try:
+        exact_id = alias_store.lookup(candidate, entity_type, source_config)
+        if exact_id is not None:
+            return exact_id, _STEM_CONFIDENCE
+
+        norm = normalize(candidate)
+        canonical_id = norm_lookup.get(norm)
+        if canonical_id is not None:
+            return canonical_id, _STEM_CONFIDENCE
+
+    return None, 0.0

packages/eval-entity-resolver/src/eval_entity_resolver/strategies/normalized.py

@@ -0,0 +1,20 @@
+from typing import Optional
+
+from eval_entity_resolver.normalization import normalize
+
+
+def normalized_match(
+    raw_value: str,
+    entity_type: str,
+    alias_store,
+    source_config: Optional[str] = None,
+) -> Optional[str]:
+    """Normalize input and look up against normalized alias index.
+
+    When ``source_config`` is given, scoped aliases for that config are
+    considered in addition to global aliases; otherwise scoped aliases are
+    excluded.
+    """
+    norm = normalize(raw_value)
+    lookup = alias_store.get_normalized_lookup(entity_type, source_config)
+    return lookup.get(norm)

pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "eval-card-registry"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi>=0.111.0",
+    "uvicorn[standard]>=0.29.0",
+    "typer>=0.12.0",
+    "pydantic>=2.7.0",
+    "pydantic-settings>=2.2.0",
+    "pandas>=2.2.0",
+    "pyarrow>=16.0.0",
+    "huggingface-hub>=0.23.0",
+    "datasets>=2.19.0",
+    "pyyaml>=6.0",
+    "eval-entity-resolver",
+]
+
+[project.scripts]
+eval-card-registry = "eval_card_registry.cli:app"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/eval_card_registry"]
+
+[tool.uv.workspace]
+members = ["packages/eval-entity-resolver"]
+
+[tool.uv.sources]
+eval-entity-resolver = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "httpx>=0.27.0",
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests", "packages/eval-entity-resolver/tests"]

src/eval_card_registry/api/routes_aliases.py

@@ -0,0 +1,42 @@
+"""Alias management routes — v0-defer (read + patch only; no review UI yet)."""
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from eval_card_registry.api.schemas import AliasPatch, AliasStatus, EntityType
+from eval_card_registry.config import settings
+from eval_card_registry.store.hf_store import get_store, RegistryStore
+from eval_card_registry.store import queries
+
+
+def _check_writable():
+    if settings.read_only:
+        raise HTTPException(status_code=405, detail="Write operations disabled in read-only mode")
+
+
+router = APIRouter()
+_writable = [Depends(_check_writable)]
+
+
+@router.get("/aliases")
+def list_aliases(
+    status: Optional[AliasStatus] = None,
+    entity_type: Optional[EntityType] = None,
+    source_config: Optional[str] = None,
+    store: RegistryStore = Depends(get_store),
+):
+    return queries.list_entities(
+        store,
+        "aliases",
+        review_status=None,
+        **{k: v for k, v in {"status": status, "entity_type": entity_type, "source_config": source_config}.items() if v is not None},
+    )
+
+
+@router.patch("/aliases/{alias_id}", dependencies=_writable)
+def patch_alias(alias_id: str, body: AliasPatch, store: RegistryStore = Depends(get_store)):
+    updates = {k: v for k, v in body.model_dump().items() if v is not None}
+    result = queries.update_alias(store, alias_id, updates)
+    if result is None:
+        raise HTTPException(status_code=404, detail=f"Alias '{alias_id}' not found")
+    return result

src/eval_card_registry/api/routes_entities.py

@@ -0,0 +1,183 @@
+import json
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from eval_card_registry.api.schemas import (
+    BenchmarkCreate, BenchmarkPatch,
+    HarnessCreate, HarnessPatch,
+    MetricCreate, MetricPatch,
+    ModelCreate, ModelPatch,
+    ReviewStatus,
+)
+from eval_card_registry.config import settings
+from eval_card_registry.store.hf_store import get_store, RegistryStore
+from eval_card_registry.store import queries
+
+
+def _check_writable():
+    if settings.read_only:
+        raise HTTPException(status_code=405, detail="Write operations disabled in read-only mode")
+
+router = APIRouter()
+_writable = [Depends(_check_writable)]
+
+
+def _get_or_404(store, table, entity_id):
+    entity = queries.get_entity(store, table, entity_id)
+    if entity is None:
+        raise HTTPException(status_code=404, detail=f"{table} '{entity_id}' not found")
+    return entity
+
+
+_JSON_FIELDS = {"tags", "metadata"}
+
+
+def _encode(data: dict) -> dict:
+    """JSON-encode list/dict fields for parquet storage."""
+    out = {}
+    for k, v in data.items():
+        if k in _JSON_FIELDS and isinstance(v, (list, dict)):
+            out[k] = json.dumps(v)
+        else:
+            out[k] = v
+    return out
+
+
+def _decode(entity: dict) -> dict:
+    """JSON-decode string fields that should be list/dict in API responses."""
+    out = {}
+    for k, v in entity.items():
+        if k in _JSON_FIELDS and isinstance(v, str):
+            try:
+                out[k] = json.loads(v)
+            except (json.JSONDecodeError, TypeError):
+                out[k] = v
+        else:
+            out[k] = v
+    return out
+
+
+# ------------------------------------------------------------------
+# Models
+# ------------------------------------------------------------------
+
+@router.get("/models")
+def list_models(
+    search: Optional[str] = None,
+    developer: Optional[str] = None,
+    review_status: Optional[ReviewStatus] = None,
+    store: RegistryStore = Depends(get_store),
+):
+    return [_decode(e) for e in queries.list_entities(store, "canonical_models", search=search, review_status=review_status, developer=developer)]
+
+
+@router.get("/models/{model_id:path}")
+def get_model(model_id: str, store: RegistryStore = Depends(get_store)):
+    return _decode(_get_or_404(store, "canonical_models", model_id))
+
+
+@router.post("/models", status_code=201, dependencies=_writable)
+def create_model(body: ModelCreate, store: RegistryStore = Depends(get_store)):
+    return _decode(queries.upsert_entity(store, "canonical_models", _encode(body.model_dump())))
+
+
+@router.patch("/models/{model_id:path}", dependencies=_writable)
+def patch_model(model_id: str, body: ModelPatch, store: RegistryStore = Depends(get_store)):
+    _get_or_404(store, "canonical_models", model_id)
+    data = {k: v for k, v in body.model_dump().items() if v is not None}
+    data["id"] = model_id
+    return _decode(queries.upsert_entity(store, "canonical_models", _encode(data)))
+
+
+# ------------------------------------------------------------------
+# Benchmarks
+# ------------------------------------------------------------------
+
+@router.get("/benchmarks")
+def list_benchmarks(
+    search: Optional[str] = None,
+    review_status: Optional[ReviewStatus] = None,
+    store: RegistryStore = Depends(get_store),
+):
+    return [_decode(e) for e in queries.list_entities(store, "canonical_benchmarks", search=search, review_status=review_status)]
+
+
+@router.get("/benchmarks/{benchmark_id}")
+def get_benchmark(benchmark_id: str, store: RegistryStore = Depends(get_store)):
+    return _decode(_get_or_404(store, "canonical_benchmarks", benchmark_id))
+
+
+@router.post("/benchmarks", status_code=201, dependencies=_writable)
+def create_benchmark(body: BenchmarkCreate, store: RegistryStore = Depends(get_store)):
+    return _decode(queries.upsert_entity(store, "canonical_benchmarks", _encode(body.model_dump())))
+
+
+@router.patch("/benchmarks/{benchmark_id}", dependencies=_writable)
+def patch_benchmark(benchmark_id: str, body: BenchmarkPatch, store: RegistryStore = Depends(get_store)):
+    _get_or_404(store, "canonical_benchmarks", benchmark_id)
+    data = {k: v for k, v in body.model_dump().items() if v is not None}
+    data["id"] = benchmark_id
+    return _decode(queries.upsert_entity(store, "canonical_benchmarks", _encode(data)))
+
+
+# ------------------------------------------------------------------
+# Metrics
+# ------------------------------------------------------------------
+
+@router.get("/metrics")
+def list_metrics(
+    search: Optional[str] = None,
+    review_status: Optional[ReviewStatus] = None,
+    store: RegistryStore = Depends(get_store),
+):
+    return [_decode(e) for e in queries.list_entities(store, "canonical_metrics", search=search, review_status=review_status)]
+
+
+@router.get("/metrics/{metric_id}")
+def get_metric(metric_id: str, store: RegistryStore = Depends(get_store)):
+    return _decode(_get_or_404(store, "canonical_metrics", metric_id))
+
+
+@router.post("/metrics", status_code=201, dependencies=_writable)
+def create_metric(body: MetricCreate, store: RegistryStore = Depends(get_store)):
+    return _decode(queries.upsert_entity(store, "canonical_metrics", _encode(body.model_dump())))
+
+
+@router.patch("/metrics/{metric_id}", dependencies=_writable)
+def patch_metric(metric_id: str, body: MetricPatch, store: RegistryStore = Depends(get_store)):
+    _get_or_404(store, "canonical_metrics", metric_id)
+    data = {k: v for k, v in body.model_dump().items() if v is not None}
+    data["id"] = metric_id
+    return _decode(queries.upsert_entity(store, "canonical_metrics", _encode(data)))
+
+
+# ------------------------------------------------------------------
+# Harnesses
+# ------------------------------------------------------------------
+
+@router.get("/harnesses")
+def list_harnesses(
+    search: Optional[str] = None,
+    review_status: Optional[ReviewStatus] = None,
+    store: RegistryStore = Depends(get_store),
+):
+    return [_decode(e) for e in queries.list_entities(store, "eval_harnesses", search=search, review_status=review_status)]
+
+
+@router.get("/harnesses/{harness_id}")
+def get_harness(harness_id: str, store: RegistryStore = Depends(get_store)):
+    return _decode(_get_or_404(store, "eval_harnesses", harness_id))
+
+
+@router.post("/harnesses", status_code=201, dependencies=_writable)
+def create_harness(body: HarnessCreate, store: RegistryStore = Depends(get_store)):
+    return _decode(queries.upsert_entity(store, "eval_harnesses", _encode(body.model_dump())))
+
+
+@router.patch("/harnesses/{harness_id}", dependencies=_writable)
+def patch_harness(harness_id: str, body: HarnessPatch, store: RegistryStore = Depends(get_store)):
+    _get_or_404(store, "eval_harnesses", harness_id)
+    data = {k: v for k, v in body.model_dump().items() if v is not None}
+    data["id"] = harness_id
+    return _decode(queries.upsert_entity(store, "eval_harnesses", _encode(data)))

src/eval_card_registry/api/routes_health.py

@@ -0,0 +1,47 @@
+from fastapi import APIRouter, Depends
+
+from eval_card_registry.store.hf_store import get_store, RegistryStore
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health(store: RegistryStore = Depends(get_store)):
+    return {
+        "status": "ok",
+        "store": "loaded" if store.loaded else "not_loaded",
+        "entities": {
+            "models": len(store.table("canonical_models")) if store.has_table("canonical_models") else 0,
+            "benchmarks": len(store.table("canonical_benchmarks")) if store.has_table("canonical_benchmarks") else 0,
+            "metrics": len(store.table("canonical_metrics")) if store.has_table("canonical_metrics") else 0,
+            "harnesses": len(store.table("eval_harnesses")) if store.has_table("eval_harnesses") else 0,
+        },
+    }
+
+
+@router.get("/stats")
+def stats(store: RegistryStore = Depends(get_store)):
+    def _counts(table: str) -> dict:
+        if not store.has_table(table):
+            return {"total": 0, "draft": 0, "reviewed": 0}
+        df = store.table(table)
+        total = len(df)
+        draft = int((df["review_status"] == "draft").sum()) if "review_status" in df.columns else 0
+        return {"total": total, "draft": draft, "reviewed": total - draft}
+
+    if store.has_table("aliases"):
+        aliases_df = store.table("aliases")
+        uncertain = int((aliases_df["status"] == "uncertain").sum()) if "status" in aliases_df.columns else 0
+        aliases_stats = {"total": len(aliases_df), "uncertain": uncertain}
+    else:
+        aliases_stats = {"total": 0, "uncertain": 0}
+
+    return {
+        "models": _counts("canonical_models"),
+        "benchmarks": _counts("canonical_benchmarks"),
+        "metrics": _counts("canonical_metrics"),
+        "harnesses": _counts("eval_harnesses"),
+        "aliases": aliases_stats,
+        "resolution_log": {"total": len(store.table("resolution_log")) if store.has_table("resolution_log") else 0},
+        "sync_runs": {"total": len(store.table("sync_runs")) if store.has_table("sync_runs") else 0},
+    }

src/eval_card_registry/api/routes_resolve.py

@@ -0,0 +1,72 @@
+import uuid
+
+from fastapi import APIRouter, Depends, Request
+from datetime import datetime, timezone
+
+from eval_card_registry.api.schemas import ResolveRequest, ResolveResponse
+from eval_card_registry.services.resolution_service import ResolutionService
+from eval_card_registry.services.log_writer import ResolveLogWriter
+
+router = APIRouter()
+
+
+def _svc(request: Request) -> ResolutionService:
+    return request.app.state.resolution_service
+
+
+def _log_writer(request: Request) -> ResolveLogWriter:
+    return request.app.state.log_writer
+
+
+def _log_resolve(
+    log_writer: ResolveLogWriter,
+    request_id: str,
+    req: ResolveRequest,
+    result: dict,
+) -> None:
+    log_writer.append({
+        "request_id": request_id,
+        "raw_value": req.raw_value,
+        "entity_type": req.entity_type,
+        "source_config": req.source_config,
+        "canonical_id": result.get("canonical_id"),
+        "strategy": result.get("strategy"),
+        "confidence": result.get("confidence"),
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    })
+
+
+@router.post("/resolve", response_model=ResolveResponse)
+def resolve(
+    req: ResolveRequest,
+    svc: ResolutionService = Depends(_svc),
+    log_writer: ResolveLogWriter = Depends(_log_writer),
+):
+    result = svc.resolve(
+        raw_value=req.raw_value,
+        entity_type=req.entity_type,
+        source_config=req.source_config,
+        source_field=req.source_field,
+    )
+    _log_resolve(log_writer, str(uuid.uuid4()), req, result)
+    return ResolveResponse(**result)
+
+
+@router.post("/resolve/batch", response_model=list[ResolveResponse])
+def resolve_batch(
+    reqs: list[ResolveRequest],
+    svc: ResolutionService = Depends(_svc),
+    log_writer: ResolveLogWriter = Depends(_log_writer),
+):
+    request_id = str(uuid.uuid4())
+    responses = []
+    for r in reqs:
+        result = svc.resolve(
+            raw_value=r.raw_value,
+            entity_type=r.entity_type,
+            source_config=r.source_config,
+            source_field=r.source_field,
+        )
+        _log_resolve(log_writer, request_id, r, result)
+        responses.append(ResolveResponse(**result))
+    return responses

src/eval_card_registry/api/schemas.py

@@ -0,0 +1,116 @@
+from typing import Any, Literal, Optional
+from pydantic import BaseModel
+
+
+EntityType = Literal["benchmark", "model", "metric", "harness"]
+ReviewStatus = Literal["draft", "reviewed"]
+AliasStatus = Literal["auto", "uncertain", "confirmed", "rejected"]
+
+
+# --- Resolve ---
+
+class ResolveRequest(BaseModel):
+    raw_value: str
+    entity_type: EntityType
+    source_config: Optional[str] = None
+    source_field: Optional[str] = None
+
+
+class ResolveResponse(BaseModel):
+    canonical_id: Optional[str]
+    strategy: str
+    confidence: float
+    created_new: bool
+    review_status: Optional[str]
+
+
+# --- Entities ---
+
+class ModelCreate(BaseModel):
+    id: str
+    display_name: str
+    developer: Optional[str] = None
+    family: Optional[str] = None
+    architecture: Optional[str] = None
+    params_billions: Optional[float] = None
+    tags: list[str] = []
+    metadata: dict[str, Any] = {}
+    review_status: str = "draft"
+
+
+class ModelPatch(BaseModel):
+    display_name: Optional[str] = None
+    developer: Optional[str] = None
+    family: Optional[str] = None
+    architecture: Optional[str] = None
+    params_billions: Optional[float] = None
+    tags: Optional[list[str]] = None
+    metadata: Optional[dict[str, Any]] = None
+    review_status: Optional[str] = None
+
+
+class BenchmarkCreate(BaseModel):
+    id: str
+    display_name: str
+    description: Optional[str] = None
+    dataset_repo: Optional[str] = None
+    parent_benchmark_id: Optional[str] = None
+    tags: list[str] = []
+    metadata: dict[str, Any] = {}
+    review_status: str = "draft"
+
+
+class BenchmarkPatch(BaseModel):
+    display_name: Optional[str] = None
+    description: Optional[str] = None
+    dataset_repo: Optional[str] = None
+    parent_benchmark_id: Optional[str] = None
+    tags: Optional[list[str]] = None
+    metadata: Optional[dict[str, Any]] = None
+    review_status: Optional[str] = None
+
+
+class MetricCreate(BaseModel):
+    id: str
+    display_name: str
+    score_type: Optional[str] = None
+    lower_is_better: bool = False
+    min_score: Optional[float] = None
+    max_score: Optional[float] = None
+    metadata: dict[str, Any] = {}
+    review_status: str = "draft"
+
+
+class MetricPatch(BaseModel):
+    display_name: Optional[str] = None
+    score_type: Optional[str] = None
+    lower_is_better: Optional[bool] = None
+    min_score: Optional[float] = None
+    max_score: Optional[float] = None
+    metadata: Optional[dict[str, Any]] = None
+    review_status: Optional[str] = None
+
+
+class HarnessCreate(BaseModel):
+    id: str
+    display_name: str
+    version: Optional[str] = None
+    fork_url: Optional[str] = None
+    metadata: dict[str, Any] = {}
+    review_status: str = "draft"
+
+
+class HarnessPatch(BaseModel):
+    display_name: Optional[str] = None
+    version: Optional[str] = None
+    fork_url: Optional[str] = None
+    metadata: Optional[dict[str, Any]] = None
+    review_status: Optional[str] = None
+
+
+# --- Aliases ---
+
+class AliasPatch(BaseModel):
+    canonical_id: Optional[str] = None
+    status: Optional[str] = None
+    notes: Optional[str] = None

src/eval_card_registry/cli.py

@@ -0,0 +1,259 @@
+"""
+eval-card-registry CLI.
+
+Commands:
+  seed      Load known entities from seed/ YAML files
+  stats     Print registry summary
+  sync      Batch sync one or all EEE configs → eval_results table
+"""
+from pathlib import Path
+from typing import Optional
+
+import typer
+import yaml
+
+from eval_card_registry.store.hf_store import get_store
+from eval_card_registry.store import queries
+
+app = typer.Typer(help="eval-card-registry CLI")
+
+
+def _load_store():
+    store = get_store()
+    if not store.loaded:
+        store.load()
+    return store
+
+
+# ------------------------------------------------------------------
+# seed
+# ------------------------------------------------------------------
+
+@app.command()
+def seed(
+    local: bool = typer.Option(False, "--local", help="Write to fixtures/ instead of HF Hub"),
+    seed_dir: str = typer.Option("./seed", "--seed-dir"),
+):
+    """Load known canonical entities from seed YAML files."""
+    import os
+    if local:
+        os.environ["LOCAL_MODE"] = "true"
+
+    store = _load_store()
+    seed_path = Path(seed_dir)
+
+    # table name, yaml file, label, entity_type (for alias creation)
+    seed_specs = [
+        ("canonical_benchmarks", seed_path / "benchmarks.yaml", "benchmarks", "benchmark"),
+        ("canonical_metrics", seed_path / "metrics.yaml", "metrics", "metric"),
+        ("eval_harnesses", seed_path / "harnesses.yaml", "harnesses", "harness"),
+    ]
+
+    alias_count = 0
+    # Track all seed entity IDs and alias keys so we can remove stale ones.
+    # Alias key: (raw_value, entity_type, canonical_id, source_config)
+    seed_snapshot: list[tuple[str, str, set[str], set[tuple[str, str, str, Optional[str]]]]] = []
+
+    for table, yaml_file, label, entity_type in seed_specs:
+        if not yaml_file.exists():
+            typer.echo(f"  [skip] {yaml_file} not found")
+            continue
+        with open(yaml_file) as f:
+            items = yaml.safe_load(f) or []
+
+        yaml_ids: set[str] = set()
+        yaml_alias_keys: set[tuple[str, str, str, Optional[str]]] = set()
+
+        for item in items:
+            # Pop 'aliases' / 'scoped_aliases' before upserting — not table columns.
+            extra_aliases = item.pop("aliases", []) or []
+            scoped_aliases = item.pop("scoped_aliases", {}) or {}
+            queries.upsert_entity(store, table, item)
+            canonical_id = item["id"]
+            display_name = item.get("display_name", "")
+            yaml_ids.add(canonical_id)
+
+            # Global aliases (source_config=None): matched regardless of caller's source_config.
+            # Scoped aliases (source_config=<name>): matched only when the caller passes that
+            # source_config — lets short tokens ("Overall", "Arabic") map to different
+            # benchmarks depending on which EEE config they came from.
+            global_aliases = {canonical_id, display_name} | set(extra_aliases)
+
+            alias_specs: list[tuple[str, Optional[str]]] = [
+                (raw, None) for raw in global_aliases if raw
+            ]
+            for source_cfg, raw_values in scoped_aliases.items():
+                for raw in raw_values or []:
+                    if raw:
+                        alias_specs.append((raw, source_cfg))
+
+            for raw_value, source_cfg in alias_specs:
+                # Index stale-removal by (raw_value, entity_type, canonical_id, source_config)
+                yaml_alias_keys.add((raw_value, entity_type, canonical_id, source_cfg))
+                try:
+                    queries.add_alias(store, {
+                        "raw_value": raw_value,
+                        "entity_type": entity_type,
+                        "canonical_id": canonical_id,
+                        "source_config": source_cfg,
+                        "source_field": "seed",
+                        "status": "confirmed",
+                        "strategy": "seed",
+                        "confidence": 1.0,
+                        "notes": None,
+                    })
+                    alias_count += 1
+                except ValueError:
+                    pass  # alias already exists (e.g. re-seeding)
+
+        seed_snapshot.append((table, entity_type, yaml_ids, yaml_alias_keys))
+        typer.echo(f"  {label}: {len(items)}")
+
+    # Remove seed-originated entities and aliases that are no longer in the YAML.
+    # Only touches rows that were created by seed (strategy == "seed"), never
+    # sync-created aliases or auto-draft entities.
+    removed_entities = 0
+    removed_aliases = 0
+    for table, entity_type, yaml_ids, yaml_alias_keys in seed_snapshot:
+        # Remove stale seed aliases for this entity type
+        aliases_df = store.table("aliases")
+        seed_mask = (aliases_df["strategy"] == "seed") & (aliases_df["entity_type"] == entity_type)
+        if seed_mask.any():
+            seed_aliases = aliases_df[seed_mask]
+            stale_alias_mask = seed_mask.copy()
+            for idx in seed_aliases.index:
+                row = seed_aliases.loc[idx]
+                sc = row.get("source_config")
+                # pandas NA → None so comparison matches how aliases were added.
+                if sc is None or (isinstance(sc, float) and sc != sc) or (hasattr(sc, "__class__") and sc.__class__.__name__ == "NAType"):
+                    sc = None
+                key = (row["raw_value"], row["entity_type"], row["canonical_id"], sc)
+                if key in yaml_alias_keys:
+                    stale_alias_mask[idx] = False
+            n_stale = stale_alias_mask.sum()
+            if n_stale > 0:
+                store.set_table("aliases", aliases_df[~stale_alias_mask].reset_index(drop=True))
+                removed_aliases += int(n_stale)
+
+        # Remove stale seed entities — only those with review_status "reviewed"
+        # that came from seed and are no longer in the YAML.
+        entity_df = store.table(table)
+        if len(entity_df) > 0:
+            stale = entity_df["id"].isin(yaml_ids)
+            stale_entities = entity_df[~stale & (entity_df["review_status"] == "reviewed")]
+            # Only remove if every alias for this entity is also seed-originated,
+            # meaning it wasn't referenced by sync data.
+            current_aliases = store.table("aliases")
+            for eid in stale_entities["id"]:
+                entity_aliases = current_aliases[
+                    (current_aliases["canonical_id"] == eid)
+                    & (current_aliases["entity_type"] == entity_type)
+                ]
+                if len(entity_aliases) == 0 or (entity_aliases["strategy"] == "seed").all():
+                    entity_df = entity_df[entity_df["id"] != eid]
+                    # Also remove any remaining aliases pointing to it
+                    current_aliases = current_aliases[
+                        ~((current_aliases["canonical_id"] == eid)
+                          & (current_aliases["entity_type"] == entity_type))
+                    ]
+                    removed_entities += 1
+            store.set_table(table, entity_df.reset_index(drop=True))
+            store.set_table("aliases", current_aliases.reset_index(drop=True))
+
+    typer.echo(f"  aliases: {alias_count} added, {removed_aliases} removed")
+    if removed_entities:
+        typer.echo(f"  stale entities removed: {removed_entities}")
+
+    store.push_to_hub()
+    typer.echo("Seed complete.")
+
+
+# ------------------------------------------------------------------
+# stats
+# ------------------------------------------------------------------
+
+@app.command()
+def stats(
+    local: bool = typer.Option(False, "--local", help="Read from fixtures/ instead of HF Hub"),
+):
+    """Print registry entity counts and pending review summary."""
+    import os
+    if local:
+        os.environ["LOCAL_MODE"] = "true"
+
+    store = _load_store()
+
+    def _row(table):
+        df = store.table(table)
+        total = len(df)
+        draft = int((df["review_status"] == "draft").sum()) if "review_status" in df.columns else 0
+        return total, draft
+
+    for label, table in [
+        ("models    ", "canonical_models"),
+        ("benchmarks", "canonical_benchmarks"),
+        ("metrics   ", "canonical_metrics"),
+        ("harnesses ", "eval_harnesses"),
+    ]:
+        total, draft = _row(table)
+        typer.echo(f"  {label}  total={total}  draft={draft}")
+
+    aliases_df = store.table("aliases")
+    uncertain = int((aliases_df["status"] == "uncertain").sum()) if "status" in aliases_df.columns else 0
+    typer.echo(f"\n  aliases        total={len(aliases_df)}  uncertain={uncertain}")
+    typer.echo(f"  eval_results   total={len(store.table('eval_results'))}")
+    typer.echo(f"  resolution_log total={len(store.table('resolution_log'))}")
+    typer.echo(f"  sync_runs      total={len(store.table('sync_runs'))}")
+
+
+# ------------------------------------------------------------------
+# sync
+# ------------------------------------------------------------------
+
+@app.command()
+def sync(
+    config: Optional[str] = typer.Option(None, "--config", help="EEE config name"),
+    all_configs: bool = typer.Option(False, "--all", help="Sync all EEE configs"),
+    rerun: bool = typer.Option(False, "--rerun", help="Re-resolve all raw strings even if already aliased"),
+    local: bool = typer.Option(False, "--local"),
+):
+    """
+    Batch sync EEE config(s) → writes resolved results to eval_results table.
+    Each result row is one (model × benchmark × metric) combination with resolved canonical IDs.
+    """
+    import os
+    if local:
+        os.environ["LOCAL_MODE"] = "true"
+
+    if not config and not all_configs:
+        typer.echo("Specify --config <name> or --all", err=True)
+        raise typer.Exit(1)
+
+    from eval_card_registry.services.ingestion import run_sync
+    import datasets as ds_lib
+
+    store = _load_store()
+
+    configs_to_run: list[str] = []
+    if all_configs:
+        configs_to_run = ds_lib.get_dataset_config_names("evaleval/EEE_datastore")
+    else:
+        configs_to_run = [config]
+
+    failed = []
+    for cfg in configs_to_run:
+        typer.echo(f"Syncing {cfg}...")
+        try:
+            counts = run_sync(cfg, store, rerun=rerun)
+            typer.echo(f"  {cfg}: {counts}")
+        except Exception as e:
+            typer.echo(f"  {cfg}: FAILED — {e}", err=True)
+            failed.append(cfg)
+
+    typer.echo("Persisting tables...")
+    store.push_to_hub()
+
+    if failed:
+        typer.echo(f"Done with {len(failed)} failed config(s): {', '.join(failed)}")
+    else:
+        typer.echo("Done.")

src/eval_card_registry/config.py

@@ -0,0 +1,25 @@
+import os
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    local_mode: bool = False
+    fixtures_path: str = "./fixtures"
+    hf_dataset_repo: str = ""
+    hf_token: str = ""
+    resolver_auto_merge_threshold: float = 0.85
+    read_only: bool = False
+    hf_log_bucket: str = ""
+    log_flush_interval_seconds: int = 300
+
+
+settings = Settings()
+
+# Export HF_TOKEN to the environment so that libraries that read it directly
+# (e.g. `datasets.load_dataset`) pick it up, not just code that uses
+# `settings.hf_token`.
+if settings.hf_token and not os.environ.get("HF_TOKEN"):
+    os.environ["HF_TOKEN"] = settings.hf_token

src/eval_card_registry/main.py

@@ -0,0 +1,48 @@
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from eval_card_registry.config import settings
+from eval_card_registry.store.hf_store import get_store, _QUERY_TABLE_NAMES
+from eval_card_registry.services.resolution_service import ResolutionService
+from eval_card_registry.services.log_writer import ResolveLogWriter
+from eval_card_registry.api.routes_resolve import router as resolve_router
+from eval_card_registry.api.routes_entities import router as entities_router
+from eval_card_registry.api.routes_aliases import router as aliases_router
+from eval_card_registry.api.routes_health import router as health_router
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    store = get_store()
+    if settings.read_only:
+        store.load(tables=_QUERY_TABLE_NAMES)
+    else:
+        store.load()
+
+    # Singleton ResolutionService — avoids rebuilding AliasStore per request
+    app.state.resolution_service = ResolutionService(store)
+
+    # Resolve log writer
+    log_writer = ResolveLogWriter(settings.hf_log_bucket)
+    app.state.log_writer = log_writer
+    log_writer.start(settings.log_flush_interval_seconds)
+
+    yield
+
+    await log_writer.stop()
+
+
+app = FastAPI(
+    title="eval-card-registry",
+    description="Entity resolution registry for EEE evaluation data.",
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+PREFIX = "/api/v1"
+
+app.include_router(resolve_router, prefix=PREFIX)
+app.include_router(entities_router, prefix=PREFIX)
+app.include_router(aliases_router, prefix=PREFIX)
+app.include_router(health_router, prefix=PREFIX)

src/eval_card_registry/services/ingestion.py

@@ -0,0 +1,299 @@
+"""
+EEE sync pipeline.
+
+Per-record processing → sub-category detection → resolution → eval_results table.
+Pushes HF Hub at end of run.
+
+Known data-quality issues
+---------------------------------------------------
+Some configs use a metric keyword as ``evaluation_name`` for aggregate /
+summary rows instead of a real benchmark name.  These create problematic benchmark
+entities (``score``, ``overall``, ``mean-score``, ``mean-win-rate``).
+
+* ``reward-bench``: ``"Score"`` — composite metric across rewardbench
+  subcategories (chat, chat-hard, safety, reasoning).
+* ``ace``: ``"Overall"`` / ``"overall"`` — rollup across ACE/APEX
+  sub-evaluations.
+* ``helm_capabilities``: ``"Mean score"`` — mean across all HELM capability
+  benchmarks for a model.
+* ``helm_instruct``: ``"Mean win rate"`` — mean win rate across all HELM
+  instruct benchmarks.
+
+Fix should happen upstream as part of wider discussion on schema design.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from collections import defaultdict
+from typing import Any, Iterator, Optional
+
+from eval_entity_resolver.eee import clean_eval_name, extract_metric
+
+from eval_card_registry.config import settings
+from eval_card_registry.store.hf_store import RegistryStore
+from eval_card_registry.store import queries
+from eval_card_registry.services.resolution_service import ResolutionService
+
+
+def _iter_eee_config(source_config: str) -> Iterator[dict]:
+    """Load EEE datastore config and yield rows as dicts."""
+    import datasets
+
+    ds = datasets.load_dataset("evaleval/EEE_datastore", source_config)["train"]
+    for i in range(len(ds)):
+        yield dict(ds[i])
+
+
+def _detect_sub_categories(
+    evaluation_results: list[dict],
+) -> dict[str, Optional[str]]:
+    """
+    Inspect all evaluation_results in a record.
+    Returns {evaluation_name: parent_dataset_name or None}.
+
+    If multiple evaluation_name values share the same dataset_name → sub-category pattern.
+    The dataset_name becomes the parent benchmark; each evaluation_name is a child.
+    """
+    dataset_to_eval_names: dict[str, list[str]] = defaultdict(list)
+    for result in evaluation_results:
+        eval_name = result.get("evaluation_name", "")
+        dataset_name = (result.get("source_data") or {}).get("dataset_name", "")
+        if eval_name and dataset_name:
+            dataset_to_eval_names[dataset_name].append(eval_name)
+
+    # Build mapping: eval_name → parent dataset_name (if sub-category) or None
+    eval_to_parent: dict[str, Optional[str]] = {}
+    for result in evaluation_results:
+        eval_name = result.get("evaluation_name", "")
+        dataset_name = (result.get("source_data") or {}).get("dataset_name", "")
+        if not eval_name:
+            continue
+        if dataset_name and len(dataset_to_eval_names.get(dataset_name, [])) > 1:
+            eval_to_parent[eval_name] = dataset_name
+        else:
+            eval_to_parent[eval_name] = None
+    return eval_to_parent
+
+
+def process_record(
+    record: dict,
+    source_config: str,
+    svc: ResolutionService,
+    sync_run_id: str,
+    rerun: bool,
+) -> Optional[list[dict]]:
+    """
+    Resolve all entities in one EEE record.
+    Returns a flat list of result rows (one per evaluation result) or None if
+    the record has no model info.
+    """
+    # --- Model ---
+    model_info = record.get("model_info") or {}
+    model_raw = model_info.get("id") or model_info.get("model_name") or model_info.get("name")
+    if not model_raw:
+        return None
+
+    model_res = svc.resolve(
+        raw_value=model_raw,
+        entity_type="model",
+        source_config=source_config,
+        source_field="model_info.id",
+        sync_run_id=sync_run_id,
+        rerun=rerun,
+    )
+
+    # --- Harness ---
+    eval_lib = record.get("eval_library") or {}
+    harness_raw = eval_lib.get("name") or eval_lib.get("library_name")
+    harness_res = None
+    if harness_raw:
+        harness_res = svc.resolve(
+            raw_value=harness_raw,
+            entity_type="harness",
+            source_config=source_config,
+            source_field="eval_library.name",
+            sync_run_id=sync_run_id,
+            rerun=rerun,
+        )
+
+    # --- Benchmarks & Metrics ---
+    evaluation_results = record.get("evaluation_results") or []
+    if not isinstance(evaluation_results, list):
+        evaluation_results = []
+
+    eval_to_parent = _detect_sub_categories(evaluation_results)
+
+    # Resolve parent benchmarks once
+    parent_cache: dict[str, str] = {}
+    for dataset_name in set(v for v in eval_to_parent.values() if v is not None):
+        parent_res = svc.resolve(
+            raw_value=dataset_name,
+            entity_type="benchmark",
+            source_config=source_config,
+            source_field="source_data.dataset_name",
+            sync_run_id=sync_run_id,
+            rerun=rerun,
+        )
+        parent_cache[dataset_name] = parent_res["canonical_id"]
+
+    # Build evaluation_id — must be deterministic for stable eval_results row keys.
+    # The EEE schema requires evaluation_id, but we fall back to a hash of
+    # model + source_config if missing, to avoid non-deterministic id(record).
+    source_meta = record.get("source_metadata") or {}
+    eval_id = record.get("evaluation_id") or source_meta.get("evaluation_id") or source_meta.get("id")
+    if not eval_id:
+        import hashlib
+        fallback_key = f"{source_config}:{model_raw}"
+        eval_id = f"{source_config}/auto-{hashlib.sha256(fallback_key.encode()).hexdigest()[:12]}"
+
+    result_rows = []
+    for idx, er in enumerate(evaluation_results):
+        eval_name = er.get("evaluation_name")
+        if not eval_name:
+            continue
+
+        parent_dataset = eval_to_parent.get(eval_name)
+        parent_benchmark_id = parent_cache.get(parent_dataset) if parent_dataset else None
+
+        bench_name = clean_eval_name(eval_name)
+        bench_res = svc.resolve(
+            raw_value=bench_name,
+            entity_type="benchmark",
+            source_config=source_config,
+            source_field="evaluation_results[].evaluation_name",
+            sync_run_id=sync_run_id,
+            rerun=rerun,
+        )
+
+        # Metric — try metric_name first (human-readable, e.g. "Win Rate"),
+        # then metric_id (may be dot-notation like "bfcl.live.accuracy"),
+        # then evaluation_description (verbose, e.g. "Accuracy on IFEval").
+        # extract_metric normalises all three forms to a reusable metric name.
+        metric_config = er.get("metric_config") or {}
+        metric_raw = extract_metric(
+            metric_config.get("metric_name")
+            or metric_config.get("metric_id")
+            or metric_config.get("evaluation_description")
+            or ""
+        )
+        metric_res = None
+        if metric_raw:
+            metric_res = svc.resolve(
+                raw_value=metric_raw,
+                entity_type="metric",
+                source_config=source_config,
+                source_field="metric_config",
+                sync_run_id=sync_run_id,
+                rerun=rerun,
+            )
+
+        # Score — use `is not None` checks to preserve valid 0 / 0.0 scores.
+        score_details_raw = er.get("score_details") or er.get("details") or {}
+        score = None
+        if isinstance(score_details_raw, dict):
+            score = score_details_raw.get("score")
+        if score is None:
+            for key in ("score", "value", "result"):
+                val = er.get(key)
+                if val is not None:
+                    score = val
+                    break
+
+        result_rows.append(
+            {
+                "evaluation_id": eval_id,
+                "result_index": idx,
+                "source_config": source_config,
+                "model_id": model_res["canonical_id"],
+                "harness_id": harness_res["canonical_id"] if harness_res else None,
+                "benchmark_id": bench_res["canonical_id"],
+                "parent_benchmark_id": parent_benchmark_id,
+                "metric_id": metric_res["canonical_id"] if metric_res else None,
+                "benchmark_card_id": None,
+                "score": score,
+                "score_details": json.dumps(score_details_raw) if score_details_raw else None,
+            }
+        )
+
+    return result_rows
+
+
+def run_sync(
+    source_config: str,
+    registry_store: RegistryStore,
+    rerun: bool = False,
+) -> dict:
+    """
+    Sync one EEE config. Returns counts dict.
+    Does NOT push to HF Hub — caller is responsible for calling push_to_hub()
+    once after all configs are done.
+    """
+    svc = ResolutionService(registry_store)
+
+    # Reset module-level caches from any prior (possibly crashed) sync
+    queries._alias_index.clear()
+    queries._pending_result_ids.clear()
+
+    # Build alias index for fast lookups during sync
+    queries._rebuild_alias_index(registry_store)
+
+    run_id = queries.start_sync_run(registry_store, source_config, rerun)
+
+    counts = {
+        "entities_created": 0,
+        "entities_updated": 0,
+        "aliases_created": 0,
+        "aliases_updated": 0,
+    }
+    errors = []
+
+    # Snapshot table lengths before sync to count actual changes
+    aliases_before = len(registry_store.table("aliases")) + len(queries._get_pending(registry_store, "aliases"))
+    models_before = len(registry_store.table("canonical_models"))
+    benchmarks_before = len(registry_store.table("canonical_benchmarks"))
+    metrics_before = len(registry_store.table("canonical_metrics"))
+    harnesses_before = len(registry_store.table("eval_harnesses"))
+
+    record_count = 0
+    try:
+        for record in _iter_eee_config(source_config):
+            try:
+                result_rows = process_record(record, source_config, svc, run_id, rerun)
+                if result_rows:
+                    for row in result_rows:
+                        queries.upsert_eval_result(registry_store, row)
+            except Exception as e:
+                errors.append(str(e))
+
+            record_count += 1
+            if record_count % 500 == 0:
+                print(f"  [{source_config}] {record_count} records processed...", file=sys.stderr)
+    finally:
+        # Always flush pending rows — even on crash, preserve successfully
+        # processed records rather than silently losing them.
+        queries.flush_pending(registry_store)
+
+        # Reset module-level caches for next sync run
+        queries._alias_index.clear()
+        queries._pending_result_ids.clear()
+
+    print(f"  [{source_config}] {record_count} records total, flushed.", file=sys.stderr)
+
+    # Count changes by comparing table lengths
+    entities_after = (
+        len(registry_store.table("canonical_models"))
+        + len(registry_store.table("canonical_benchmarks"))
+        + len(registry_store.table("canonical_metrics"))
+        + len(registry_store.table("eval_harnesses"))
+    )
+    entities_before = models_before + benchmarks_before + metrics_before + harnesses_before
+    aliases_after = len(registry_store.table("aliases"))
+
+    counts["entities_created"] = max(0, entities_after - entities_before)
+    counts["aliases_created"] = max(0, aliases_after - aliases_before) if not rerun else 0
+    counts["aliases_updated"] = max(0, aliases_after - aliases_before) if rerun else 0
+
+    queries.finish_sync_run(registry_store, run_id, counts, errors)
+
+    return counts