Spaces:
Paused
Paused
Onur Solmaz commited on
Commit ·
3d7768f
1
Parent(s): bab91f3
fix: resolve packaged component artifacts
Browse files- leaderboard_core.py +9 -6
- tests/test_leaderboard_core.py +44 -0
leaderboard_core.py
CHANGED
|
@@ -911,7 +911,7 @@ def base_model_refs(row: dict[str, Any]) -> list[tuple[str, str]]:
|
|
| 911 |
ref = value
|
| 912 |
if ":" in value:
|
| 913 |
prefix, rest = value.split(":", 1)
|
| 914 |
-
if prefix in {"
|
| 915 |
relation = prefix
|
| 916 |
ref = rest
|
| 917 |
if ref:
|
|
@@ -935,10 +935,12 @@ def packaging_base_refs(row: dict[str, Any]) -> list[str]:
|
|
| 935 |
relation = str(row.get("base_model_relation") or "").lower()
|
| 936 |
if relation and relation not in PACKAGING_RELATION_TERMS:
|
| 937 |
return unique_preserve_order(refs)
|
| 938 |
-
|
| 939 |
-
|
| 940 |
-
if ref_relation
|
| 941 |
-
|
|
|
|
|
|
|
| 942 |
refs.append(ref)
|
| 943 |
return unique_preserve_order(refs)
|
| 944 |
|
|
@@ -1204,9 +1206,10 @@ def group_models(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
| 1204 |
key = resolve_lineage_key(slug_dedupe_key(base), row, known_keys, key_representatives)
|
| 1205 |
break
|
| 1206 |
else:
|
|
|
|
| 1207 |
contained_key = contained_lineage_key(
|
| 1208 |
normalized_repo_slug(str(row["id"])),
|
| 1209 |
-
|
| 1210 |
row,
|
| 1211 |
key_representatives,
|
| 1212 |
)
|
|
|
|
| 911 |
ref = value
|
| 912 |
if ":" in value:
|
| 913 |
prefix, rest = value.split(":", 1)
|
| 914 |
+
if prefix in PACKAGING_RELATION_TERMS or prefix in {"finetune", "merge"}:
|
| 915 |
relation = prefix
|
| 916 |
ref = rest
|
| 917 |
if ref:
|
|
|
|
| 935 |
relation = str(row.get("base_model_relation") or "").lower()
|
| 936 |
if relation and relation not in PACKAGING_RELATION_TERMS:
|
| 937 |
return unique_preserve_order(refs)
|
| 938 |
+
for ref_relation, ref in base_model_refs(row):
|
| 939 |
+
if ref_relation:
|
| 940 |
+
if ref_relation in PACKAGING_RELATION_TERMS:
|
| 941 |
+
refs.append(ref)
|
| 942 |
+
continue
|
| 943 |
+
if relation in PACKAGING_RELATION_TERMS:
|
| 944 |
refs.append(ref)
|
| 945 |
return unique_preserve_order(refs)
|
| 946 |
|
|
|
|
| 1206 |
key = resolve_lineage_key(slug_dedupe_key(base), row, known_keys, key_representatives)
|
| 1207 |
break
|
| 1208 |
else:
|
| 1209 |
+
candidate_keys = known_keys - {key}
|
| 1210 |
contained_key = contained_lineage_key(
|
| 1211 |
normalized_repo_slug(str(row["id"])),
|
| 1212 |
+
candidate_keys,
|
| 1213 |
row,
|
| 1214 |
key_representatives,
|
| 1215 |
)
|
tests/test_leaderboard_core.py
CHANGED
|
@@ -167,6 +167,34 @@ class SlugDedupeTests(unittest.TestCase):
|
|
| 167 |
self.assertEqual(groups[0]["group_key"], "qwen3-8b")
|
| 168 |
self.assertIn("nvidia/KVzap-mlp-Qwen3-8B", groups[0]["artifacts"])
|
| 169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
def test_groups_small_assistant_component_into_base_lineage(self):
|
| 171 |
groups = group_models(
|
| 172 |
[
|
|
@@ -218,6 +246,22 @@ class SlugDedupeTests(unittest.TestCase):
|
|
| 218 |
self.assertEqual(groups[0]["group_key"], "deepseek-v4-flash")
|
| 219 |
self.assertIn("antirez/deepseek-v4-gguf", groups[0]["artifacts"])
|
| 220 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
def test_non_packaging_relation_does_not_group_bare_base_model_ref(self):
|
| 222 |
groups = group_models(
|
| 223 |
[
|
|
|
|
| 167 |
self.assertEqual(groups[0]["group_key"], "qwen3-8b")
|
| 168 |
self.assertIn("nvidia/KVzap-mlp-Qwen3-8B", groups[0]["artifacts"])
|
| 169 |
|
| 170 |
+
def test_groups_suffixed_component_artifact_into_base_lineage(self):
|
| 171 |
+
groups = group_models(
|
| 172 |
+
[
|
| 173 |
+
row("Qwen/Qwen3-8B", downloads=220_000, downloads_all_time=440_000, parameter_count_b=8),
|
| 174 |
+
row(
|
| 175 |
+
"nvidia/KVzap-mlp-Qwen3-8B",
|
| 176 |
+
downloads=50_000,
|
| 177 |
+
downloads_all_time=80_000,
|
| 178 |
+
parameter_count_b=0.076,
|
| 179 |
+
pipeline_tag="other",
|
| 180 |
+
tags="kvzap,nvidia,pytorch,other",
|
| 181 |
+
),
|
| 182 |
+
row(
|
| 183 |
+
"nvidia/KVzap-mlp-Qwen3-8B-GGUF",
|
| 184 |
+
downloads=40_000,
|
| 185 |
+
downloads_all_time=70_000,
|
| 186 |
+
parameter_count_b=0.076,
|
| 187 |
+
pipeline_tag="other",
|
| 188 |
+
has_gguf=True,
|
| 189 |
+
library_name="gguf",
|
| 190 |
+
tags="gguf",
|
| 191 |
+
),
|
| 192 |
+
]
|
| 193 |
+
)
|
| 194 |
+
self.assertEqual(len(groups), 1)
|
| 195 |
+
self.assertEqual(groups[0]["group_key"], "qwen3-8b")
|
| 196 |
+
self.assertIn("nvidia/KVzap-mlp-Qwen3-8B-GGUF", groups[0]["artifacts"])
|
| 197 |
+
|
| 198 |
def test_groups_small_assistant_component_into_base_lineage(self):
|
| 199 |
groups = group_models(
|
| 200 |
[
|
|
|
|
| 246 |
self.assertEqual(groups[0]["group_key"], "deepseek-v4-flash")
|
| 247 |
self.assertIn("antirez/deepseek-v4-gguf", groups[0]["artifacts"])
|
| 248 |
|
| 249 |
+
def test_packaging_relation_prefixes_group_without_card_relation(self):
|
| 250 |
+
groups = group_models(
|
| 251 |
+
[
|
| 252 |
+
row("Qwen/Qwen3-8B", downloads=220_000, downloads_all_time=440_000, parameter_count_b=8),
|
| 253 |
+
row(
|
| 254 |
+
"runtime/Qwen3-8B-Converted",
|
| 255 |
+
downloads=200_000,
|
| 256 |
+
downloads_all_time=300_000,
|
| 257 |
+
base_models="converted:Qwen/Qwen3-8B",
|
| 258 |
+
tags="base_model:converted:Qwen/Qwen3-8B",
|
| 259 |
+
),
|
| 260 |
+
]
|
| 261 |
+
)
|
| 262 |
+
self.assertEqual(len(groups), 1)
|
| 263 |
+
self.assertEqual(groups[0]["group_key"], "qwen3-8b")
|
| 264 |
+
|
| 265 |
def test_non_packaging_relation_does_not_group_bare_base_model_ref(self):
|
| 266 |
groups = group_models(
|
| 267 |
[
|