osolmaz commited on
Commit
5962efd
·
verified ·
1 Parent(s): 101f149

feat: rank by normalized engagement rates

Browse files
leaderboard_core.py CHANGED
@@ -131,8 +131,10 @@ GROUP_CSV_FIELDS = [
131
  "group_key",
132
  "representative_id",
133
  "author",
 
134
  "downloads",
135
  "downloads_all_time",
 
136
  "likes",
137
  "parameter_count_b",
138
  "capacity_class",
@@ -149,6 +151,10 @@ GROUP_CSV_FIELDS = [
149
  "last_modified",
150
  ]
151
 
 
 
 
 
152
  ARTIFACT_REVISION_SUFFIX_RE = re.compile(r"-(?:gguf|awq|gptq|fp8|fp4|mxfp8|nvfp4)-v[0-9]+$")
153
  ARTIFACT_SUFFIX_RE = re.compile(
154
  r"-(?:"
@@ -745,6 +751,40 @@ def group_models(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
745
  return groups
746
 
747
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
748
  def collect_metadata(
749
  *,
750
  creators: list[str],
@@ -819,10 +859,12 @@ def build_payload(
819
  )
820
  candidates = [row for row in popular if row.get("candidate") is True]
821
  groups = group_models(candidates)
 
 
822
  class_counts = Counter(group["candidate_class"] for group in groups)
823
  creator_counts = Counter(group["author"] for group in groups)
824
  summary = {
825
- "generated_at": iso(),
826
  "started_at": started,
827
  "source_model_records": len(normalized),
828
  "popular_model_records": len(popular),
@@ -864,6 +906,16 @@ def format_int(value: Any) -> str:
864
  return "0"
865
 
866
 
 
 
 
 
 
 
 
 
 
 
867
  def format_params(value: Any) -> str:
868
  if value in {"", None}:
869
  return "unknown"
@@ -900,26 +952,29 @@ def render_html(payload: dict[str, Any]) -> str:
900
  generated_js = json.dumps(str(payload["generated_at"]))
901
  default_age_days = str(filters.get("default_age_filter_days") or filters.get("max_age_days") or DEFAULT_MAX_AGE_DAYS)
902
  top_n_js = json.dumps(filters.get("top_n") or DEFAULT_TOP_N)
 
903
 
904
  def age_selected(value: str) -> str:
905
  return " selected" if value == default_age_days else ""
906
 
907
  table_rows = []
908
  for group in groups:
 
 
909
  related_repos = "".join(
910
  f'<a href="{html.escape(model_url(str(item)))}" target="_blank" rel="noreferrer">{html.escape(str(item))}</a>'
911
  for item in group["artifacts"]
912
  )
913
  table_rows.append(
914
  f"""
915
- <tr data-class="{html.escape(str(group['candidate_class']))}" data-creator="{html.escape(str(group['author']))}" data-created="{html.escape(str(group.get('created_at') or ''))}" data-downloads="{int(group.get('downloads') or 0)}" data-likes="{int(group.get('likes') or 0)}">
916
  <td class="rank" data-label="Rank">{group['rank']}</td>
917
  <td class="model-cell" data-label="Model">
918
  <a class="model-link" href="{html.escape(model_url(str(group['representative_id'])))}" target="_blank" rel="noreferrer">{html.escape(str(group['group_key']))}</a>
919
  <div class="model-sub">{html.escape(str(group['representative_id']))}</div>
920
  </td>
921
- <td class="num" data-label="Downloads">{format_int(group['downloads'])}</td>
922
- <td class="num" data-label="Likes">{format_int(group['likes'])}</td>
923
  <td data-label="Params">{format_params(group['parameter_count_b'])}</td>
924
  <td data-label="Created">{format_date(group.get('created_at'))}</td>
925
  <td class="artifacts" data-label="Related repos">
@@ -1011,8 +1066,8 @@ def render_html(payload: dict[str, Any]) -> str:
1011
  th {{ position: sticky; top: 0; background: var(--header); z-index: 1; color: var(--muted); font-size: 12px; text-transform: uppercase; }}
1012
  th:nth-child(1), td:nth-child(1) {{ width: 62px; }}
1013
  th:nth-child(2), td:nth-child(2) {{ width: 260px; }}
1014
- th:nth-child(3), td:nth-child(3) {{ width: 108px; }}
1015
- th:nth-child(4), td:nth-child(4) {{ width: 82px; }}
1016
  th:nth-child(5), td:nth-child(5) {{ width: 80px; }}
1017
  th:nth-child(6), td:nth-child(6) {{ width: 104px; }}
1018
  tbody tr:hover {{ background: var(--hover); }}
@@ -1022,6 +1077,7 @@ def render_html(payload: dict[str, Any]) -> str:
1022
  .model-link:hover {{ color: var(--accent); }}
1023
  .model-sub {{ margin-top: 4px; color: var(--muted); font-size: 12px; overflow-wrap: anywhere; }}
1024
  .num {{ text-align: right; white-space: nowrap; font-variant-numeric: tabular-nums; }}
 
1025
  .artifacts {{ min-width: 280px; color: var(--muted); overflow-wrap: anywhere; }}
1026
  .artifacts span {{ display: flex; flex-wrap: wrap; gap: 5px 10px; overflow-wrap: anywhere; word-break: break-word; }}
1027
  .artifacts a {{ color: var(--link); text-decoration: none; overflow-wrap: anywhere; word-break: break-word; }}
@@ -1098,8 +1154,8 @@ def render_html(payload: dict[str, Any]) -> str:
1098
  <div class="control">
1099
  <label for="sortFilter">Sort by</label>
1100
  <select id="sortFilter">
1101
- <option value="downloads">Downloads</option>
1102
- <option value="likes">Likes</option>
1103
  </select>
1104
  </div>
1105
  <div class="control">
@@ -1125,8 +1181,8 @@ def render_html(payload: dict[str, Any]) -> str:
1125
  <tr>
1126
  <th data-sort="num">Rank</th>
1127
  <th data-sort="text">Model group</th>
1128
- <th data-sort="num">Downloads</th>
1129
- <th data-sort="num">Likes</th>
1130
  <th data-sort="text">Params</th>
1131
  <th data-sort="text">Created</th>
1132
  <th data-sort="text">Related repos</th>
@@ -1154,7 +1210,7 @@ def render_html(payload: dict[str, Any]) -> str:
1154
  return ((generatedAt - createdAt) / dayMs) <= Number(maxDays);
1155
  }}
1156
  function sortRows() {{
1157
- const key = sortFilter.value === 'likes' ? 'likes' : 'downloads';
1158
  rows.sort((a, b) => {{
1159
  const bv = Number(b.dataset[key]) || 0;
1160
  const av = Number(a.dataset[key]) || 0;
 
131
  "group_key",
132
  "representative_id",
133
  "author",
134
+ "downloads_per_day",
135
  "downloads",
136
  "downloads_all_time",
137
+ "likes_per_year",
138
  "likes",
139
  "parameter_count_b",
140
  "capacity_class",
 
151
  "last_modified",
152
  ]
153
 
154
+ SECONDS_PER_DAY = 86_400
155
+ DAYS_PER_YEAR = 365.2425
156
+ MIN_AGE_DAYS = 1 / 24
157
+
158
  ARTIFACT_REVISION_SUFFIX_RE = re.compile(r"-(?:gguf|awq|gptq|fp8|fp4|mxfp8|nvfp4)-v[0-9]+$")
159
  ARTIFACT_SUFFIX_RE = re.compile(
160
  r"-(?:"
 
751
  return groups
752
 
753
 
754
+ def age_days(created_at: Any, as_of: datetime) -> float:
755
+ created = parse_hf_datetime(str(created_at or ""))
756
+ if created is None:
757
+ return MIN_AGE_DAYS
758
+ return max((as_of - created).total_seconds() / SECONDS_PER_DAY, MIN_AGE_DAYS)
759
+
760
+
761
+ def rate_metrics(group: dict[str, Any], as_of: datetime) -> dict[str, float]:
762
+ age = age_days(group.get("created_at"), as_of)
763
+ downloads = int(group.get("downloads") or 0)
764
+ likes = int(group.get("likes") or 0)
765
+ return {
766
+ "age_days": round(age, 4),
767
+ "downloads_per_day": round(downloads / age, 4),
768
+ "likes_per_year": round(likes * DAYS_PER_YEAR / age, 4),
769
+ }
770
+
771
+
772
+ def add_rate_metrics(groups: list[dict[str, Any]], as_of: datetime) -> None:
773
+ for group in groups:
774
+ group.update(rate_metrics(group, as_of))
775
+ groups.sort(
776
+ key=lambda row: (
777
+ float(row.get("downloads_per_day") or 0),
778
+ int(row.get("downloads") or 0),
779
+ float(row.get("likes_per_year") or 0),
780
+ str(row.get("group_key") or ""),
781
+ ),
782
+ reverse=True,
783
+ )
784
+ for index, group in enumerate(groups, start=1):
785
+ group["rank"] = index
786
+
787
+
788
  def collect_metadata(
789
  *,
790
  creators: list[str],
 
859
  )
860
  candidates = [row for row in popular if row.get("candidate") is True]
861
  groups = group_models(candidates)
862
+ generated_at_dt = now_utc()
863
+ add_rate_metrics(groups, generated_at_dt)
864
  class_counts = Counter(group["candidate_class"] for group in groups)
865
  creator_counts = Counter(group["author"] for group in groups)
866
  summary = {
867
+ "generated_at": iso(generated_at_dt),
868
  "started_at": started,
869
  "source_model_records": len(normalized),
870
  "popular_model_records": len(popular),
 
906
  return "0"
907
 
908
 
909
+ def format_rate(value: Any) -> str:
910
+ try:
911
+ number = float(value)
912
+ except (TypeError, ValueError):
913
+ return "0.0"
914
+ if number >= 100:
915
+ return f"{number:,.1f}"
916
+ return f"{number:,.2f}"
917
+
918
+
919
  def format_params(value: Any) -> str:
920
  if value in {"", None}:
921
  return "unknown"
 
952
  generated_js = json.dumps(str(payload["generated_at"]))
953
  default_age_days = str(filters.get("default_age_filter_days") or filters.get("max_age_days") or DEFAULT_MAX_AGE_DAYS)
954
  top_n_js = json.dumps(filters.get("top_n") or DEFAULT_TOP_N)
955
+ generated_at_dt = parse_hf_datetime(str(payload["generated_at"])) or now_utc()
956
 
957
  def age_selected(value: str) -> str:
958
  return " selected" if value == default_age_days else ""
959
 
960
  table_rows = []
961
  for group in groups:
962
+ if "downloads_per_day" not in group or "likes_per_year" not in group:
963
+ group.update(rate_metrics(group, generated_at_dt))
964
  related_repos = "".join(
965
  f'<a href="{html.escape(model_url(str(item)))}" target="_blank" rel="noreferrer">{html.escape(str(item))}</a>'
966
  for item in group["artifacts"]
967
  )
968
  table_rows.append(
969
  f"""
970
+ <tr data-class="{html.escape(str(group['candidate_class']))}" data-creator="{html.escape(str(group['author']))}" data-created="{html.escape(str(group.get('created_at') or ''))}" data-downloads-per-day="{float(group.get('downloads_per_day') or 0)}" data-likes-per-year="{float(group.get('likes_per_year') or 0)}">
971
  <td class="rank" data-label="Rank">{group['rank']}</td>
972
  <td class="model-cell" data-label="Model">
973
  <a class="model-link" href="{html.escape(model_url(str(group['representative_id'])))}" target="_blank" rel="noreferrer">{html.escape(str(group['group_key']))}</a>
974
  <div class="model-sub">{html.escape(str(group['representative_id']))}</div>
975
  </td>
976
+ <td class="num metric" data-label="Downloads / day"><span>{format_rate(group.get('downloads_per_day'))}/day</span><small>({format_int(group['downloads'])} total)</small></td>
977
+ <td class="num metric" data-label="Likes / year"><span>{format_rate(group.get('likes_per_year'))}/year</span><small>({format_int(group['likes'])} total)</small></td>
978
  <td data-label="Params">{format_params(group['parameter_count_b'])}</td>
979
  <td data-label="Created">{format_date(group.get('created_at'))}</td>
980
  <td class="artifacts" data-label="Related repos">
 
1066
  th {{ position: sticky; top: 0; background: var(--header); z-index: 1; color: var(--muted); font-size: 12px; text-transform: uppercase; }}
1067
  th:nth-child(1), td:nth-child(1) {{ width: 62px; }}
1068
  th:nth-child(2), td:nth-child(2) {{ width: 260px; }}
1069
+ th:nth-child(3), td:nth-child(3) {{ width: 132px; }}
1070
+ th:nth-child(4), td:nth-child(4) {{ width: 116px; }}
1071
  th:nth-child(5), td:nth-child(5) {{ width: 80px; }}
1072
  th:nth-child(6), td:nth-child(6) {{ width: 104px; }}
1073
  tbody tr:hover {{ background: var(--hover); }}
 
1077
  .model-link:hover {{ color: var(--accent); }}
1078
  .model-sub {{ margin-top: 4px; color: var(--muted); font-size: 12px; overflow-wrap: anywhere; }}
1079
  .num {{ text-align: right; white-space: nowrap; font-variant-numeric: tabular-nums; }}
1080
+ .metric small {{ display: block; margin-top: 3px; color: var(--muted); font-size: 11px; white-space: normal; }}
1081
  .artifacts {{ min-width: 280px; color: var(--muted); overflow-wrap: anywhere; }}
1082
  .artifacts span {{ display: flex; flex-wrap: wrap; gap: 5px 10px; overflow-wrap: anywhere; word-break: break-word; }}
1083
  .artifacts a {{ color: var(--link); text-decoration: none; overflow-wrap: anywhere; word-break: break-word; }}
 
1154
  <div class="control">
1155
  <label for="sortFilter">Sort by</label>
1156
  <select id="sortFilter">
1157
+ <option value="downloads_per_day">Downloads / day</option>
1158
+ <option value="likes_per_year">Likes / year</option>
1159
  </select>
1160
  </div>
1161
  <div class="control">
 
1181
  <tr>
1182
  <th data-sort="num">Rank</th>
1183
  <th data-sort="text">Model group</th>
1184
+ <th data-sort="num">Downloads / day</th>
1185
+ <th data-sort="num">Likes / year</th>
1186
  <th data-sort="text">Params</th>
1187
  <th data-sort="text">Created</th>
1188
  <th data-sort="text">Related repos</th>
 
1210
  return ((generatedAt - createdAt) / dayMs) <= Number(maxDays);
1211
  }}
1212
  function sortRows() {{
1213
+ const key = sortFilter.value === 'likes_per_year' ? 'likesPerYear' : 'downloadsPerDay';
1214
  rows.sort((a, b) => {{
1215
  const bv = Number(b.dataset[key]) || 0;
1216
  const av = Number(a.dataset[key]) || 0;
tests/test_leaderboard_core.py CHANGED
@@ -148,7 +148,9 @@ class RenderTests(unittest.TestCase):
148
  "representative_id": "Qwen/Qwen3-8B",
149
  "author": "Qwen",
150
  "downloads": 123_456,
 
151
  "likes": 789,
 
152
  "parameter_count_b": 8,
153
  "capacity_class": "small",
154
  "candidate_class": "text",
@@ -168,8 +170,15 @@ class RenderTests(unittest.TestCase):
168
  self.assertIn("Open Model Leaderboard", text)
169
  self.assertIn("id=\"search\"", text)
170
  self.assertIn("id=\"ageFilter\"", text)
 
171
  self.assertIn('<option value="365" selected>1 year</option>', text)
 
 
 
 
172
  self.assertNotIn(">Score<", text)
 
 
173
  self.assertIn("qwen3-8b", text)
174
  self.assertIn("leaderboard", text)
175
 
 
148
  "representative_id": "Qwen/Qwen3-8B",
149
  "author": "Qwen",
150
  "downloads": 123_456,
151
+ "downloads_per_day": 734.25,
152
  "likes": 789,
153
+ "likes_per_year": 1_708.5,
154
  "parameter_count_b": 8,
155
  "capacity_class": "small",
156
  "candidate_class": "text",
 
170
  self.assertIn("Open Model Leaderboard", text)
171
  self.assertIn("id=\"search\"", text)
172
  self.assertIn("id=\"ageFilter\"", text)
173
+ self.assertIn("id=\"sortFilter\"", text)
174
  self.assertIn('<option value="365" selected>1 year</option>', text)
175
+ self.assertIn("Downloads / day", text)
176
+ self.assertIn("Likes / year", text)
177
+ self.assertIn("(123,456 total)", text)
178
+ self.assertIn("(789 total)", text)
179
  self.assertNotIn(">Score<", text)
180
+ self.assertNotIn(">Downloads<", text)
181
+ self.assertNotIn(">Likes<", text)
182
  self.assertIn("qwen3-8b", text)
183
  self.assertIn("leaderboard", text)
184