gsaltintas commited on
Commit
6d2e42d
·
verified ·
1 Parent(s): 46704cd

Upload safe_v2/flexitok--bpe_hun_Latn_8000_overlap.json with huggingface_hub

Browse files
safe_v2/flexitok--bpe_hun_Latn_8000_overlap.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"1": {"ratio_to_total_tokens": 0.679625, "expected_training_ratio_in_superset": 0.03236309523809523, "num_tokens": 5437}, "2": {"ratio_to_total_tokens": 0.047875, "expected_training_ratio_in_superset": 0.004559523809523809, "num_tokens": 383}, "3": {"ratio_to_total_tokens": 0.028, "expected_training_ratio_in_superset": 0.004, "num_tokens": 224}, "4": {"ratio_to_total_tokens": 0.02225, "expected_training_ratio_in_superset": 0.004238095238095238, "num_tokens": 178}, "5": {"ratio_to_total_tokens": 0.0155, "expected_training_ratio_in_superset": 0.00369047619047619, "num_tokens": 124}, "6": {"ratio_to_total_tokens": 0.01625, "expected_training_ratio_in_superset": 0.004642857142857142, "num_tokens": 130}, "7": {"ratio_to_total_tokens": 0.012125, "expected_training_ratio_in_superset": 0.0040416666666666665, "num_tokens": 97}, "8": {"ratio_to_total_tokens": 0.012, "expected_training_ratio_in_superset": 0.004571428571428571, "num_tokens": 96}, "9": {"ratio_to_total_tokens": 0.012625, "expected_training_ratio_in_superset": 0.005410714285714285, "num_tokens": 101}, "10": {"ratio_to_total_tokens": 0.012375, "expected_training_ratio_in_superset": 0.005892857142857142, "num_tokens": 99}, "11": {"ratio_to_total_tokens": 0.0115, "expected_training_ratio_in_superset": 0.006023809523809523, "num_tokens": 92}, "12": {"ratio_to_total_tokens": 0.01025, "expected_training_ratio_in_superset": 0.005857142857142857, "num_tokens": 82}, "13": {"ratio_to_total_tokens": 0.010125, "expected_training_ratio_in_superset": 0.006267857142857144, "num_tokens": 81}, "14": {"ratio_to_total_tokens": 0.01175, "expected_training_ratio_in_superset": 0.007833333333333333, "num_tokens": 94}, "15": {"ratio_to_total_tokens": 0.0125, "expected_training_ratio_in_superset": 0.008928571428571428, "num_tokens": 100}, "16": {"ratio_to_total_tokens": 0.00775, "expected_training_ratio_in_superset": 0.005904761904761904, "num_tokens": 62}, "17": {"ratio_to_total_tokens": 0.00575, "expected_training_ratio_in_superset": 0.0046547619047619055, "num_tokens": 46}, "18": {"ratio_to_total_tokens": 0.009, "expected_training_ratio_in_superset": 0.0077142857142857135, "num_tokens": 72}, "19": {"ratio_to_total_tokens": 0.0065, "expected_training_ratio_in_superset": 0.005880952380952382, "num_tokens": 52}, "20": {"ratio_to_total_tokens": 0.005625, "expected_training_ratio_in_superset": 0.005357142857142857, "num_tokens": 45}, "21": {"ratio_to_total_tokens": 0.050625, "expected_training_ratio_in_superset": 0.050625, "num_tokens": 405}, "total_training_compared_to_full_model": 0.18845833333333334}