{ "base": { "dir": "/ai/text-generation/models/zai-org_GLM-4.6-2.0bpw-h6-exl3", "bpw": 2.0162048233621186 }, "alts": [ { "dir": "/ai/text-generation/models/zai-org_GLM-4.6-3.0bpw-h6-exl3", "bpw": 3.0159972500619223 } ], "groups": [ { "idx": 0, "layers": [ "model.layers.0.self_attn.q_proj" ], "candidates": [ { "dkld": 0.000309491157531705, "dbits": 62914560 } ] }, { "idx": 1, "layers": [ "model.layers.0.self_attn.k_proj", "model.layers.0.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0002951681613921897, "dbits": 10485760 } ] }, { "idx": 2, "layers": [ "model.layers.0.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00021884366869923122, "dbits": 62914560 } ] }, { "idx": 3, "layers": [ "model.layers.0.mlp.gate_proj", "model.layers.0.mlp.up_proj" ], "candidates": [ { "dkld": -0.00010240301489833348, "dbits": 125829120 } ] }, { "idx": 4, "layers": [ "model.layers.0.mlp.down_proj" ], "candidates": [ { "dkld": -0.0006501421332359425, "dbits": 62914560 } ] }, { "idx": 5, "layers": [ "model.layers.1.self_attn.q_proj" ], "candidates": [ { "dkld": 0.000216712802648511, "dbits": 62914560 } ] }, { "idx": 6, "layers": [ "model.layers.1.self_attn.k_proj", "model.layers.1.self_attn.v_proj" ], "candidates": [ { "dkld": 0.000852769613265969, "dbits": 10485760 } ] }, { "idx": 7, "layers": [ "model.layers.1.self_attn.o_proj" ], "candidates": [ { "dkld": -6.86943531036377e-05, "dbits": 62914560 } ] }, { "idx": 8, "layers": [ "model.layers.1.mlp.gate_proj", "model.layers.1.mlp.up_proj" ], "candidates": [ { "dkld": -0.0021039567887783384, "dbits": 125829120 } ] }, { "idx": 9, "layers": [ "model.layers.1.mlp.down_proj" ], "candidates": [ { "dkld": 0.0011117316782474629, "dbits": 62914560 } ] }, { "idx": 10, "layers": [ "model.layers.2.self_attn.q_proj" ], "candidates": [ { "dkld": 0.000774417817592632, "dbits": 62914560 } ] }, { "idx": 11, "layers": [ "model.layers.2.self_attn.k_proj", "model.layers.2.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00011329278349875294, "dbits": 10485760 } ] }, { "idx": 12, "layers": [ "model.layers.2.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0013314291834831127, "dbits": 62914560 } ] }, { "idx": 13, "layers": [ "model.layers.2.mlp.gate_proj", "model.layers.2.mlp.up_proj" ], "candidates": [ { "dkld": -0.007626063376665104, "dbits": 125829120 } ] }, { "idx": 14, "layers": [ "model.layers.2.mlp.down_proj" ], "candidates": [ { "dkld": -0.001953936368226994, "dbits": 62914560 } ] }, { "idx": 15, "layers": [ "model.layers.3.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0018831357359885836, "dbits": 62914560 } ] }, { "idx": 16, "layers": [ "model.layers.3.self_attn.k_proj", "model.layers.3.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0004919447004794741, "dbits": 10485760 } ] }, { "idx": 17, "layers": [ "model.layers.3.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0012530252337455416, "dbits": 62914560 } ] }, { "idx": 18, "layers": [ "model.layers.3.mlp.shared_experts.gate_proj", "model.layers.3.mlp.shared_experts.up_proj", "model.layers.3.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0002859249711036904, "dbits": 23592960 } ] }, { "idx": 19, "layers": [ "model.layers.3.mlp.experts.0.gate_proj", "model.layers.3.mlp.experts.1.gate_proj", "model.layers.3.mlp.experts.2.gate_proj", "model.layers.3.mlp.experts.3.gate_proj", "model.layers.3.mlp.experts.4.gate_proj", "model.layers.3.mlp.experts.5.gate_proj", "model.layers.3.mlp.experts.6.gate_proj", "model.layers.3.mlp.experts.7.gate_proj", "model.layers.3.mlp.experts.8.gate_proj", "model.layers.3.mlp.experts.9.gate_proj", "model.layers.3.mlp.experts.10.gate_proj", "model.layers.3.mlp.experts.11.gate_proj", "model.layers.3.mlp.experts.12.gate_proj", "model.layers.3.mlp.experts.13.gate_proj", "model.layers.3.mlp.experts.14.gate_proj", "model.layers.3.mlp.experts.15.gate_proj", "model.layers.3.mlp.experts.16.gate_proj", "model.layers.3.mlp.experts.17.gate_proj", "model.layers.3.mlp.experts.18.gate_proj", "model.layers.3.mlp.experts.19.gate_proj", "model.layers.3.mlp.experts.20.gate_proj", "model.layers.3.mlp.experts.21.gate_proj", "model.layers.3.mlp.experts.22.gate_proj", "model.layers.3.mlp.experts.23.gate_proj", "model.layers.3.mlp.experts.24.gate_proj", "model.layers.3.mlp.experts.25.gate_proj", "model.layers.3.mlp.experts.26.gate_proj", "model.layers.3.mlp.experts.27.gate_proj", "model.layers.3.mlp.experts.28.gate_proj", "model.layers.3.mlp.experts.29.gate_proj", "model.layers.3.mlp.experts.30.gate_proj", "model.layers.3.mlp.experts.31.gate_proj", "model.layers.3.mlp.experts.32.gate_proj", "model.layers.3.mlp.experts.33.gate_proj", "model.layers.3.mlp.experts.34.gate_proj", "model.layers.3.mlp.experts.35.gate_proj", "model.layers.3.mlp.experts.36.gate_proj", "model.layers.3.mlp.experts.37.gate_proj", "model.layers.3.mlp.experts.38.gate_proj", "model.layers.3.mlp.experts.39.gate_proj", "model.layers.3.mlp.experts.40.gate_proj", "model.layers.3.mlp.experts.41.gate_proj", "model.layers.3.mlp.experts.42.gate_proj", "model.layers.3.mlp.experts.43.gate_proj", "model.layers.3.mlp.experts.44.gate_proj", "model.layers.3.mlp.experts.45.gate_proj", "model.layers.3.mlp.experts.46.gate_proj", "model.layers.3.mlp.experts.47.gate_proj", "model.layers.3.mlp.experts.48.gate_proj", "model.layers.3.mlp.experts.49.gate_proj", "model.layers.3.mlp.experts.50.gate_proj", "model.layers.3.mlp.experts.51.gate_proj", "model.layers.3.mlp.experts.52.gate_proj", "model.layers.3.mlp.experts.53.gate_proj", "model.layers.3.mlp.experts.54.gate_proj", "model.layers.3.mlp.experts.55.gate_proj", "model.layers.3.mlp.experts.56.gate_proj", "model.layers.3.mlp.experts.57.gate_proj", "model.layers.3.mlp.experts.58.gate_proj", "model.layers.3.mlp.experts.59.gate_proj", "model.layers.3.mlp.experts.60.gate_proj", "model.layers.3.mlp.experts.61.gate_proj", "model.layers.3.mlp.experts.62.gate_proj", "model.layers.3.mlp.experts.63.gate_proj", "model.layers.3.mlp.experts.64.gate_proj", "model.layers.3.mlp.experts.65.gate_proj", "model.layers.3.mlp.experts.66.gate_proj", "model.layers.3.mlp.experts.67.gate_proj", "model.layers.3.mlp.experts.68.gate_proj", "model.layers.3.mlp.experts.69.gate_proj", "model.layers.3.mlp.experts.70.gate_proj", "model.layers.3.mlp.experts.71.gate_proj", "model.layers.3.mlp.experts.72.gate_proj", "model.layers.3.mlp.experts.73.gate_proj", "model.layers.3.mlp.experts.74.gate_proj", "model.layers.3.mlp.experts.75.gate_proj", "model.layers.3.mlp.experts.76.gate_proj", "model.layers.3.mlp.experts.77.gate_proj", "model.layers.3.mlp.experts.78.gate_proj", "model.layers.3.mlp.experts.79.gate_proj", "model.layers.3.mlp.experts.80.gate_proj", "model.layers.3.mlp.experts.81.gate_proj", "model.layers.3.mlp.experts.82.gate_proj", "model.layers.3.mlp.experts.83.gate_proj", "model.layers.3.mlp.experts.84.gate_proj", "model.layers.3.mlp.experts.85.gate_proj", "model.layers.3.mlp.experts.86.gate_proj", "model.layers.3.mlp.experts.87.gate_proj", "model.layers.3.mlp.experts.88.gate_proj", "model.layers.3.mlp.experts.89.gate_proj", "model.layers.3.mlp.experts.90.gate_proj", "model.layers.3.mlp.experts.91.gate_proj", "model.layers.3.mlp.experts.92.gate_proj", "model.layers.3.mlp.experts.93.gate_proj", "model.layers.3.mlp.experts.94.gate_proj", "model.layers.3.mlp.experts.95.gate_proj", "model.layers.3.mlp.experts.96.gate_proj", "model.layers.3.mlp.experts.97.gate_proj", "model.layers.3.mlp.experts.98.gate_proj", "model.layers.3.mlp.experts.99.gate_proj", "model.layers.3.mlp.experts.100.gate_proj", "model.layers.3.mlp.experts.101.gate_proj", "model.layers.3.mlp.experts.102.gate_proj", "model.layers.3.mlp.experts.103.gate_proj", "model.layers.3.mlp.experts.104.gate_proj", "model.layers.3.mlp.experts.105.gate_proj", "model.layers.3.mlp.experts.106.gate_proj", "model.layers.3.mlp.experts.107.gate_proj", "model.layers.3.mlp.experts.108.gate_proj", "model.layers.3.mlp.experts.109.gate_proj", "model.layers.3.mlp.experts.110.gate_proj", "model.layers.3.mlp.experts.111.gate_proj", "model.layers.3.mlp.experts.112.gate_proj", "model.layers.3.mlp.experts.113.gate_proj", "model.layers.3.mlp.experts.114.gate_proj", "model.layers.3.mlp.experts.115.gate_proj", "model.layers.3.mlp.experts.116.gate_proj", "model.layers.3.mlp.experts.117.gate_proj", "model.layers.3.mlp.experts.118.gate_proj", "model.layers.3.mlp.experts.119.gate_proj", "model.layers.3.mlp.experts.120.gate_proj", "model.layers.3.mlp.experts.121.gate_proj", "model.layers.3.mlp.experts.122.gate_proj", "model.layers.3.mlp.experts.123.gate_proj", "model.layers.3.mlp.experts.124.gate_proj", "model.layers.3.mlp.experts.125.gate_proj", "model.layers.3.mlp.experts.126.gate_proj", "model.layers.3.mlp.experts.127.gate_proj", "model.layers.3.mlp.experts.128.gate_proj", "model.layers.3.mlp.experts.129.gate_proj", "model.layers.3.mlp.experts.130.gate_proj", "model.layers.3.mlp.experts.131.gate_proj", "model.layers.3.mlp.experts.132.gate_proj", "model.layers.3.mlp.experts.133.gate_proj", "model.layers.3.mlp.experts.134.gate_proj", "model.layers.3.mlp.experts.135.gate_proj", "model.layers.3.mlp.experts.136.gate_proj", "model.layers.3.mlp.experts.137.gate_proj", "model.layers.3.mlp.experts.138.gate_proj", "model.layers.3.mlp.experts.139.gate_proj", "model.layers.3.mlp.experts.140.gate_proj", "model.layers.3.mlp.experts.141.gate_proj", "model.layers.3.mlp.experts.142.gate_proj", "model.layers.3.mlp.experts.143.gate_proj", "model.layers.3.mlp.experts.144.gate_proj", "model.layers.3.mlp.experts.145.gate_proj", "model.layers.3.mlp.experts.146.gate_proj", "model.layers.3.mlp.experts.147.gate_proj", "model.layers.3.mlp.experts.148.gate_proj", "model.layers.3.mlp.experts.149.gate_proj", "model.layers.3.mlp.experts.150.gate_proj", "model.layers.3.mlp.experts.151.gate_proj", "model.layers.3.mlp.experts.152.gate_proj", "model.layers.3.mlp.experts.153.gate_proj", "model.layers.3.mlp.experts.154.gate_proj", "model.layers.3.mlp.experts.155.gate_proj", "model.layers.3.mlp.experts.156.gate_proj", "model.layers.3.mlp.experts.157.gate_proj", "model.layers.3.mlp.experts.158.gate_proj", "model.layers.3.mlp.experts.159.gate_proj", "model.layers.3.mlp.experts.0.up_proj", "model.layers.3.mlp.experts.1.up_proj", "model.layers.3.mlp.experts.2.up_proj", "model.layers.3.mlp.experts.3.up_proj", "model.layers.3.mlp.experts.4.up_proj", "model.layers.3.mlp.experts.5.up_proj", "model.layers.3.mlp.experts.6.up_proj", "model.layers.3.mlp.experts.7.up_proj", "model.layers.3.mlp.experts.8.up_proj", "model.layers.3.mlp.experts.9.up_proj", "model.layers.3.mlp.experts.10.up_proj", "model.layers.3.mlp.experts.11.up_proj", "model.layers.3.mlp.experts.12.up_proj", "model.layers.3.mlp.experts.13.up_proj", "model.layers.3.mlp.experts.14.up_proj", "model.layers.3.mlp.experts.15.up_proj", "model.layers.3.mlp.experts.16.up_proj", "model.layers.3.mlp.experts.17.up_proj", "model.layers.3.mlp.experts.18.up_proj", "model.layers.3.mlp.experts.19.up_proj", "model.layers.3.mlp.experts.20.up_proj", "model.layers.3.mlp.experts.21.up_proj", "model.layers.3.mlp.experts.22.up_proj", "model.layers.3.mlp.experts.23.up_proj", "model.layers.3.mlp.experts.24.up_proj", "model.layers.3.mlp.experts.25.up_proj", "model.layers.3.mlp.experts.26.up_proj", "model.layers.3.mlp.experts.27.up_proj", "model.layers.3.mlp.experts.28.up_proj", "model.layers.3.mlp.experts.29.up_proj", "model.layers.3.mlp.experts.30.up_proj", "model.layers.3.mlp.experts.31.up_proj", "model.layers.3.mlp.experts.32.up_proj", "model.layers.3.mlp.experts.33.up_proj", "model.layers.3.mlp.experts.34.up_proj", "model.layers.3.mlp.experts.35.up_proj", "model.layers.3.mlp.experts.36.up_proj", "model.layers.3.mlp.experts.37.up_proj", "model.layers.3.mlp.experts.38.up_proj", "model.layers.3.mlp.experts.39.up_proj", "model.layers.3.mlp.experts.40.up_proj", "model.layers.3.mlp.experts.41.up_proj", "model.layers.3.mlp.experts.42.up_proj", "model.layers.3.mlp.experts.43.up_proj", "model.layers.3.mlp.experts.44.up_proj", "model.layers.3.mlp.experts.45.up_proj", "model.layers.3.mlp.experts.46.up_proj", "model.layers.3.mlp.experts.47.up_proj", "model.layers.3.mlp.experts.48.up_proj", "model.layers.3.mlp.experts.49.up_proj", "model.layers.3.mlp.experts.50.up_proj", "model.layers.3.mlp.experts.51.up_proj", "model.layers.3.mlp.experts.52.up_proj", "model.layers.3.mlp.experts.53.up_proj", "model.layers.3.mlp.experts.54.up_proj", "model.layers.3.mlp.experts.55.up_proj", "model.layers.3.mlp.experts.56.up_proj", "model.layers.3.mlp.experts.57.up_proj", "model.layers.3.mlp.experts.58.up_proj", "model.layers.3.mlp.experts.59.up_proj", "model.layers.3.mlp.experts.60.up_proj", "model.layers.3.mlp.experts.61.up_proj", "model.layers.3.mlp.experts.62.up_proj", "model.layers.3.mlp.experts.63.up_proj", "model.layers.3.mlp.experts.64.up_proj", "model.layers.3.mlp.experts.65.up_proj", "model.layers.3.mlp.experts.66.up_proj", "model.layers.3.mlp.experts.67.up_proj", "model.layers.3.mlp.experts.68.up_proj", "model.layers.3.mlp.experts.69.up_proj", "model.layers.3.mlp.experts.70.up_proj", "model.layers.3.mlp.experts.71.up_proj", "model.layers.3.mlp.experts.72.up_proj", "model.layers.3.mlp.experts.73.up_proj", "model.layers.3.mlp.experts.74.up_proj", "model.layers.3.mlp.experts.75.up_proj", "model.layers.3.mlp.experts.76.up_proj", "model.layers.3.mlp.experts.77.up_proj", "model.layers.3.mlp.experts.78.up_proj", "model.layers.3.mlp.experts.79.up_proj", "model.layers.3.mlp.experts.80.up_proj", "model.layers.3.mlp.experts.81.up_proj", "model.layers.3.mlp.experts.82.up_proj", "model.layers.3.mlp.experts.83.up_proj", "model.layers.3.mlp.experts.84.up_proj", "model.layers.3.mlp.experts.85.up_proj", "model.layers.3.mlp.experts.86.up_proj", "model.layers.3.mlp.experts.87.up_proj", "model.layers.3.mlp.experts.88.up_proj", "model.layers.3.mlp.experts.89.up_proj", "model.layers.3.mlp.experts.90.up_proj", "model.layers.3.mlp.experts.91.up_proj", "model.layers.3.mlp.experts.92.up_proj", "model.layers.3.mlp.experts.93.up_proj", "model.layers.3.mlp.experts.94.up_proj", "model.layers.3.mlp.experts.95.up_proj", "model.layers.3.mlp.experts.96.up_proj", "model.layers.3.mlp.experts.97.up_proj", "model.layers.3.mlp.experts.98.up_proj", "model.layers.3.mlp.experts.99.up_proj", "model.layers.3.mlp.experts.100.up_proj", "model.layers.3.mlp.experts.101.up_proj", "model.layers.3.mlp.experts.102.up_proj", "model.layers.3.mlp.experts.103.up_proj", "model.layers.3.mlp.experts.104.up_proj", "model.layers.3.mlp.experts.105.up_proj", "model.layers.3.mlp.experts.106.up_proj", "model.layers.3.mlp.experts.107.up_proj", "model.layers.3.mlp.experts.108.up_proj", "model.layers.3.mlp.experts.109.up_proj", "model.layers.3.mlp.experts.110.up_proj", "model.layers.3.mlp.experts.111.up_proj", "model.layers.3.mlp.experts.112.up_proj", "model.layers.3.mlp.experts.113.up_proj", "model.layers.3.mlp.experts.114.up_proj", "model.layers.3.mlp.experts.115.up_proj", "model.layers.3.mlp.experts.116.up_proj", "model.layers.3.mlp.experts.117.up_proj", "model.layers.3.mlp.experts.118.up_proj", "model.layers.3.mlp.experts.119.up_proj", "model.layers.3.mlp.experts.120.up_proj", "model.layers.3.mlp.experts.121.up_proj", "model.layers.3.mlp.experts.122.up_proj", "model.layers.3.mlp.experts.123.up_proj", "model.layers.3.mlp.experts.124.up_proj", "model.layers.3.mlp.experts.125.up_proj", "model.layers.3.mlp.experts.126.up_proj", "model.layers.3.mlp.experts.127.up_proj", "model.layers.3.mlp.experts.128.up_proj", "model.layers.3.mlp.experts.129.up_proj", "model.layers.3.mlp.experts.130.up_proj", "model.layers.3.mlp.experts.131.up_proj", "model.layers.3.mlp.experts.132.up_proj", "model.layers.3.mlp.experts.133.up_proj", "model.layers.3.mlp.experts.134.up_proj", "model.layers.3.mlp.experts.135.up_proj", "model.layers.3.mlp.experts.136.up_proj", "model.layers.3.mlp.experts.137.up_proj", "model.layers.3.mlp.experts.138.up_proj", "model.layers.3.mlp.experts.139.up_proj", "model.layers.3.mlp.experts.140.up_proj", "model.layers.3.mlp.experts.141.up_proj", "model.layers.3.mlp.experts.142.up_proj", "model.layers.3.mlp.experts.143.up_proj", "model.layers.3.mlp.experts.144.up_proj", "model.layers.3.mlp.experts.145.up_proj", "model.layers.3.mlp.experts.146.up_proj", "model.layers.3.mlp.experts.147.up_proj", "model.layers.3.mlp.experts.148.up_proj", "model.layers.3.mlp.experts.149.up_proj", "model.layers.3.mlp.experts.150.up_proj", "model.layers.3.mlp.experts.151.up_proj", "model.layers.3.mlp.experts.152.up_proj", "model.layers.3.mlp.experts.153.up_proj", "model.layers.3.mlp.experts.154.up_proj", "model.layers.3.mlp.experts.155.up_proj", "model.layers.3.mlp.experts.156.up_proj", "model.layers.3.mlp.experts.157.up_proj", "model.layers.3.mlp.experts.158.up_proj", "model.layers.3.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0008694708347320446, "dbits": 2516582400 } ] }, { "idx": 20, "layers": [ "model.layers.3.mlp.experts.0.down_proj", "model.layers.3.mlp.experts.1.down_proj", "model.layers.3.mlp.experts.2.down_proj", "model.layers.3.mlp.experts.3.down_proj", "model.layers.3.mlp.experts.4.down_proj", "model.layers.3.mlp.experts.5.down_proj", "model.layers.3.mlp.experts.6.down_proj", "model.layers.3.mlp.experts.7.down_proj", "model.layers.3.mlp.experts.8.down_proj", "model.layers.3.mlp.experts.9.down_proj", "model.layers.3.mlp.experts.10.down_proj", "model.layers.3.mlp.experts.11.down_proj", "model.layers.3.mlp.experts.12.down_proj", "model.layers.3.mlp.experts.13.down_proj", "model.layers.3.mlp.experts.14.down_proj", "model.layers.3.mlp.experts.15.down_proj", "model.layers.3.mlp.experts.16.down_proj", "model.layers.3.mlp.experts.17.down_proj", "model.layers.3.mlp.experts.18.down_proj", "model.layers.3.mlp.experts.19.down_proj", "model.layers.3.mlp.experts.20.down_proj", "model.layers.3.mlp.experts.21.down_proj", "model.layers.3.mlp.experts.22.down_proj", "model.layers.3.mlp.experts.23.down_proj", "model.layers.3.mlp.experts.24.down_proj", "model.layers.3.mlp.experts.25.down_proj", "model.layers.3.mlp.experts.26.down_proj", "model.layers.3.mlp.experts.27.down_proj", "model.layers.3.mlp.experts.28.down_proj", "model.layers.3.mlp.experts.29.down_proj", "model.layers.3.mlp.experts.30.down_proj", "model.layers.3.mlp.experts.31.down_proj", "model.layers.3.mlp.experts.32.down_proj", "model.layers.3.mlp.experts.33.down_proj", "model.layers.3.mlp.experts.34.down_proj", "model.layers.3.mlp.experts.35.down_proj", "model.layers.3.mlp.experts.36.down_proj", "model.layers.3.mlp.experts.37.down_proj", "model.layers.3.mlp.experts.38.down_proj", "model.layers.3.mlp.experts.39.down_proj", "model.layers.3.mlp.experts.40.down_proj", "model.layers.3.mlp.experts.41.down_proj", "model.layers.3.mlp.experts.42.down_proj", "model.layers.3.mlp.experts.43.down_proj", "model.layers.3.mlp.experts.44.down_proj", "model.layers.3.mlp.experts.45.down_proj", "model.layers.3.mlp.experts.46.down_proj", "model.layers.3.mlp.experts.47.down_proj", "model.layers.3.mlp.experts.48.down_proj", "model.layers.3.mlp.experts.49.down_proj", "model.layers.3.mlp.experts.50.down_proj", "model.layers.3.mlp.experts.51.down_proj", "model.layers.3.mlp.experts.52.down_proj", "model.layers.3.mlp.experts.53.down_proj", "model.layers.3.mlp.experts.54.down_proj", "model.layers.3.mlp.experts.55.down_proj", "model.layers.3.mlp.experts.56.down_proj", "model.layers.3.mlp.experts.57.down_proj", "model.layers.3.mlp.experts.58.down_proj", "model.layers.3.mlp.experts.59.down_proj", "model.layers.3.mlp.experts.60.down_proj", "model.layers.3.mlp.experts.61.down_proj", "model.layers.3.mlp.experts.62.down_proj", "model.layers.3.mlp.experts.63.down_proj", "model.layers.3.mlp.experts.64.down_proj", "model.layers.3.mlp.experts.65.down_proj", "model.layers.3.mlp.experts.66.down_proj", "model.layers.3.mlp.experts.67.down_proj", "model.layers.3.mlp.experts.68.down_proj", "model.layers.3.mlp.experts.69.down_proj", "model.layers.3.mlp.experts.70.down_proj", "model.layers.3.mlp.experts.71.down_proj", "model.layers.3.mlp.experts.72.down_proj", "model.layers.3.mlp.experts.73.down_proj", "model.layers.3.mlp.experts.74.down_proj", "model.layers.3.mlp.experts.75.down_proj", "model.layers.3.mlp.experts.76.down_proj", "model.layers.3.mlp.experts.77.down_proj", "model.layers.3.mlp.experts.78.down_proj", "model.layers.3.mlp.experts.79.down_proj", "model.layers.3.mlp.experts.80.down_proj", "model.layers.3.mlp.experts.81.down_proj", "model.layers.3.mlp.experts.82.down_proj", "model.layers.3.mlp.experts.83.down_proj", "model.layers.3.mlp.experts.84.down_proj", "model.layers.3.mlp.experts.85.down_proj", "model.layers.3.mlp.experts.86.down_proj", "model.layers.3.mlp.experts.87.down_proj", "model.layers.3.mlp.experts.88.down_proj", "model.layers.3.mlp.experts.89.down_proj", "model.layers.3.mlp.experts.90.down_proj", "model.layers.3.mlp.experts.91.down_proj", "model.layers.3.mlp.experts.92.down_proj", "model.layers.3.mlp.experts.93.down_proj", "model.layers.3.mlp.experts.94.down_proj", "model.layers.3.mlp.experts.95.down_proj", "model.layers.3.mlp.experts.96.down_proj", "model.layers.3.mlp.experts.97.down_proj", "model.layers.3.mlp.experts.98.down_proj", "model.layers.3.mlp.experts.99.down_proj", "model.layers.3.mlp.experts.100.down_proj", "model.layers.3.mlp.experts.101.down_proj", "model.layers.3.mlp.experts.102.down_proj", "model.layers.3.mlp.experts.103.down_proj", "model.layers.3.mlp.experts.104.down_proj", "model.layers.3.mlp.experts.105.down_proj", "model.layers.3.mlp.experts.106.down_proj", "model.layers.3.mlp.experts.107.down_proj", "model.layers.3.mlp.experts.108.down_proj", "model.layers.3.mlp.experts.109.down_proj", "model.layers.3.mlp.experts.110.down_proj", "model.layers.3.mlp.experts.111.down_proj", "model.layers.3.mlp.experts.112.down_proj", "model.layers.3.mlp.experts.113.down_proj", "model.layers.3.mlp.experts.114.down_proj", "model.layers.3.mlp.experts.115.down_proj", "model.layers.3.mlp.experts.116.down_proj", "model.layers.3.mlp.experts.117.down_proj", "model.layers.3.mlp.experts.118.down_proj", "model.layers.3.mlp.experts.119.down_proj", "model.layers.3.mlp.experts.120.down_proj", "model.layers.3.mlp.experts.121.down_proj", "model.layers.3.mlp.experts.122.down_proj", "model.layers.3.mlp.experts.123.down_proj", "model.layers.3.mlp.experts.124.down_proj", "model.layers.3.mlp.experts.125.down_proj", "model.layers.3.mlp.experts.126.down_proj", "model.layers.3.mlp.experts.127.down_proj", "model.layers.3.mlp.experts.128.down_proj", "model.layers.3.mlp.experts.129.down_proj", "model.layers.3.mlp.experts.130.down_proj", "model.layers.3.mlp.experts.131.down_proj", "model.layers.3.mlp.experts.132.down_proj", "model.layers.3.mlp.experts.133.down_proj", "model.layers.3.mlp.experts.134.down_proj", "model.layers.3.mlp.experts.135.down_proj", "model.layers.3.mlp.experts.136.down_proj", "model.layers.3.mlp.experts.137.down_proj", "model.layers.3.mlp.experts.138.down_proj", "model.layers.3.mlp.experts.139.down_proj", "model.layers.3.mlp.experts.140.down_proj", "model.layers.3.mlp.experts.141.down_proj", "model.layers.3.mlp.experts.142.down_proj", "model.layers.3.mlp.experts.143.down_proj", "model.layers.3.mlp.experts.144.down_proj", "model.layers.3.mlp.experts.145.down_proj", "model.layers.3.mlp.experts.146.down_proj", "model.layers.3.mlp.experts.147.down_proj", "model.layers.3.mlp.experts.148.down_proj", "model.layers.3.mlp.experts.149.down_proj", "model.layers.3.mlp.experts.150.down_proj", "model.layers.3.mlp.experts.151.down_proj", "model.layers.3.mlp.experts.152.down_proj", "model.layers.3.mlp.experts.153.down_proj", "model.layers.3.mlp.experts.154.down_proj", "model.layers.3.mlp.experts.155.down_proj", "model.layers.3.mlp.experts.156.down_proj", "model.layers.3.mlp.experts.157.down_proj", "model.layers.3.mlp.experts.158.down_proj", "model.layers.3.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0005423903465270885, "dbits": 1258291200 } ] }, { "idx": 21, "layers": [ "model.layers.4.self_attn.q_proj" ], "candidates": [ { "dkld": 2.911165356633827e-05, "dbits": 62914560 } ] }, { "idx": 22, "layers": [ "model.layers.4.self_attn.k_proj", "model.layers.4.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0009578339755534904, "dbits": 10485760 } ] }, { "idx": 23, "layers": [ "model.layers.4.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00016218572854992397, "dbits": 62914560 } ] }, { "idx": 24, "layers": [ "model.layers.4.mlp.shared_experts.gate_proj", "model.layers.4.mlp.shared_experts.up_proj", "model.layers.4.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -6.86153769492992e-05, "dbits": 23592960 } ] }, { "idx": 25, "layers": [ "model.layers.4.mlp.experts.0.gate_proj", "model.layers.4.mlp.experts.1.gate_proj", "model.layers.4.mlp.experts.2.gate_proj", "model.layers.4.mlp.experts.3.gate_proj", "model.layers.4.mlp.experts.4.gate_proj", "model.layers.4.mlp.experts.5.gate_proj", "model.layers.4.mlp.experts.6.gate_proj", "model.layers.4.mlp.experts.7.gate_proj", "model.layers.4.mlp.experts.8.gate_proj", "model.layers.4.mlp.experts.9.gate_proj", "model.layers.4.mlp.experts.10.gate_proj", "model.layers.4.mlp.experts.11.gate_proj", "model.layers.4.mlp.experts.12.gate_proj", "model.layers.4.mlp.experts.13.gate_proj", "model.layers.4.mlp.experts.14.gate_proj", "model.layers.4.mlp.experts.15.gate_proj", "model.layers.4.mlp.experts.16.gate_proj", "model.layers.4.mlp.experts.17.gate_proj", "model.layers.4.mlp.experts.18.gate_proj", "model.layers.4.mlp.experts.19.gate_proj", "model.layers.4.mlp.experts.20.gate_proj", "model.layers.4.mlp.experts.21.gate_proj", "model.layers.4.mlp.experts.22.gate_proj", "model.layers.4.mlp.experts.23.gate_proj", "model.layers.4.mlp.experts.24.gate_proj", "model.layers.4.mlp.experts.25.gate_proj", "model.layers.4.mlp.experts.26.gate_proj", "model.layers.4.mlp.experts.27.gate_proj", "model.layers.4.mlp.experts.28.gate_proj", "model.layers.4.mlp.experts.29.gate_proj", "model.layers.4.mlp.experts.30.gate_proj", "model.layers.4.mlp.experts.31.gate_proj", "model.layers.4.mlp.experts.32.gate_proj", "model.layers.4.mlp.experts.33.gate_proj", "model.layers.4.mlp.experts.34.gate_proj", "model.layers.4.mlp.experts.35.gate_proj", "model.layers.4.mlp.experts.36.gate_proj", "model.layers.4.mlp.experts.37.gate_proj", "model.layers.4.mlp.experts.38.gate_proj", "model.layers.4.mlp.experts.39.gate_proj", "model.layers.4.mlp.experts.40.gate_proj", "model.layers.4.mlp.experts.41.gate_proj", "model.layers.4.mlp.experts.42.gate_proj", "model.layers.4.mlp.experts.43.gate_proj", "model.layers.4.mlp.experts.44.gate_proj", "model.layers.4.mlp.experts.45.gate_proj", "model.layers.4.mlp.experts.46.gate_proj", "model.layers.4.mlp.experts.47.gate_proj", "model.layers.4.mlp.experts.48.gate_proj", "model.layers.4.mlp.experts.49.gate_proj", "model.layers.4.mlp.experts.50.gate_proj", "model.layers.4.mlp.experts.51.gate_proj", "model.layers.4.mlp.experts.52.gate_proj", "model.layers.4.mlp.experts.53.gate_proj", "model.layers.4.mlp.experts.54.gate_proj", "model.layers.4.mlp.experts.55.gate_proj", "model.layers.4.mlp.experts.56.gate_proj", "model.layers.4.mlp.experts.57.gate_proj", "model.layers.4.mlp.experts.58.gate_proj", "model.layers.4.mlp.experts.59.gate_proj", "model.layers.4.mlp.experts.60.gate_proj", "model.layers.4.mlp.experts.61.gate_proj", "model.layers.4.mlp.experts.62.gate_proj", "model.layers.4.mlp.experts.63.gate_proj", "model.layers.4.mlp.experts.64.gate_proj", "model.layers.4.mlp.experts.65.gate_proj", "model.layers.4.mlp.experts.66.gate_proj", "model.layers.4.mlp.experts.67.gate_proj", "model.layers.4.mlp.experts.68.gate_proj", "model.layers.4.mlp.experts.69.gate_proj", "model.layers.4.mlp.experts.70.gate_proj", "model.layers.4.mlp.experts.71.gate_proj", "model.layers.4.mlp.experts.72.gate_proj", "model.layers.4.mlp.experts.73.gate_proj", "model.layers.4.mlp.experts.74.gate_proj", "model.layers.4.mlp.experts.75.gate_proj", "model.layers.4.mlp.experts.76.gate_proj", "model.layers.4.mlp.experts.77.gate_proj", "model.layers.4.mlp.experts.78.gate_proj", "model.layers.4.mlp.experts.79.gate_proj", "model.layers.4.mlp.experts.80.gate_proj", "model.layers.4.mlp.experts.81.gate_proj", "model.layers.4.mlp.experts.82.gate_proj", "model.layers.4.mlp.experts.83.gate_proj", "model.layers.4.mlp.experts.84.gate_proj", "model.layers.4.mlp.experts.85.gate_proj", "model.layers.4.mlp.experts.86.gate_proj", "model.layers.4.mlp.experts.87.gate_proj", "model.layers.4.mlp.experts.88.gate_proj", "model.layers.4.mlp.experts.89.gate_proj", "model.layers.4.mlp.experts.90.gate_proj", "model.layers.4.mlp.experts.91.gate_proj", "model.layers.4.mlp.experts.92.gate_proj", "model.layers.4.mlp.experts.93.gate_proj", "model.layers.4.mlp.experts.94.gate_proj", "model.layers.4.mlp.experts.95.gate_proj", "model.layers.4.mlp.experts.96.gate_proj", "model.layers.4.mlp.experts.97.gate_proj", "model.layers.4.mlp.experts.98.gate_proj", "model.layers.4.mlp.experts.99.gate_proj", "model.layers.4.mlp.experts.100.gate_proj", "model.layers.4.mlp.experts.101.gate_proj", "model.layers.4.mlp.experts.102.gate_proj", "model.layers.4.mlp.experts.103.gate_proj", "model.layers.4.mlp.experts.104.gate_proj", "model.layers.4.mlp.experts.105.gate_proj", "model.layers.4.mlp.experts.106.gate_proj", "model.layers.4.mlp.experts.107.gate_proj", "model.layers.4.mlp.experts.108.gate_proj", "model.layers.4.mlp.experts.109.gate_proj", "model.layers.4.mlp.experts.110.gate_proj", "model.layers.4.mlp.experts.111.gate_proj", "model.layers.4.mlp.experts.112.gate_proj", "model.layers.4.mlp.experts.113.gate_proj", "model.layers.4.mlp.experts.114.gate_proj", "model.layers.4.mlp.experts.115.gate_proj", "model.layers.4.mlp.experts.116.gate_proj", "model.layers.4.mlp.experts.117.gate_proj", "model.layers.4.mlp.experts.118.gate_proj", "model.layers.4.mlp.experts.119.gate_proj", "model.layers.4.mlp.experts.120.gate_proj", "model.layers.4.mlp.experts.121.gate_proj", "model.layers.4.mlp.experts.122.gate_proj", "model.layers.4.mlp.experts.123.gate_proj", "model.layers.4.mlp.experts.124.gate_proj", "model.layers.4.mlp.experts.125.gate_proj", "model.layers.4.mlp.experts.126.gate_proj", "model.layers.4.mlp.experts.127.gate_proj", "model.layers.4.mlp.experts.128.gate_proj", "model.layers.4.mlp.experts.129.gate_proj", "model.layers.4.mlp.experts.130.gate_proj", "model.layers.4.mlp.experts.131.gate_proj", "model.layers.4.mlp.experts.132.gate_proj", "model.layers.4.mlp.experts.133.gate_proj", "model.layers.4.mlp.experts.134.gate_proj", "model.layers.4.mlp.experts.135.gate_proj", "model.layers.4.mlp.experts.136.gate_proj", "model.layers.4.mlp.experts.137.gate_proj", "model.layers.4.mlp.experts.138.gate_proj", "model.layers.4.mlp.experts.139.gate_proj", "model.layers.4.mlp.experts.140.gate_proj", "model.layers.4.mlp.experts.141.gate_proj", "model.layers.4.mlp.experts.142.gate_proj", "model.layers.4.mlp.experts.143.gate_proj", "model.layers.4.mlp.experts.144.gate_proj", "model.layers.4.mlp.experts.145.gate_proj", "model.layers.4.mlp.experts.146.gate_proj", "model.layers.4.mlp.experts.147.gate_proj", "model.layers.4.mlp.experts.148.gate_proj", "model.layers.4.mlp.experts.149.gate_proj", "model.layers.4.mlp.experts.150.gate_proj", "model.layers.4.mlp.experts.151.gate_proj", "model.layers.4.mlp.experts.152.gate_proj", "model.layers.4.mlp.experts.153.gate_proj", "model.layers.4.mlp.experts.154.gate_proj", "model.layers.4.mlp.experts.155.gate_proj", "model.layers.4.mlp.experts.156.gate_proj", "model.layers.4.mlp.experts.157.gate_proj", "model.layers.4.mlp.experts.158.gate_proj", "model.layers.4.mlp.experts.159.gate_proj", "model.layers.4.mlp.experts.0.up_proj", "model.layers.4.mlp.experts.1.up_proj", "model.layers.4.mlp.experts.2.up_proj", "model.layers.4.mlp.experts.3.up_proj", "model.layers.4.mlp.experts.4.up_proj", "model.layers.4.mlp.experts.5.up_proj", "model.layers.4.mlp.experts.6.up_proj", "model.layers.4.mlp.experts.7.up_proj", "model.layers.4.mlp.experts.8.up_proj", "model.layers.4.mlp.experts.9.up_proj", "model.layers.4.mlp.experts.10.up_proj", "model.layers.4.mlp.experts.11.up_proj", "model.layers.4.mlp.experts.12.up_proj", "model.layers.4.mlp.experts.13.up_proj", "model.layers.4.mlp.experts.14.up_proj", "model.layers.4.mlp.experts.15.up_proj", "model.layers.4.mlp.experts.16.up_proj", "model.layers.4.mlp.experts.17.up_proj", "model.layers.4.mlp.experts.18.up_proj", "model.layers.4.mlp.experts.19.up_proj", "model.layers.4.mlp.experts.20.up_proj", "model.layers.4.mlp.experts.21.up_proj", "model.layers.4.mlp.experts.22.up_proj", "model.layers.4.mlp.experts.23.up_proj", "model.layers.4.mlp.experts.24.up_proj", "model.layers.4.mlp.experts.25.up_proj", "model.layers.4.mlp.experts.26.up_proj", "model.layers.4.mlp.experts.27.up_proj", "model.layers.4.mlp.experts.28.up_proj", "model.layers.4.mlp.experts.29.up_proj", "model.layers.4.mlp.experts.30.up_proj", "model.layers.4.mlp.experts.31.up_proj", "model.layers.4.mlp.experts.32.up_proj", "model.layers.4.mlp.experts.33.up_proj", "model.layers.4.mlp.experts.34.up_proj", "model.layers.4.mlp.experts.35.up_proj", "model.layers.4.mlp.experts.36.up_proj", "model.layers.4.mlp.experts.37.up_proj", "model.layers.4.mlp.experts.38.up_proj", "model.layers.4.mlp.experts.39.up_proj", "model.layers.4.mlp.experts.40.up_proj", "model.layers.4.mlp.experts.41.up_proj", "model.layers.4.mlp.experts.42.up_proj", "model.layers.4.mlp.experts.43.up_proj", "model.layers.4.mlp.experts.44.up_proj", "model.layers.4.mlp.experts.45.up_proj", "model.layers.4.mlp.experts.46.up_proj", "model.layers.4.mlp.experts.47.up_proj", "model.layers.4.mlp.experts.48.up_proj", "model.layers.4.mlp.experts.49.up_proj", "model.layers.4.mlp.experts.50.up_proj", "model.layers.4.mlp.experts.51.up_proj", "model.layers.4.mlp.experts.52.up_proj", "model.layers.4.mlp.experts.53.up_proj", "model.layers.4.mlp.experts.54.up_proj", "model.layers.4.mlp.experts.55.up_proj", "model.layers.4.mlp.experts.56.up_proj", "model.layers.4.mlp.experts.57.up_proj", "model.layers.4.mlp.experts.58.up_proj", "model.layers.4.mlp.experts.59.up_proj", "model.layers.4.mlp.experts.60.up_proj", "model.layers.4.mlp.experts.61.up_proj", "model.layers.4.mlp.experts.62.up_proj", "model.layers.4.mlp.experts.63.up_proj", "model.layers.4.mlp.experts.64.up_proj", "model.layers.4.mlp.experts.65.up_proj", "model.layers.4.mlp.experts.66.up_proj", "model.layers.4.mlp.experts.67.up_proj", "model.layers.4.mlp.experts.68.up_proj", "model.layers.4.mlp.experts.69.up_proj", "model.layers.4.mlp.experts.70.up_proj", "model.layers.4.mlp.experts.71.up_proj", "model.layers.4.mlp.experts.72.up_proj", "model.layers.4.mlp.experts.73.up_proj", "model.layers.4.mlp.experts.74.up_proj", "model.layers.4.mlp.experts.75.up_proj", "model.layers.4.mlp.experts.76.up_proj", "model.layers.4.mlp.experts.77.up_proj", "model.layers.4.mlp.experts.78.up_proj", "model.layers.4.mlp.experts.79.up_proj", "model.layers.4.mlp.experts.80.up_proj", "model.layers.4.mlp.experts.81.up_proj", "model.layers.4.mlp.experts.82.up_proj", "model.layers.4.mlp.experts.83.up_proj", "model.layers.4.mlp.experts.84.up_proj", "model.layers.4.mlp.experts.85.up_proj", "model.layers.4.mlp.experts.86.up_proj", "model.layers.4.mlp.experts.87.up_proj", "model.layers.4.mlp.experts.88.up_proj", "model.layers.4.mlp.experts.89.up_proj", "model.layers.4.mlp.experts.90.up_proj", "model.layers.4.mlp.experts.91.up_proj", "model.layers.4.mlp.experts.92.up_proj", "model.layers.4.mlp.experts.93.up_proj", "model.layers.4.mlp.experts.94.up_proj", "model.layers.4.mlp.experts.95.up_proj", "model.layers.4.mlp.experts.96.up_proj", "model.layers.4.mlp.experts.97.up_proj", "model.layers.4.mlp.experts.98.up_proj", "model.layers.4.mlp.experts.99.up_proj", "model.layers.4.mlp.experts.100.up_proj", "model.layers.4.mlp.experts.101.up_proj", "model.layers.4.mlp.experts.102.up_proj", "model.layers.4.mlp.experts.103.up_proj", "model.layers.4.mlp.experts.104.up_proj", "model.layers.4.mlp.experts.105.up_proj", "model.layers.4.mlp.experts.106.up_proj", "model.layers.4.mlp.experts.107.up_proj", "model.layers.4.mlp.experts.108.up_proj", "model.layers.4.mlp.experts.109.up_proj", "model.layers.4.mlp.experts.110.up_proj", "model.layers.4.mlp.experts.111.up_proj", "model.layers.4.mlp.experts.112.up_proj", "model.layers.4.mlp.experts.113.up_proj", "model.layers.4.mlp.experts.114.up_proj", "model.layers.4.mlp.experts.115.up_proj", "model.layers.4.mlp.experts.116.up_proj", "model.layers.4.mlp.experts.117.up_proj", "model.layers.4.mlp.experts.118.up_proj", "model.layers.4.mlp.experts.119.up_proj", "model.layers.4.mlp.experts.120.up_proj", "model.layers.4.mlp.experts.121.up_proj", "model.layers.4.mlp.experts.122.up_proj", "model.layers.4.mlp.experts.123.up_proj", "model.layers.4.mlp.experts.124.up_proj", "model.layers.4.mlp.experts.125.up_proj", "model.layers.4.mlp.experts.126.up_proj", "model.layers.4.mlp.experts.127.up_proj", "model.layers.4.mlp.experts.128.up_proj", "model.layers.4.mlp.experts.129.up_proj", "model.layers.4.mlp.experts.130.up_proj", "model.layers.4.mlp.experts.131.up_proj", "model.layers.4.mlp.experts.132.up_proj", "model.layers.4.mlp.experts.133.up_proj", "model.layers.4.mlp.experts.134.up_proj", "model.layers.4.mlp.experts.135.up_proj", "model.layers.4.mlp.experts.136.up_proj", "model.layers.4.mlp.experts.137.up_proj", "model.layers.4.mlp.experts.138.up_proj", "model.layers.4.mlp.experts.139.up_proj", "model.layers.4.mlp.experts.140.up_proj", "model.layers.4.mlp.experts.141.up_proj", "model.layers.4.mlp.experts.142.up_proj", "model.layers.4.mlp.experts.143.up_proj", "model.layers.4.mlp.experts.144.up_proj", "model.layers.4.mlp.experts.145.up_proj", "model.layers.4.mlp.experts.146.up_proj", "model.layers.4.mlp.experts.147.up_proj", "model.layers.4.mlp.experts.148.up_proj", "model.layers.4.mlp.experts.149.up_proj", "model.layers.4.mlp.experts.150.up_proj", "model.layers.4.mlp.experts.151.up_proj", "model.layers.4.mlp.experts.152.up_proj", "model.layers.4.mlp.experts.153.up_proj", "model.layers.4.mlp.experts.154.up_proj", "model.layers.4.mlp.experts.155.up_proj", "model.layers.4.mlp.experts.156.up_proj", "model.layers.4.mlp.experts.157.up_proj", "model.layers.4.mlp.experts.158.up_proj", "model.layers.4.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0006915442645549774, "dbits": 2516582400 } ] }, { "idx": 26, "layers": [ "model.layers.4.mlp.experts.0.down_proj", "model.layers.4.mlp.experts.1.down_proj", "model.layers.4.mlp.experts.2.down_proj", "model.layers.4.mlp.experts.3.down_proj", "model.layers.4.mlp.experts.4.down_proj", "model.layers.4.mlp.experts.5.down_proj", "model.layers.4.mlp.experts.6.down_proj", "model.layers.4.mlp.experts.7.down_proj", "model.layers.4.mlp.experts.8.down_proj", "model.layers.4.mlp.experts.9.down_proj", "model.layers.4.mlp.experts.10.down_proj", "model.layers.4.mlp.experts.11.down_proj", "model.layers.4.mlp.experts.12.down_proj", "model.layers.4.mlp.experts.13.down_proj", "model.layers.4.mlp.experts.14.down_proj", "model.layers.4.mlp.experts.15.down_proj", "model.layers.4.mlp.experts.16.down_proj", "model.layers.4.mlp.experts.17.down_proj", "model.layers.4.mlp.experts.18.down_proj", "model.layers.4.mlp.experts.19.down_proj", "model.layers.4.mlp.experts.20.down_proj", "model.layers.4.mlp.experts.21.down_proj", "model.layers.4.mlp.experts.22.down_proj", "model.layers.4.mlp.experts.23.down_proj", "model.layers.4.mlp.experts.24.down_proj", "model.layers.4.mlp.experts.25.down_proj", "model.layers.4.mlp.experts.26.down_proj", "model.layers.4.mlp.experts.27.down_proj", "model.layers.4.mlp.experts.28.down_proj", "model.layers.4.mlp.experts.29.down_proj", "model.layers.4.mlp.experts.30.down_proj", "model.layers.4.mlp.experts.31.down_proj", "model.layers.4.mlp.experts.32.down_proj", "model.layers.4.mlp.experts.33.down_proj", "model.layers.4.mlp.experts.34.down_proj", "model.layers.4.mlp.experts.35.down_proj", "model.layers.4.mlp.experts.36.down_proj", "model.layers.4.mlp.experts.37.down_proj", "model.layers.4.mlp.experts.38.down_proj", "model.layers.4.mlp.experts.39.down_proj", "model.layers.4.mlp.experts.40.down_proj", "model.layers.4.mlp.experts.41.down_proj", "model.layers.4.mlp.experts.42.down_proj", "model.layers.4.mlp.experts.43.down_proj", "model.layers.4.mlp.experts.44.down_proj", "model.layers.4.mlp.experts.45.down_proj", "model.layers.4.mlp.experts.46.down_proj", "model.layers.4.mlp.experts.47.down_proj", "model.layers.4.mlp.experts.48.down_proj", "model.layers.4.mlp.experts.49.down_proj", "model.layers.4.mlp.experts.50.down_proj", "model.layers.4.mlp.experts.51.down_proj", "model.layers.4.mlp.experts.52.down_proj", "model.layers.4.mlp.experts.53.down_proj", "model.layers.4.mlp.experts.54.down_proj", "model.layers.4.mlp.experts.55.down_proj", "model.layers.4.mlp.experts.56.down_proj", "model.layers.4.mlp.experts.57.down_proj", "model.layers.4.mlp.experts.58.down_proj", "model.layers.4.mlp.experts.59.down_proj", "model.layers.4.mlp.experts.60.down_proj", "model.layers.4.mlp.experts.61.down_proj", "model.layers.4.mlp.experts.62.down_proj", "model.layers.4.mlp.experts.63.down_proj", "model.layers.4.mlp.experts.64.down_proj", "model.layers.4.mlp.experts.65.down_proj", "model.layers.4.mlp.experts.66.down_proj", "model.layers.4.mlp.experts.67.down_proj", "model.layers.4.mlp.experts.68.down_proj", "model.layers.4.mlp.experts.69.down_proj", "model.layers.4.mlp.experts.70.down_proj", "model.layers.4.mlp.experts.71.down_proj", "model.layers.4.mlp.experts.72.down_proj", "model.layers.4.mlp.experts.73.down_proj", "model.layers.4.mlp.experts.74.down_proj", "model.layers.4.mlp.experts.75.down_proj", "model.layers.4.mlp.experts.76.down_proj", "model.layers.4.mlp.experts.77.down_proj", "model.layers.4.mlp.experts.78.down_proj", "model.layers.4.mlp.experts.79.down_proj", "model.layers.4.mlp.experts.80.down_proj", "model.layers.4.mlp.experts.81.down_proj", "model.layers.4.mlp.experts.82.down_proj", "model.layers.4.mlp.experts.83.down_proj", "model.layers.4.mlp.experts.84.down_proj", "model.layers.4.mlp.experts.85.down_proj", "model.layers.4.mlp.experts.86.down_proj", "model.layers.4.mlp.experts.87.down_proj", "model.layers.4.mlp.experts.88.down_proj", "model.layers.4.mlp.experts.89.down_proj", "model.layers.4.mlp.experts.90.down_proj", "model.layers.4.mlp.experts.91.down_proj", "model.layers.4.mlp.experts.92.down_proj", "model.layers.4.mlp.experts.93.down_proj", "model.layers.4.mlp.experts.94.down_proj", "model.layers.4.mlp.experts.95.down_proj", "model.layers.4.mlp.experts.96.down_proj", "model.layers.4.mlp.experts.97.down_proj", "model.layers.4.mlp.experts.98.down_proj", "model.layers.4.mlp.experts.99.down_proj", "model.layers.4.mlp.experts.100.down_proj", "model.layers.4.mlp.experts.101.down_proj", "model.layers.4.mlp.experts.102.down_proj", "model.layers.4.mlp.experts.103.down_proj", "model.layers.4.mlp.experts.104.down_proj", "model.layers.4.mlp.experts.105.down_proj", "model.layers.4.mlp.experts.106.down_proj", "model.layers.4.mlp.experts.107.down_proj", "model.layers.4.mlp.experts.108.down_proj", "model.layers.4.mlp.experts.109.down_proj", "model.layers.4.mlp.experts.110.down_proj", "model.layers.4.mlp.experts.111.down_proj", "model.layers.4.mlp.experts.112.down_proj", "model.layers.4.mlp.experts.113.down_proj", "model.layers.4.mlp.experts.114.down_proj", "model.layers.4.mlp.experts.115.down_proj", "model.layers.4.mlp.experts.116.down_proj", "model.layers.4.mlp.experts.117.down_proj", "model.layers.4.mlp.experts.118.down_proj", "model.layers.4.mlp.experts.119.down_proj", "model.layers.4.mlp.experts.120.down_proj", "model.layers.4.mlp.experts.121.down_proj", "model.layers.4.mlp.experts.122.down_proj", "model.layers.4.mlp.experts.123.down_proj", "model.layers.4.mlp.experts.124.down_proj", "model.layers.4.mlp.experts.125.down_proj", "model.layers.4.mlp.experts.126.down_proj", "model.layers.4.mlp.experts.127.down_proj", "model.layers.4.mlp.experts.128.down_proj", "model.layers.4.mlp.experts.129.down_proj", "model.layers.4.mlp.experts.130.down_proj", "model.layers.4.mlp.experts.131.down_proj", "model.layers.4.mlp.experts.132.down_proj", "model.layers.4.mlp.experts.133.down_proj", "model.layers.4.mlp.experts.134.down_proj", "model.layers.4.mlp.experts.135.down_proj", "model.layers.4.mlp.experts.136.down_proj", "model.layers.4.mlp.experts.137.down_proj", "model.layers.4.mlp.experts.138.down_proj", "model.layers.4.mlp.experts.139.down_proj", "model.layers.4.mlp.experts.140.down_proj", "model.layers.4.mlp.experts.141.down_proj", "model.layers.4.mlp.experts.142.down_proj", "model.layers.4.mlp.experts.143.down_proj", "model.layers.4.mlp.experts.144.down_proj", "model.layers.4.mlp.experts.145.down_proj", "model.layers.4.mlp.experts.146.down_proj", "model.layers.4.mlp.experts.147.down_proj", "model.layers.4.mlp.experts.148.down_proj", "model.layers.4.mlp.experts.149.down_proj", "model.layers.4.mlp.experts.150.down_proj", "model.layers.4.mlp.experts.151.down_proj", "model.layers.4.mlp.experts.152.down_proj", "model.layers.4.mlp.experts.153.down_proj", "model.layers.4.mlp.experts.154.down_proj", "model.layers.4.mlp.experts.155.down_proj", "model.layers.4.mlp.experts.156.down_proj", "model.layers.4.mlp.experts.157.down_proj", "model.layers.4.mlp.experts.158.down_proj", "model.layers.4.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006203711032867765, "dbits": 1258291200 } ] }, { "idx": 27, "layers": [ "model.layers.5.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0001385316252708324, "dbits": 62914560 } ] }, { "idx": 28, "layers": [ "model.layers.5.self_attn.k_proj", "model.layers.5.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0005419872701168393, "dbits": 10485760 } ] }, { "idx": 29, "layers": [ "model.layers.5.self_attn.o_proj" ], "candidates": [ { "dkld": -0.001318069547414813, "dbits": 62914560 } ] }, { "idx": 30, "layers": [ "model.layers.5.mlp.shared_experts.gate_proj", "model.layers.5.mlp.shared_experts.up_proj", "model.layers.5.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0012716814875602944, "dbits": 23592960 } ] }, { "idx": 31, "layers": [ "model.layers.5.mlp.experts.0.gate_proj", "model.layers.5.mlp.experts.1.gate_proj", "model.layers.5.mlp.experts.2.gate_proj", "model.layers.5.mlp.experts.3.gate_proj", "model.layers.5.mlp.experts.4.gate_proj", "model.layers.5.mlp.experts.5.gate_proj", "model.layers.5.mlp.experts.6.gate_proj", "model.layers.5.mlp.experts.7.gate_proj", "model.layers.5.mlp.experts.8.gate_proj", "model.layers.5.mlp.experts.9.gate_proj", "model.layers.5.mlp.experts.10.gate_proj", "model.layers.5.mlp.experts.11.gate_proj", "model.layers.5.mlp.experts.12.gate_proj", "model.layers.5.mlp.experts.13.gate_proj", "model.layers.5.mlp.experts.14.gate_proj", "model.layers.5.mlp.experts.15.gate_proj", "model.layers.5.mlp.experts.16.gate_proj", "model.layers.5.mlp.experts.17.gate_proj", "model.layers.5.mlp.experts.18.gate_proj", "model.layers.5.mlp.experts.19.gate_proj", "model.layers.5.mlp.experts.20.gate_proj", "model.layers.5.mlp.experts.21.gate_proj", "model.layers.5.mlp.experts.22.gate_proj", "model.layers.5.mlp.experts.23.gate_proj", "model.layers.5.mlp.experts.24.gate_proj", "model.layers.5.mlp.experts.25.gate_proj", "model.layers.5.mlp.experts.26.gate_proj", "model.layers.5.mlp.experts.27.gate_proj", "model.layers.5.mlp.experts.28.gate_proj", "model.layers.5.mlp.experts.29.gate_proj", "model.layers.5.mlp.experts.30.gate_proj", "model.layers.5.mlp.experts.31.gate_proj", "model.layers.5.mlp.experts.32.gate_proj", "model.layers.5.mlp.experts.33.gate_proj", "model.layers.5.mlp.experts.34.gate_proj", "model.layers.5.mlp.experts.35.gate_proj", "model.layers.5.mlp.experts.36.gate_proj", "model.layers.5.mlp.experts.37.gate_proj", "model.layers.5.mlp.experts.38.gate_proj", "model.layers.5.mlp.experts.39.gate_proj", "model.layers.5.mlp.experts.40.gate_proj", "model.layers.5.mlp.experts.41.gate_proj", "model.layers.5.mlp.experts.42.gate_proj", "model.layers.5.mlp.experts.43.gate_proj", "model.layers.5.mlp.experts.44.gate_proj", "model.layers.5.mlp.experts.45.gate_proj", "model.layers.5.mlp.experts.46.gate_proj", "model.layers.5.mlp.experts.47.gate_proj", "model.layers.5.mlp.experts.48.gate_proj", "model.layers.5.mlp.experts.49.gate_proj", "model.layers.5.mlp.experts.50.gate_proj", "model.layers.5.mlp.experts.51.gate_proj", "model.layers.5.mlp.experts.52.gate_proj", "model.layers.5.mlp.experts.53.gate_proj", "model.layers.5.mlp.experts.54.gate_proj", "model.layers.5.mlp.experts.55.gate_proj", "model.layers.5.mlp.experts.56.gate_proj", "model.layers.5.mlp.experts.57.gate_proj", "model.layers.5.mlp.experts.58.gate_proj", "model.layers.5.mlp.experts.59.gate_proj", "model.layers.5.mlp.experts.60.gate_proj", "model.layers.5.mlp.experts.61.gate_proj", "model.layers.5.mlp.experts.62.gate_proj", "model.layers.5.mlp.experts.63.gate_proj", "model.layers.5.mlp.experts.64.gate_proj", "model.layers.5.mlp.experts.65.gate_proj", "model.layers.5.mlp.experts.66.gate_proj", "model.layers.5.mlp.experts.67.gate_proj", "model.layers.5.mlp.experts.68.gate_proj", "model.layers.5.mlp.experts.69.gate_proj", "model.layers.5.mlp.experts.70.gate_proj", "model.layers.5.mlp.experts.71.gate_proj", "model.layers.5.mlp.experts.72.gate_proj", "model.layers.5.mlp.experts.73.gate_proj", "model.layers.5.mlp.experts.74.gate_proj", "model.layers.5.mlp.experts.75.gate_proj", "model.layers.5.mlp.experts.76.gate_proj", "model.layers.5.mlp.experts.77.gate_proj", "model.layers.5.mlp.experts.78.gate_proj", "model.layers.5.mlp.experts.79.gate_proj", "model.layers.5.mlp.experts.80.gate_proj", "model.layers.5.mlp.experts.81.gate_proj", "model.layers.5.mlp.experts.82.gate_proj", "model.layers.5.mlp.experts.83.gate_proj", "model.layers.5.mlp.experts.84.gate_proj", "model.layers.5.mlp.experts.85.gate_proj", "model.layers.5.mlp.experts.86.gate_proj", "model.layers.5.mlp.experts.87.gate_proj", "model.layers.5.mlp.experts.88.gate_proj", "model.layers.5.mlp.experts.89.gate_proj", "model.layers.5.mlp.experts.90.gate_proj", "model.layers.5.mlp.experts.91.gate_proj", "model.layers.5.mlp.experts.92.gate_proj", "model.layers.5.mlp.experts.93.gate_proj", "model.layers.5.mlp.experts.94.gate_proj", "model.layers.5.mlp.experts.95.gate_proj", "model.layers.5.mlp.experts.96.gate_proj", "model.layers.5.mlp.experts.97.gate_proj", "model.layers.5.mlp.experts.98.gate_proj", "model.layers.5.mlp.experts.99.gate_proj", "model.layers.5.mlp.experts.100.gate_proj", "model.layers.5.mlp.experts.101.gate_proj", "model.layers.5.mlp.experts.102.gate_proj", "model.layers.5.mlp.experts.103.gate_proj", "model.layers.5.mlp.experts.104.gate_proj", "model.layers.5.mlp.experts.105.gate_proj", "model.layers.5.mlp.experts.106.gate_proj", "model.layers.5.mlp.experts.107.gate_proj", "model.layers.5.mlp.experts.108.gate_proj", "model.layers.5.mlp.experts.109.gate_proj", "model.layers.5.mlp.experts.110.gate_proj", "model.layers.5.mlp.experts.111.gate_proj", "model.layers.5.mlp.experts.112.gate_proj", "model.layers.5.mlp.experts.113.gate_proj", "model.layers.5.mlp.experts.114.gate_proj", "model.layers.5.mlp.experts.115.gate_proj", "model.layers.5.mlp.experts.116.gate_proj", "model.layers.5.mlp.experts.117.gate_proj", "model.layers.5.mlp.experts.118.gate_proj", "model.layers.5.mlp.experts.119.gate_proj", "model.layers.5.mlp.experts.120.gate_proj", "model.layers.5.mlp.experts.121.gate_proj", "model.layers.5.mlp.experts.122.gate_proj", "model.layers.5.mlp.experts.123.gate_proj", "model.layers.5.mlp.experts.124.gate_proj", "model.layers.5.mlp.experts.125.gate_proj", "model.layers.5.mlp.experts.126.gate_proj", "model.layers.5.mlp.experts.127.gate_proj", "model.layers.5.mlp.experts.128.gate_proj", "model.layers.5.mlp.experts.129.gate_proj", "model.layers.5.mlp.experts.130.gate_proj", "model.layers.5.mlp.experts.131.gate_proj", "model.layers.5.mlp.experts.132.gate_proj", "model.layers.5.mlp.experts.133.gate_proj", "model.layers.5.mlp.experts.134.gate_proj", "model.layers.5.mlp.experts.135.gate_proj", "model.layers.5.mlp.experts.136.gate_proj", "model.layers.5.mlp.experts.137.gate_proj", "model.layers.5.mlp.experts.138.gate_proj", "model.layers.5.mlp.experts.139.gate_proj", "model.layers.5.mlp.experts.140.gate_proj", "model.layers.5.mlp.experts.141.gate_proj", "model.layers.5.mlp.experts.142.gate_proj", "model.layers.5.mlp.experts.143.gate_proj", "model.layers.5.mlp.experts.144.gate_proj", "model.layers.5.mlp.experts.145.gate_proj", "model.layers.5.mlp.experts.146.gate_proj", "model.layers.5.mlp.experts.147.gate_proj", "model.layers.5.mlp.experts.148.gate_proj", "model.layers.5.mlp.experts.149.gate_proj", "model.layers.5.mlp.experts.150.gate_proj", "model.layers.5.mlp.experts.151.gate_proj", "model.layers.5.mlp.experts.152.gate_proj", "model.layers.5.mlp.experts.153.gate_proj", "model.layers.5.mlp.experts.154.gate_proj", "model.layers.5.mlp.experts.155.gate_proj", "model.layers.5.mlp.experts.156.gate_proj", "model.layers.5.mlp.experts.157.gate_proj", "model.layers.5.mlp.experts.158.gate_proj", "model.layers.5.mlp.experts.159.gate_proj", "model.layers.5.mlp.experts.0.up_proj", "model.layers.5.mlp.experts.1.up_proj", "model.layers.5.mlp.experts.2.up_proj", "model.layers.5.mlp.experts.3.up_proj", "model.layers.5.mlp.experts.4.up_proj", "model.layers.5.mlp.experts.5.up_proj", "model.layers.5.mlp.experts.6.up_proj", "model.layers.5.mlp.experts.7.up_proj", "model.layers.5.mlp.experts.8.up_proj", "model.layers.5.mlp.experts.9.up_proj", "model.layers.5.mlp.experts.10.up_proj", "model.layers.5.mlp.experts.11.up_proj", "model.layers.5.mlp.experts.12.up_proj", "model.layers.5.mlp.experts.13.up_proj", "model.layers.5.mlp.experts.14.up_proj", "model.layers.5.mlp.experts.15.up_proj", "model.layers.5.mlp.experts.16.up_proj", "model.layers.5.mlp.experts.17.up_proj", "model.layers.5.mlp.experts.18.up_proj", "model.layers.5.mlp.experts.19.up_proj", "model.layers.5.mlp.experts.20.up_proj", "model.layers.5.mlp.experts.21.up_proj", "model.layers.5.mlp.experts.22.up_proj", "model.layers.5.mlp.experts.23.up_proj", "model.layers.5.mlp.experts.24.up_proj", "model.layers.5.mlp.experts.25.up_proj", "model.layers.5.mlp.experts.26.up_proj", "model.layers.5.mlp.experts.27.up_proj", "model.layers.5.mlp.experts.28.up_proj", "model.layers.5.mlp.experts.29.up_proj", "model.layers.5.mlp.experts.30.up_proj", "model.layers.5.mlp.experts.31.up_proj", "model.layers.5.mlp.experts.32.up_proj", "model.layers.5.mlp.experts.33.up_proj", "model.layers.5.mlp.experts.34.up_proj", "model.layers.5.mlp.experts.35.up_proj", "model.layers.5.mlp.experts.36.up_proj", "model.layers.5.mlp.experts.37.up_proj", "model.layers.5.mlp.experts.38.up_proj", "model.layers.5.mlp.experts.39.up_proj", "model.layers.5.mlp.experts.40.up_proj", "model.layers.5.mlp.experts.41.up_proj", "model.layers.5.mlp.experts.42.up_proj", "model.layers.5.mlp.experts.43.up_proj", "model.layers.5.mlp.experts.44.up_proj", "model.layers.5.mlp.experts.45.up_proj", "model.layers.5.mlp.experts.46.up_proj", "model.layers.5.mlp.experts.47.up_proj", "model.layers.5.mlp.experts.48.up_proj", "model.layers.5.mlp.experts.49.up_proj", "model.layers.5.mlp.experts.50.up_proj", "model.layers.5.mlp.experts.51.up_proj", "model.layers.5.mlp.experts.52.up_proj", "model.layers.5.mlp.experts.53.up_proj", "model.layers.5.mlp.experts.54.up_proj", "model.layers.5.mlp.experts.55.up_proj", "model.layers.5.mlp.experts.56.up_proj", "model.layers.5.mlp.experts.57.up_proj", "model.layers.5.mlp.experts.58.up_proj", "model.layers.5.mlp.experts.59.up_proj", "model.layers.5.mlp.experts.60.up_proj", "model.layers.5.mlp.experts.61.up_proj", "model.layers.5.mlp.experts.62.up_proj", "model.layers.5.mlp.experts.63.up_proj", "model.layers.5.mlp.experts.64.up_proj", "model.layers.5.mlp.experts.65.up_proj", "model.layers.5.mlp.experts.66.up_proj", "model.layers.5.mlp.experts.67.up_proj", "model.layers.5.mlp.experts.68.up_proj", "model.layers.5.mlp.experts.69.up_proj", "model.layers.5.mlp.experts.70.up_proj", "model.layers.5.mlp.experts.71.up_proj", "model.layers.5.mlp.experts.72.up_proj", "model.layers.5.mlp.experts.73.up_proj", "model.layers.5.mlp.experts.74.up_proj", "model.layers.5.mlp.experts.75.up_proj", "model.layers.5.mlp.experts.76.up_proj", "model.layers.5.mlp.experts.77.up_proj", "model.layers.5.mlp.experts.78.up_proj", "model.layers.5.mlp.experts.79.up_proj", "model.layers.5.mlp.experts.80.up_proj", "model.layers.5.mlp.experts.81.up_proj", "model.layers.5.mlp.experts.82.up_proj", "model.layers.5.mlp.experts.83.up_proj", "model.layers.5.mlp.experts.84.up_proj", "model.layers.5.mlp.experts.85.up_proj", "model.layers.5.mlp.experts.86.up_proj", "model.layers.5.mlp.experts.87.up_proj", "model.layers.5.mlp.experts.88.up_proj", "model.layers.5.mlp.experts.89.up_proj", "model.layers.5.mlp.experts.90.up_proj", "model.layers.5.mlp.experts.91.up_proj", "model.layers.5.mlp.experts.92.up_proj", "model.layers.5.mlp.experts.93.up_proj", "model.layers.5.mlp.experts.94.up_proj", "model.layers.5.mlp.experts.95.up_proj", "model.layers.5.mlp.experts.96.up_proj", "model.layers.5.mlp.experts.97.up_proj", "model.layers.5.mlp.experts.98.up_proj", "model.layers.5.mlp.experts.99.up_proj", "model.layers.5.mlp.experts.100.up_proj", "model.layers.5.mlp.experts.101.up_proj", "model.layers.5.mlp.experts.102.up_proj", "model.layers.5.mlp.experts.103.up_proj", "model.layers.5.mlp.experts.104.up_proj", "model.layers.5.mlp.experts.105.up_proj", "model.layers.5.mlp.experts.106.up_proj", "model.layers.5.mlp.experts.107.up_proj", "model.layers.5.mlp.experts.108.up_proj", "model.layers.5.mlp.experts.109.up_proj", "model.layers.5.mlp.experts.110.up_proj", "model.layers.5.mlp.experts.111.up_proj", "model.layers.5.mlp.experts.112.up_proj", "model.layers.5.mlp.experts.113.up_proj", "model.layers.5.mlp.experts.114.up_proj", "model.layers.5.mlp.experts.115.up_proj", "model.layers.5.mlp.experts.116.up_proj", "model.layers.5.mlp.experts.117.up_proj", "model.layers.5.mlp.experts.118.up_proj", "model.layers.5.mlp.experts.119.up_proj", "model.layers.5.mlp.experts.120.up_proj", "model.layers.5.mlp.experts.121.up_proj", "model.layers.5.mlp.experts.122.up_proj", "model.layers.5.mlp.experts.123.up_proj", "model.layers.5.mlp.experts.124.up_proj", "model.layers.5.mlp.experts.125.up_proj", "model.layers.5.mlp.experts.126.up_proj", "model.layers.5.mlp.experts.127.up_proj", "model.layers.5.mlp.experts.128.up_proj", "model.layers.5.mlp.experts.129.up_proj", "model.layers.5.mlp.experts.130.up_proj", "model.layers.5.mlp.experts.131.up_proj", "model.layers.5.mlp.experts.132.up_proj", "model.layers.5.mlp.experts.133.up_proj", "model.layers.5.mlp.experts.134.up_proj", "model.layers.5.mlp.experts.135.up_proj", "model.layers.5.mlp.experts.136.up_proj", "model.layers.5.mlp.experts.137.up_proj", "model.layers.5.mlp.experts.138.up_proj", "model.layers.5.mlp.experts.139.up_proj", "model.layers.5.mlp.experts.140.up_proj", "model.layers.5.mlp.experts.141.up_proj", "model.layers.5.mlp.experts.142.up_proj", "model.layers.5.mlp.experts.143.up_proj", "model.layers.5.mlp.experts.144.up_proj", "model.layers.5.mlp.experts.145.up_proj", "model.layers.5.mlp.experts.146.up_proj", "model.layers.5.mlp.experts.147.up_proj", "model.layers.5.mlp.experts.148.up_proj", "model.layers.5.mlp.experts.149.up_proj", "model.layers.5.mlp.experts.150.up_proj", "model.layers.5.mlp.experts.151.up_proj", "model.layers.5.mlp.experts.152.up_proj", "model.layers.5.mlp.experts.153.up_proj", "model.layers.5.mlp.experts.154.up_proj", "model.layers.5.mlp.experts.155.up_proj", "model.layers.5.mlp.experts.156.up_proj", "model.layers.5.mlp.experts.157.up_proj", "model.layers.5.mlp.experts.158.up_proj", "model.layers.5.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00021215304732324913, "dbits": 2516582400 } ] }, { "idx": 32, "layers": [ "model.layers.5.mlp.experts.0.down_proj", "model.layers.5.mlp.experts.1.down_proj", "model.layers.5.mlp.experts.2.down_proj", "model.layers.5.mlp.experts.3.down_proj", "model.layers.5.mlp.experts.4.down_proj", "model.layers.5.mlp.experts.5.down_proj", "model.layers.5.mlp.experts.6.down_proj", "model.layers.5.mlp.experts.7.down_proj", "model.layers.5.mlp.experts.8.down_proj", "model.layers.5.mlp.experts.9.down_proj", "model.layers.5.mlp.experts.10.down_proj", "model.layers.5.mlp.experts.11.down_proj", "model.layers.5.mlp.experts.12.down_proj", "model.layers.5.mlp.experts.13.down_proj", "model.layers.5.mlp.experts.14.down_proj", "model.layers.5.mlp.experts.15.down_proj", "model.layers.5.mlp.experts.16.down_proj", "model.layers.5.mlp.experts.17.down_proj", "model.layers.5.mlp.experts.18.down_proj", "model.layers.5.mlp.experts.19.down_proj", "model.layers.5.mlp.experts.20.down_proj", "model.layers.5.mlp.experts.21.down_proj", "model.layers.5.mlp.experts.22.down_proj", "model.layers.5.mlp.experts.23.down_proj", "model.layers.5.mlp.experts.24.down_proj", "model.layers.5.mlp.experts.25.down_proj", "model.layers.5.mlp.experts.26.down_proj", "model.layers.5.mlp.experts.27.down_proj", "model.layers.5.mlp.experts.28.down_proj", "model.layers.5.mlp.experts.29.down_proj", "model.layers.5.mlp.experts.30.down_proj", "model.layers.5.mlp.experts.31.down_proj", "model.layers.5.mlp.experts.32.down_proj", "model.layers.5.mlp.experts.33.down_proj", "model.layers.5.mlp.experts.34.down_proj", "model.layers.5.mlp.experts.35.down_proj", "model.layers.5.mlp.experts.36.down_proj", "model.layers.5.mlp.experts.37.down_proj", "model.layers.5.mlp.experts.38.down_proj", "model.layers.5.mlp.experts.39.down_proj", "model.layers.5.mlp.experts.40.down_proj", "model.layers.5.mlp.experts.41.down_proj", "model.layers.5.mlp.experts.42.down_proj", "model.layers.5.mlp.experts.43.down_proj", "model.layers.5.mlp.experts.44.down_proj", "model.layers.5.mlp.experts.45.down_proj", "model.layers.5.mlp.experts.46.down_proj", "model.layers.5.mlp.experts.47.down_proj", "model.layers.5.mlp.experts.48.down_proj", "model.layers.5.mlp.experts.49.down_proj", "model.layers.5.mlp.experts.50.down_proj", "model.layers.5.mlp.experts.51.down_proj", "model.layers.5.mlp.experts.52.down_proj", "model.layers.5.mlp.experts.53.down_proj", "model.layers.5.mlp.experts.54.down_proj", "model.layers.5.mlp.experts.55.down_proj", "model.layers.5.mlp.experts.56.down_proj", "model.layers.5.mlp.experts.57.down_proj", "model.layers.5.mlp.experts.58.down_proj", "model.layers.5.mlp.experts.59.down_proj", "model.layers.5.mlp.experts.60.down_proj", "model.layers.5.mlp.experts.61.down_proj", "model.layers.5.mlp.experts.62.down_proj", "model.layers.5.mlp.experts.63.down_proj", "model.layers.5.mlp.experts.64.down_proj", "model.layers.5.mlp.experts.65.down_proj", "model.layers.5.mlp.experts.66.down_proj", "model.layers.5.mlp.experts.67.down_proj", "model.layers.5.mlp.experts.68.down_proj", "model.layers.5.mlp.experts.69.down_proj", "model.layers.5.mlp.experts.70.down_proj", "model.layers.5.mlp.experts.71.down_proj", "model.layers.5.mlp.experts.72.down_proj", "model.layers.5.mlp.experts.73.down_proj", "model.layers.5.mlp.experts.74.down_proj", "model.layers.5.mlp.experts.75.down_proj", "model.layers.5.mlp.experts.76.down_proj", "model.layers.5.mlp.experts.77.down_proj", "model.layers.5.mlp.experts.78.down_proj", "model.layers.5.mlp.experts.79.down_proj", "model.layers.5.mlp.experts.80.down_proj", "model.layers.5.mlp.experts.81.down_proj", "model.layers.5.mlp.experts.82.down_proj", "model.layers.5.mlp.experts.83.down_proj", "model.layers.5.mlp.experts.84.down_proj", "model.layers.5.mlp.experts.85.down_proj", "model.layers.5.mlp.experts.86.down_proj", "model.layers.5.mlp.experts.87.down_proj", "model.layers.5.mlp.experts.88.down_proj", "model.layers.5.mlp.experts.89.down_proj", "model.layers.5.mlp.experts.90.down_proj", "model.layers.5.mlp.experts.91.down_proj", "model.layers.5.mlp.experts.92.down_proj", "model.layers.5.mlp.experts.93.down_proj", "model.layers.5.mlp.experts.94.down_proj", "model.layers.5.mlp.experts.95.down_proj", "model.layers.5.mlp.experts.96.down_proj", "model.layers.5.mlp.experts.97.down_proj", "model.layers.5.mlp.experts.98.down_proj", "model.layers.5.mlp.experts.99.down_proj", "model.layers.5.mlp.experts.100.down_proj", "model.layers.5.mlp.experts.101.down_proj", "model.layers.5.mlp.experts.102.down_proj", "model.layers.5.mlp.experts.103.down_proj", "model.layers.5.mlp.experts.104.down_proj", "model.layers.5.mlp.experts.105.down_proj", "model.layers.5.mlp.experts.106.down_proj", "model.layers.5.mlp.experts.107.down_proj", "model.layers.5.mlp.experts.108.down_proj", "model.layers.5.mlp.experts.109.down_proj", "model.layers.5.mlp.experts.110.down_proj", "model.layers.5.mlp.experts.111.down_proj", "model.layers.5.mlp.experts.112.down_proj", "model.layers.5.mlp.experts.113.down_proj", "model.layers.5.mlp.experts.114.down_proj", "model.layers.5.mlp.experts.115.down_proj", "model.layers.5.mlp.experts.116.down_proj", "model.layers.5.mlp.experts.117.down_proj", "model.layers.5.mlp.experts.118.down_proj", "model.layers.5.mlp.experts.119.down_proj", "model.layers.5.mlp.experts.120.down_proj", "model.layers.5.mlp.experts.121.down_proj", "model.layers.5.mlp.experts.122.down_proj", "model.layers.5.mlp.experts.123.down_proj", "model.layers.5.mlp.experts.124.down_proj", "model.layers.5.mlp.experts.125.down_proj", "model.layers.5.mlp.experts.126.down_proj", "model.layers.5.mlp.experts.127.down_proj", "model.layers.5.mlp.experts.128.down_proj", "model.layers.5.mlp.experts.129.down_proj", "model.layers.5.mlp.experts.130.down_proj", "model.layers.5.mlp.experts.131.down_proj", "model.layers.5.mlp.experts.132.down_proj", "model.layers.5.mlp.experts.133.down_proj", "model.layers.5.mlp.experts.134.down_proj", "model.layers.5.mlp.experts.135.down_proj", "model.layers.5.mlp.experts.136.down_proj", "model.layers.5.mlp.experts.137.down_proj", "model.layers.5.mlp.experts.138.down_proj", "model.layers.5.mlp.experts.139.down_proj", "model.layers.5.mlp.experts.140.down_proj", "model.layers.5.mlp.experts.141.down_proj", "model.layers.5.mlp.experts.142.down_proj", "model.layers.5.mlp.experts.143.down_proj", "model.layers.5.mlp.experts.144.down_proj", "model.layers.5.mlp.experts.145.down_proj", "model.layers.5.mlp.experts.146.down_proj", "model.layers.5.mlp.experts.147.down_proj", "model.layers.5.mlp.experts.148.down_proj", "model.layers.5.mlp.experts.149.down_proj", "model.layers.5.mlp.experts.150.down_proj", "model.layers.5.mlp.experts.151.down_proj", "model.layers.5.mlp.experts.152.down_proj", "model.layers.5.mlp.experts.153.down_proj", "model.layers.5.mlp.experts.154.down_proj", "model.layers.5.mlp.experts.155.down_proj", "model.layers.5.mlp.experts.156.down_proj", "model.layers.5.mlp.experts.157.down_proj", "model.layers.5.mlp.experts.158.down_proj", "model.layers.5.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.000488993525505077, "dbits": 1258291200 } ] }, { "idx": 33, "layers": [ "model.layers.6.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0010719381272792927, "dbits": 62914560 } ] }, { "idx": 34, "layers": [ "model.layers.6.self_attn.k_proj", "model.layers.6.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0012618251144885795, "dbits": 10485760 } ] }, { "idx": 35, "layers": [ "model.layers.6.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0007316976785660123, "dbits": 62914560 } ] }, { "idx": 36, "layers": [ "model.layers.6.mlp.shared_experts.gate_proj", "model.layers.6.mlp.shared_experts.up_proj", "model.layers.6.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0010945320129394642, "dbits": 23592960 } ] }, { "idx": 37, "layers": [ "model.layers.6.mlp.experts.0.gate_proj", "model.layers.6.mlp.experts.1.gate_proj", "model.layers.6.mlp.experts.2.gate_proj", "model.layers.6.mlp.experts.3.gate_proj", "model.layers.6.mlp.experts.4.gate_proj", "model.layers.6.mlp.experts.5.gate_proj", "model.layers.6.mlp.experts.6.gate_proj", "model.layers.6.mlp.experts.7.gate_proj", "model.layers.6.mlp.experts.8.gate_proj", "model.layers.6.mlp.experts.9.gate_proj", "model.layers.6.mlp.experts.10.gate_proj", "model.layers.6.mlp.experts.11.gate_proj", "model.layers.6.mlp.experts.12.gate_proj", "model.layers.6.mlp.experts.13.gate_proj", "model.layers.6.mlp.experts.14.gate_proj", "model.layers.6.mlp.experts.15.gate_proj", "model.layers.6.mlp.experts.16.gate_proj", "model.layers.6.mlp.experts.17.gate_proj", "model.layers.6.mlp.experts.18.gate_proj", "model.layers.6.mlp.experts.19.gate_proj", "model.layers.6.mlp.experts.20.gate_proj", "model.layers.6.mlp.experts.21.gate_proj", "model.layers.6.mlp.experts.22.gate_proj", "model.layers.6.mlp.experts.23.gate_proj", "model.layers.6.mlp.experts.24.gate_proj", "model.layers.6.mlp.experts.25.gate_proj", "model.layers.6.mlp.experts.26.gate_proj", "model.layers.6.mlp.experts.27.gate_proj", "model.layers.6.mlp.experts.28.gate_proj", "model.layers.6.mlp.experts.29.gate_proj", "model.layers.6.mlp.experts.30.gate_proj", "model.layers.6.mlp.experts.31.gate_proj", "model.layers.6.mlp.experts.32.gate_proj", "model.layers.6.mlp.experts.33.gate_proj", "model.layers.6.mlp.experts.34.gate_proj", "model.layers.6.mlp.experts.35.gate_proj", "model.layers.6.mlp.experts.36.gate_proj", "model.layers.6.mlp.experts.37.gate_proj", "model.layers.6.mlp.experts.38.gate_proj", "model.layers.6.mlp.experts.39.gate_proj", "model.layers.6.mlp.experts.40.gate_proj", "model.layers.6.mlp.experts.41.gate_proj", "model.layers.6.mlp.experts.42.gate_proj", "model.layers.6.mlp.experts.43.gate_proj", "model.layers.6.mlp.experts.44.gate_proj", "model.layers.6.mlp.experts.45.gate_proj", "model.layers.6.mlp.experts.46.gate_proj", "model.layers.6.mlp.experts.47.gate_proj", "model.layers.6.mlp.experts.48.gate_proj", "model.layers.6.mlp.experts.49.gate_proj", "model.layers.6.mlp.experts.50.gate_proj", "model.layers.6.mlp.experts.51.gate_proj", "model.layers.6.mlp.experts.52.gate_proj", "model.layers.6.mlp.experts.53.gate_proj", "model.layers.6.mlp.experts.54.gate_proj", "model.layers.6.mlp.experts.55.gate_proj", "model.layers.6.mlp.experts.56.gate_proj", "model.layers.6.mlp.experts.57.gate_proj", "model.layers.6.mlp.experts.58.gate_proj", "model.layers.6.mlp.experts.59.gate_proj", "model.layers.6.mlp.experts.60.gate_proj", "model.layers.6.mlp.experts.61.gate_proj", "model.layers.6.mlp.experts.62.gate_proj", "model.layers.6.mlp.experts.63.gate_proj", "model.layers.6.mlp.experts.64.gate_proj", "model.layers.6.mlp.experts.65.gate_proj", "model.layers.6.mlp.experts.66.gate_proj", "model.layers.6.mlp.experts.67.gate_proj", "model.layers.6.mlp.experts.68.gate_proj", "model.layers.6.mlp.experts.69.gate_proj", "model.layers.6.mlp.experts.70.gate_proj", "model.layers.6.mlp.experts.71.gate_proj", "model.layers.6.mlp.experts.72.gate_proj", "model.layers.6.mlp.experts.73.gate_proj", "model.layers.6.mlp.experts.74.gate_proj", "model.layers.6.mlp.experts.75.gate_proj", "model.layers.6.mlp.experts.76.gate_proj", "model.layers.6.mlp.experts.77.gate_proj", "model.layers.6.mlp.experts.78.gate_proj", "model.layers.6.mlp.experts.79.gate_proj", "model.layers.6.mlp.experts.80.gate_proj", "model.layers.6.mlp.experts.81.gate_proj", "model.layers.6.mlp.experts.82.gate_proj", "model.layers.6.mlp.experts.83.gate_proj", "model.layers.6.mlp.experts.84.gate_proj", "model.layers.6.mlp.experts.85.gate_proj", "model.layers.6.mlp.experts.86.gate_proj", "model.layers.6.mlp.experts.87.gate_proj", "model.layers.6.mlp.experts.88.gate_proj", "model.layers.6.mlp.experts.89.gate_proj", "model.layers.6.mlp.experts.90.gate_proj", "model.layers.6.mlp.experts.91.gate_proj", "model.layers.6.mlp.experts.92.gate_proj", "model.layers.6.mlp.experts.93.gate_proj", "model.layers.6.mlp.experts.94.gate_proj", "model.layers.6.mlp.experts.95.gate_proj", "model.layers.6.mlp.experts.96.gate_proj", "model.layers.6.mlp.experts.97.gate_proj", "model.layers.6.mlp.experts.98.gate_proj", "model.layers.6.mlp.experts.99.gate_proj", "model.layers.6.mlp.experts.100.gate_proj", "model.layers.6.mlp.experts.101.gate_proj", "model.layers.6.mlp.experts.102.gate_proj", "model.layers.6.mlp.experts.103.gate_proj", "model.layers.6.mlp.experts.104.gate_proj", "model.layers.6.mlp.experts.105.gate_proj", "model.layers.6.mlp.experts.106.gate_proj", "model.layers.6.mlp.experts.107.gate_proj", "model.layers.6.mlp.experts.108.gate_proj", "model.layers.6.mlp.experts.109.gate_proj", "model.layers.6.mlp.experts.110.gate_proj", "model.layers.6.mlp.experts.111.gate_proj", "model.layers.6.mlp.experts.112.gate_proj", "model.layers.6.mlp.experts.113.gate_proj", "model.layers.6.mlp.experts.114.gate_proj", "model.layers.6.mlp.experts.115.gate_proj", "model.layers.6.mlp.experts.116.gate_proj", "model.layers.6.mlp.experts.117.gate_proj", "model.layers.6.mlp.experts.118.gate_proj", "model.layers.6.mlp.experts.119.gate_proj", "model.layers.6.mlp.experts.120.gate_proj", "model.layers.6.mlp.experts.121.gate_proj", "model.layers.6.mlp.experts.122.gate_proj", "model.layers.6.mlp.experts.123.gate_proj", "model.layers.6.mlp.experts.124.gate_proj", "model.layers.6.mlp.experts.125.gate_proj", "model.layers.6.mlp.experts.126.gate_proj", "model.layers.6.mlp.experts.127.gate_proj", "model.layers.6.mlp.experts.128.gate_proj", "model.layers.6.mlp.experts.129.gate_proj", "model.layers.6.mlp.experts.130.gate_proj", "model.layers.6.mlp.experts.131.gate_proj", "model.layers.6.mlp.experts.132.gate_proj", "model.layers.6.mlp.experts.133.gate_proj", "model.layers.6.mlp.experts.134.gate_proj", "model.layers.6.mlp.experts.135.gate_proj", "model.layers.6.mlp.experts.136.gate_proj", "model.layers.6.mlp.experts.137.gate_proj", "model.layers.6.mlp.experts.138.gate_proj", "model.layers.6.mlp.experts.139.gate_proj", "model.layers.6.mlp.experts.140.gate_proj", "model.layers.6.mlp.experts.141.gate_proj", "model.layers.6.mlp.experts.142.gate_proj", "model.layers.6.mlp.experts.143.gate_proj", "model.layers.6.mlp.experts.144.gate_proj", "model.layers.6.mlp.experts.145.gate_proj", "model.layers.6.mlp.experts.146.gate_proj", "model.layers.6.mlp.experts.147.gate_proj", "model.layers.6.mlp.experts.148.gate_proj", "model.layers.6.mlp.experts.149.gate_proj", "model.layers.6.mlp.experts.150.gate_proj", "model.layers.6.mlp.experts.151.gate_proj", "model.layers.6.mlp.experts.152.gate_proj", "model.layers.6.mlp.experts.153.gate_proj", "model.layers.6.mlp.experts.154.gate_proj", "model.layers.6.mlp.experts.155.gate_proj", "model.layers.6.mlp.experts.156.gate_proj", "model.layers.6.mlp.experts.157.gate_proj", "model.layers.6.mlp.experts.158.gate_proj", "model.layers.6.mlp.experts.159.gate_proj", "model.layers.6.mlp.experts.0.up_proj", "model.layers.6.mlp.experts.1.up_proj", "model.layers.6.mlp.experts.2.up_proj", "model.layers.6.mlp.experts.3.up_proj", "model.layers.6.mlp.experts.4.up_proj", "model.layers.6.mlp.experts.5.up_proj", "model.layers.6.mlp.experts.6.up_proj", "model.layers.6.mlp.experts.7.up_proj", "model.layers.6.mlp.experts.8.up_proj", "model.layers.6.mlp.experts.9.up_proj", "model.layers.6.mlp.experts.10.up_proj", "model.layers.6.mlp.experts.11.up_proj", "model.layers.6.mlp.experts.12.up_proj", "model.layers.6.mlp.experts.13.up_proj", "model.layers.6.mlp.experts.14.up_proj", "model.layers.6.mlp.experts.15.up_proj", "model.layers.6.mlp.experts.16.up_proj", "model.layers.6.mlp.experts.17.up_proj", "model.layers.6.mlp.experts.18.up_proj", "model.layers.6.mlp.experts.19.up_proj", "model.layers.6.mlp.experts.20.up_proj", "model.layers.6.mlp.experts.21.up_proj", "model.layers.6.mlp.experts.22.up_proj", "model.layers.6.mlp.experts.23.up_proj", "model.layers.6.mlp.experts.24.up_proj", "model.layers.6.mlp.experts.25.up_proj", "model.layers.6.mlp.experts.26.up_proj", "model.layers.6.mlp.experts.27.up_proj", "model.layers.6.mlp.experts.28.up_proj", "model.layers.6.mlp.experts.29.up_proj", "model.layers.6.mlp.experts.30.up_proj", "model.layers.6.mlp.experts.31.up_proj", "model.layers.6.mlp.experts.32.up_proj", "model.layers.6.mlp.experts.33.up_proj", "model.layers.6.mlp.experts.34.up_proj", "model.layers.6.mlp.experts.35.up_proj", "model.layers.6.mlp.experts.36.up_proj", "model.layers.6.mlp.experts.37.up_proj", "model.layers.6.mlp.experts.38.up_proj", "model.layers.6.mlp.experts.39.up_proj", "model.layers.6.mlp.experts.40.up_proj", "model.layers.6.mlp.experts.41.up_proj", "model.layers.6.mlp.experts.42.up_proj", "model.layers.6.mlp.experts.43.up_proj", "model.layers.6.mlp.experts.44.up_proj", "model.layers.6.mlp.experts.45.up_proj", "model.layers.6.mlp.experts.46.up_proj", "model.layers.6.mlp.experts.47.up_proj", "model.layers.6.mlp.experts.48.up_proj", "model.layers.6.mlp.experts.49.up_proj", "model.layers.6.mlp.experts.50.up_proj", "model.layers.6.mlp.experts.51.up_proj", "model.layers.6.mlp.experts.52.up_proj", "model.layers.6.mlp.experts.53.up_proj", "model.layers.6.mlp.experts.54.up_proj", "model.layers.6.mlp.experts.55.up_proj", "model.layers.6.mlp.experts.56.up_proj", "model.layers.6.mlp.experts.57.up_proj", "model.layers.6.mlp.experts.58.up_proj", "model.layers.6.mlp.experts.59.up_proj", "model.layers.6.mlp.experts.60.up_proj", "model.layers.6.mlp.experts.61.up_proj", "model.layers.6.mlp.experts.62.up_proj", "model.layers.6.mlp.experts.63.up_proj", "model.layers.6.mlp.experts.64.up_proj", "model.layers.6.mlp.experts.65.up_proj", "model.layers.6.mlp.experts.66.up_proj", "model.layers.6.mlp.experts.67.up_proj", "model.layers.6.mlp.experts.68.up_proj", "model.layers.6.mlp.experts.69.up_proj", "model.layers.6.mlp.experts.70.up_proj", "model.layers.6.mlp.experts.71.up_proj", "model.layers.6.mlp.experts.72.up_proj", "model.layers.6.mlp.experts.73.up_proj", "model.layers.6.mlp.experts.74.up_proj", "model.layers.6.mlp.experts.75.up_proj", "model.layers.6.mlp.experts.76.up_proj", "model.layers.6.mlp.experts.77.up_proj", "model.layers.6.mlp.experts.78.up_proj", "model.layers.6.mlp.experts.79.up_proj", "model.layers.6.mlp.experts.80.up_proj", "model.layers.6.mlp.experts.81.up_proj", "model.layers.6.mlp.experts.82.up_proj", "model.layers.6.mlp.experts.83.up_proj", "model.layers.6.mlp.experts.84.up_proj", "model.layers.6.mlp.experts.85.up_proj", "model.layers.6.mlp.experts.86.up_proj", "model.layers.6.mlp.experts.87.up_proj", "model.layers.6.mlp.experts.88.up_proj", "model.layers.6.mlp.experts.89.up_proj", "model.layers.6.mlp.experts.90.up_proj", "model.layers.6.mlp.experts.91.up_proj", "model.layers.6.mlp.experts.92.up_proj", "model.layers.6.mlp.experts.93.up_proj", "model.layers.6.mlp.experts.94.up_proj", "model.layers.6.mlp.experts.95.up_proj", "model.layers.6.mlp.experts.96.up_proj", "model.layers.6.mlp.experts.97.up_proj", "model.layers.6.mlp.experts.98.up_proj", "model.layers.6.mlp.experts.99.up_proj", "model.layers.6.mlp.experts.100.up_proj", "model.layers.6.mlp.experts.101.up_proj", "model.layers.6.mlp.experts.102.up_proj", "model.layers.6.mlp.experts.103.up_proj", "model.layers.6.mlp.experts.104.up_proj", "model.layers.6.mlp.experts.105.up_proj", "model.layers.6.mlp.experts.106.up_proj", "model.layers.6.mlp.experts.107.up_proj", "model.layers.6.mlp.experts.108.up_proj", "model.layers.6.mlp.experts.109.up_proj", "model.layers.6.mlp.experts.110.up_proj", "model.layers.6.mlp.experts.111.up_proj", "model.layers.6.mlp.experts.112.up_proj", "model.layers.6.mlp.experts.113.up_proj", "model.layers.6.mlp.experts.114.up_proj", "model.layers.6.mlp.experts.115.up_proj", "model.layers.6.mlp.experts.116.up_proj", "model.layers.6.mlp.experts.117.up_proj", "model.layers.6.mlp.experts.118.up_proj", "model.layers.6.mlp.experts.119.up_proj", "model.layers.6.mlp.experts.120.up_proj", "model.layers.6.mlp.experts.121.up_proj", "model.layers.6.mlp.experts.122.up_proj", "model.layers.6.mlp.experts.123.up_proj", "model.layers.6.mlp.experts.124.up_proj", "model.layers.6.mlp.experts.125.up_proj", "model.layers.6.mlp.experts.126.up_proj", "model.layers.6.mlp.experts.127.up_proj", "model.layers.6.mlp.experts.128.up_proj", "model.layers.6.mlp.experts.129.up_proj", "model.layers.6.mlp.experts.130.up_proj", "model.layers.6.mlp.experts.131.up_proj", "model.layers.6.mlp.experts.132.up_proj", "model.layers.6.mlp.experts.133.up_proj", "model.layers.6.mlp.experts.134.up_proj", "model.layers.6.mlp.experts.135.up_proj", "model.layers.6.mlp.experts.136.up_proj", "model.layers.6.mlp.experts.137.up_proj", "model.layers.6.mlp.experts.138.up_proj", "model.layers.6.mlp.experts.139.up_proj", "model.layers.6.mlp.experts.140.up_proj", "model.layers.6.mlp.experts.141.up_proj", "model.layers.6.mlp.experts.142.up_proj", "model.layers.6.mlp.experts.143.up_proj", "model.layers.6.mlp.experts.144.up_proj", "model.layers.6.mlp.experts.145.up_proj", "model.layers.6.mlp.experts.146.up_proj", "model.layers.6.mlp.experts.147.up_proj", "model.layers.6.mlp.experts.148.up_proj", "model.layers.6.mlp.experts.149.up_proj", "model.layers.6.mlp.experts.150.up_proj", "model.layers.6.mlp.experts.151.up_proj", "model.layers.6.mlp.experts.152.up_proj", "model.layers.6.mlp.experts.153.up_proj", "model.layers.6.mlp.experts.154.up_proj", "model.layers.6.mlp.experts.155.up_proj", "model.layers.6.mlp.experts.156.up_proj", "model.layers.6.mlp.experts.157.up_proj", "model.layers.6.mlp.experts.158.up_proj", "model.layers.6.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0005376771092415189, "dbits": 2516582400 } ] }, { "idx": 38, "layers": [ "model.layers.6.mlp.experts.0.down_proj", "model.layers.6.mlp.experts.1.down_proj", "model.layers.6.mlp.experts.2.down_proj", "model.layers.6.mlp.experts.3.down_proj", "model.layers.6.mlp.experts.4.down_proj", "model.layers.6.mlp.experts.5.down_proj", "model.layers.6.mlp.experts.6.down_proj", "model.layers.6.mlp.experts.7.down_proj", "model.layers.6.mlp.experts.8.down_proj", "model.layers.6.mlp.experts.9.down_proj", "model.layers.6.mlp.experts.10.down_proj", "model.layers.6.mlp.experts.11.down_proj", "model.layers.6.mlp.experts.12.down_proj", "model.layers.6.mlp.experts.13.down_proj", "model.layers.6.mlp.experts.14.down_proj", "model.layers.6.mlp.experts.15.down_proj", "model.layers.6.mlp.experts.16.down_proj", "model.layers.6.mlp.experts.17.down_proj", "model.layers.6.mlp.experts.18.down_proj", "model.layers.6.mlp.experts.19.down_proj", "model.layers.6.mlp.experts.20.down_proj", "model.layers.6.mlp.experts.21.down_proj", "model.layers.6.mlp.experts.22.down_proj", "model.layers.6.mlp.experts.23.down_proj", "model.layers.6.mlp.experts.24.down_proj", "model.layers.6.mlp.experts.25.down_proj", "model.layers.6.mlp.experts.26.down_proj", "model.layers.6.mlp.experts.27.down_proj", "model.layers.6.mlp.experts.28.down_proj", "model.layers.6.mlp.experts.29.down_proj", "model.layers.6.mlp.experts.30.down_proj", "model.layers.6.mlp.experts.31.down_proj", "model.layers.6.mlp.experts.32.down_proj", "model.layers.6.mlp.experts.33.down_proj", "model.layers.6.mlp.experts.34.down_proj", "model.layers.6.mlp.experts.35.down_proj", "model.layers.6.mlp.experts.36.down_proj", "model.layers.6.mlp.experts.37.down_proj", "model.layers.6.mlp.experts.38.down_proj", "model.layers.6.mlp.experts.39.down_proj", "model.layers.6.mlp.experts.40.down_proj", "model.layers.6.mlp.experts.41.down_proj", "model.layers.6.mlp.experts.42.down_proj", "model.layers.6.mlp.experts.43.down_proj", "model.layers.6.mlp.experts.44.down_proj", "model.layers.6.mlp.experts.45.down_proj", "model.layers.6.mlp.experts.46.down_proj", "model.layers.6.mlp.experts.47.down_proj", "model.layers.6.mlp.experts.48.down_proj", "model.layers.6.mlp.experts.49.down_proj", "model.layers.6.mlp.experts.50.down_proj", "model.layers.6.mlp.experts.51.down_proj", "model.layers.6.mlp.experts.52.down_proj", "model.layers.6.mlp.experts.53.down_proj", "model.layers.6.mlp.experts.54.down_proj", "model.layers.6.mlp.experts.55.down_proj", "model.layers.6.mlp.experts.56.down_proj", "model.layers.6.mlp.experts.57.down_proj", "model.layers.6.mlp.experts.58.down_proj", "model.layers.6.mlp.experts.59.down_proj", "model.layers.6.mlp.experts.60.down_proj", "model.layers.6.mlp.experts.61.down_proj", "model.layers.6.mlp.experts.62.down_proj", "model.layers.6.mlp.experts.63.down_proj", "model.layers.6.mlp.experts.64.down_proj", "model.layers.6.mlp.experts.65.down_proj", "model.layers.6.mlp.experts.66.down_proj", "model.layers.6.mlp.experts.67.down_proj", "model.layers.6.mlp.experts.68.down_proj", "model.layers.6.mlp.experts.69.down_proj", "model.layers.6.mlp.experts.70.down_proj", "model.layers.6.mlp.experts.71.down_proj", "model.layers.6.mlp.experts.72.down_proj", "model.layers.6.mlp.experts.73.down_proj", "model.layers.6.mlp.experts.74.down_proj", "model.layers.6.mlp.experts.75.down_proj", "model.layers.6.mlp.experts.76.down_proj", "model.layers.6.mlp.experts.77.down_proj", "model.layers.6.mlp.experts.78.down_proj", "model.layers.6.mlp.experts.79.down_proj", "model.layers.6.mlp.experts.80.down_proj", "model.layers.6.mlp.experts.81.down_proj", "model.layers.6.mlp.experts.82.down_proj", "model.layers.6.mlp.experts.83.down_proj", "model.layers.6.mlp.experts.84.down_proj", "model.layers.6.mlp.experts.85.down_proj", "model.layers.6.mlp.experts.86.down_proj", "model.layers.6.mlp.experts.87.down_proj", "model.layers.6.mlp.experts.88.down_proj", "model.layers.6.mlp.experts.89.down_proj", "model.layers.6.mlp.experts.90.down_proj", "model.layers.6.mlp.experts.91.down_proj", "model.layers.6.mlp.experts.92.down_proj", "model.layers.6.mlp.experts.93.down_proj", "model.layers.6.mlp.experts.94.down_proj", "model.layers.6.mlp.experts.95.down_proj", "model.layers.6.mlp.experts.96.down_proj", "model.layers.6.mlp.experts.97.down_proj", "model.layers.6.mlp.experts.98.down_proj", "model.layers.6.mlp.experts.99.down_proj", "model.layers.6.mlp.experts.100.down_proj", "model.layers.6.mlp.experts.101.down_proj", "model.layers.6.mlp.experts.102.down_proj", "model.layers.6.mlp.experts.103.down_proj", "model.layers.6.mlp.experts.104.down_proj", "model.layers.6.mlp.experts.105.down_proj", "model.layers.6.mlp.experts.106.down_proj", "model.layers.6.mlp.experts.107.down_proj", "model.layers.6.mlp.experts.108.down_proj", "model.layers.6.mlp.experts.109.down_proj", "model.layers.6.mlp.experts.110.down_proj", "model.layers.6.mlp.experts.111.down_proj", "model.layers.6.mlp.experts.112.down_proj", "model.layers.6.mlp.experts.113.down_proj", "model.layers.6.mlp.experts.114.down_proj", "model.layers.6.mlp.experts.115.down_proj", "model.layers.6.mlp.experts.116.down_proj", "model.layers.6.mlp.experts.117.down_proj", "model.layers.6.mlp.experts.118.down_proj", "model.layers.6.mlp.experts.119.down_proj", "model.layers.6.mlp.experts.120.down_proj", "model.layers.6.mlp.experts.121.down_proj", "model.layers.6.mlp.experts.122.down_proj", "model.layers.6.mlp.experts.123.down_proj", "model.layers.6.mlp.experts.124.down_proj", "model.layers.6.mlp.experts.125.down_proj", "model.layers.6.mlp.experts.126.down_proj", "model.layers.6.mlp.experts.127.down_proj", "model.layers.6.mlp.experts.128.down_proj", "model.layers.6.mlp.experts.129.down_proj", "model.layers.6.mlp.experts.130.down_proj", "model.layers.6.mlp.experts.131.down_proj", "model.layers.6.mlp.experts.132.down_proj", "model.layers.6.mlp.experts.133.down_proj", "model.layers.6.mlp.experts.134.down_proj", "model.layers.6.mlp.experts.135.down_proj", "model.layers.6.mlp.experts.136.down_proj", "model.layers.6.mlp.experts.137.down_proj", "model.layers.6.mlp.experts.138.down_proj", "model.layers.6.mlp.experts.139.down_proj", "model.layers.6.mlp.experts.140.down_proj", "model.layers.6.mlp.experts.141.down_proj", "model.layers.6.mlp.experts.142.down_proj", "model.layers.6.mlp.experts.143.down_proj", "model.layers.6.mlp.experts.144.down_proj", "model.layers.6.mlp.experts.145.down_proj", "model.layers.6.mlp.experts.146.down_proj", "model.layers.6.mlp.experts.147.down_proj", "model.layers.6.mlp.experts.148.down_proj", "model.layers.6.mlp.experts.149.down_proj", "model.layers.6.mlp.experts.150.down_proj", "model.layers.6.mlp.experts.151.down_proj", "model.layers.6.mlp.experts.152.down_proj", "model.layers.6.mlp.experts.153.down_proj", "model.layers.6.mlp.experts.154.down_proj", "model.layers.6.mlp.experts.155.down_proj", "model.layers.6.mlp.experts.156.down_proj", "model.layers.6.mlp.experts.157.down_proj", "model.layers.6.mlp.experts.158.down_proj", "model.layers.6.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0009004712104797585, "dbits": 1258291200 } ] }, { "idx": 39, "layers": [ "model.layers.7.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0001417987048625835, "dbits": 62914560 } ] }, { "idx": 40, "layers": [ "model.layers.7.self_attn.k_proj", "model.layers.7.self_attn.v_proj" ], "candidates": [ { "dkld": 9.349659085272632e-05, "dbits": 10485760 } ] }, { "idx": 41, "layers": [ "model.layers.7.self_attn.o_proj" ], "candidates": [ { "dkld": 0.001130869239568677, "dbits": 62914560 } ] }, { "idx": 42, "layers": [ "model.layers.7.mlp.shared_experts.gate_proj", "model.layers.7.mlp.shared_experts.up_proj", "model.layers.7.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00017116516828535877, "dbits": 23592960 } ] }, { "idx": 43, "layers": [ "model.layers.7.mlp.experts.0.gate_proj", "model.layers.7.mlp.experts.1.gate_proj", "model.layers.7.mlp.experts.2.gate_proj", "model.layers.7.mlp.experts.3.gate_proj", "model.layers.7.mlp.experts.4.gate_proj", "model.layers.7.mlp.experts.5.gate_proj", "model.layers.7.mlp.experts.6.gate_proj", "model.layers.7.mlp.experts.7.gate_proj", "model.layers.7.mlp.experts.8.gate_proj", "model.layers.7.mlp.experts.9.gate_proj", "model.layers.7.mlp.experts.10.gate_proj", "model.layers.7.mlp.experts.11.gate_proj", "model.layers.7.mlp.experts.12.gate_proj", "model.layers.7.mlp.experts.13.gate_proj", "model.layers.7.mlp.experts.14.gate_proj", "model.layers.7.mlp.experts.15.gate_proj", "model.layers.7.mlp.experts.16.gate_proj", "model.layers.7.mlp.experts.17.gate_proj", "model.layers.7.mlp.experts.18.gate_proj", "model.layers.7.mlp.experts.19.gate_proj", "model.layers.7.mlp.experts.20.gate_proj", "model.layers.7.mlp.experts.21.gate_proj", "model.layers.7.mlp.experts.22.gate_proj", "model.layers.7.mlp.experts.23.gate_proj", "model.layers.7.mlp.experts.24.gate_proj", "model.layers.7.mlp.experts.25.gate_proj", "model.layers.7.mlp.experts.26.gate_proj", "model.layers.7.mlp.experts.27.gate_proj", "model.layers.7.mlp.experts.28.gate_proj", "model.layers.7.mlp.experts.29.gate_proj", "model.layers.7.mlp.experts.30.gate_proj", "model.layers.7.mlp.experts.31.gate_proj", "model.layers.7.mlp.experts.32.gate_proj", "model.layers.7.mlp.experts.33.gate_proj", "model.layers.7.mlp.experts.34.gate_proj", "model.layers.7.mlp.experts.35.gate_proj", "model.layers.7.mlp.experts.36.gate_proj", "model.layers.7.mlp.experts.37.gate_proj", "model.layers.7.mlp.experts.38.gate_proj", "model.layers.7.mlp.experts.39.gate_proj", "model.layers.7.mlp.experts.40.gate_proj", "model.layers.7.mlp.experts.41.gate_proj", "model.layers.7.mlp.experts.42.gate_proj", "model.layers.7.mlp.experts.43.gate_proj", "model.layers.7.mlp.experts.44.gate_proj", "model.layers.7.mlp.experts.45.gate_proj", "model.layers.7.mlp.experts.46.gate_proj", "model.layers.7.mlp.experts.47.gate_proj", "model.layers.7.mlp.experts.48.gate_proj", "model.layers.7.mlp.experts.49.gate_proj", "model.layers.7.mlp.experts.50.gate_proj", "model.layers.7.mlp.experts.51.gate_proj", "model.layers.7.mlp.experts.52.gate_proj", "model.layers.7.mlp.experts.53.gate_proj", "model.layers.7.mlp.experts.54.gate_proj", "model.layers.7.mlp.experts.55.gate_proj", "model.layers.7.mlp.experts.56.gate_proj", "model.layers.7.mlp.experts.57.gate_proj", "model.layers.7.mlp.experts.58.gate_proj", "model.layers.7.mlp.experts.59.gate_proj", "model.layers.7.mlp.experts.60.gate_proj", "model.layers.7.mlp.experts.61.gate_proj", "model.layers.7.mlp.experts.62.gate_proj", "model.layers.7.mlp.experts.63.gate_proj", "model.layers.7.mlp.experts.64.gate_proj", "model.layers.7.mlp.experts.65.gate_proj", "model.layers.7.mlp.experts.66.gate_proj", "model.layers.7.mlp.experts.67.gate_proj", "model.layers.7.mlp.experts.68.gate_proj", "model.layers.7.mlp.experts.69.gate_proj", "model.layers.7.mlp.experts.70.gate_proj", "model.layers.7.mlp.experts.71.gate_proj", "model.layers.7.mlp.experts.72.gate_proj", "model.layers.7.mlp.experts.73.gate_proj", "model.layers.7.mlp.experts.74.gate_proj", "model.layers.7.mlp.experts.75.gate_proj", "model.layers.7.mlp.experts.76.gate_proj", "model.layers.7.mlp.experts.77.gate_proj", "model.layers.7.mlp.experts.78.gate_proj", "model.layers.7.mlp.experts.79.gate_proj", "model.layers.7.mlp.experts.80.gate_proj", "model.layers.7.mlp.experts.81.gate_proj", "model.layers.7.mlp.experts.82.gate_proj", "model.layers.7.mlp.experts.83.gate_proj", "model.layers.7.mlp.experts.84.gate_proj", "model.layers.7.mlp.experts.85.gate_proj", "model.layers.7.mlp.experts.86.gate_proj", "model.layers.7.mlp.experts.87.gate_proj", "model.layers.7.mlp.experts.88.gate_proj", "model.layers.7.mlp.experts.89.gate_proj", "model.layers.7.mlp.experts.90.gate_proj", "model.layers.7.mlp.experts.91.gate_proj", "model.layers.7.mlp.experts.92.gate_proj", "model.layers.7.mlp.experts.93.gate_proj", "model.layers.7.mlp.experts.94.gate_proj", "model.layers.7.mlp.experts.95.gate_proj", "model.layers.7.mlp.experts.96.gate_proj", "model.layers.7.mlp.experts.97.gate_proj", "model.layers.7.mlp.experts.98.gate_proj", "model.layers.7.mlp.experts.99.gate_proj", "model.layers.7.mlp.experts.100.gate_proj", "model.layers.7.mlp.experts.101.gate_proj", "model.layers.7.mlp.experts.102.gate_proj", "model.layers.7.mlp.experts.103.gate_proj", "model.layers.7.mlp.experts.104.gate_proj", "model.layers.7.mlp.experts.105.gate_proj", "model.layers.7.mlp.experts.106.gate_proj", "model.layers.7.mlp.experts.107.gate_proj", "model.layers.7.mlp.experts.108.gate_proj", "model.layers.7.mlp.experts.109.gate_proj", "model.layers.7.mlp.experts.110.gate_proj", "model.layers.7.mlp.experts.111.gate_proj", "model.layers.7.mlp.experts.112.gate_proj", "model.layers.7.mlp.experts.113.gate_proj", "model.layers.7.mlp.experts.114.gate_proj", "model.layers.7.mlp.experts.115.gate_proj", "model.layers.7.mlp.experts.116.gate_proj", "model.layers.7.mlp.experts.117.gate_proj", "model.layers.7.mlp.experts.118.gate_proj", "model.layers.7.mlp.experts.119.gate_proj", "model.layers.7.mlp.experts.120.gate_proj", "model.layers.7.mlp.experts.121.gate_proj", "model.layers.7.mlp.experts.122.gate_proj", "model.layers.7.mlp.experts.123.gate_proj", "model.layers.7.mlp.experts.124.gate_proj", "model.layers.7.mlp.experts.125.gate_proj", "model.layers.7.mlp.experts.126.gate_proj", "model.layers.7.mlp.experts.127.gate_proj", "model.layers.7.mlp.experts.128.gate_proj", "model.layers.7.mlp.experts.129.gate_proj", "model.layers.7.mlp.experts.130.gate_proj", "model.layers.7.mlp.experts.131.gate_proj", "model.layers.7.mlp.experts.132.gate_proj", "model.layers.7.mlp.experts.133.gate_proj", "model.layers.7.mlp.experts.134.gate_proj", "model.layers.7.mlp.experts.135.gate_proj", "model.layers.7.mlp.experts.136.gate_proj", "model.layers.7.mlp.experts.137.gate_proj", "model.layers.7.mlp.experts.138.gate_proj", "model.layers.7.mlp.experts.139.gate_proj", "model.layers.7.mlp.experts.140.gate_proj", "model.layers.7.mlp.experts.141.gate_proj", "model.layers.7.mlp.experts.142.gate_proj", "model.layers.7.mlp.experts.143.gate_proj", "model.layers.7.mlp.experts.144.gate_proj", "model.layers.7.mlp.experts.145.gate_proj", "model.layers.7.mlp.experts.146.gate_proj", "model.layers.7.mlp.experts.147.gate_proj", "model.layers.7.mlp.experts.148.gate_proj", "model.layers.7.mlp.experts.149.gate_proj", "model.layers.7.mlp.experts.150.gate_proj", "model.layers.7.mlp.experts.151.gate_proj", "model.layers.7.mlp.experts.152.gate_proj", "model.layers.7.mlp.experts.153.gate_proj", "model.layers.7.mlp.experts.154.gate_proj", "model.layers.7.mlp.experts.155.gate_proj", "model.layers.7.mlp.experts.156.gate_proj", "model.layers.7.mlp.experts.157.gate_proj", "model.layers.7.mlp.experts.158.gate_proj", "model.layers.7.mlp.experts.159.gate_proj", "model.layers.7.mlp.experts.0.up_proj", "model.layers.7.mlp.experts.1.up_proj", "model.layers.7.mlp.experts.2.up_proj", "model.layers.7.mlp.experts.3.up_proj", "model.layers.7.mlp.experts.4.up_proj", "model.layers.7.mlp.experts.5.up_proj", "model.layers.7.mlp.experts.6.up_proj", "model.layers.7.mlp.experts.7.up_proj", "model.layers.7.mlp.experts.8.up_proj", "model.layers.7.mlp.experts.9.up_proj", "model.layers.7.mlp.experts.10.up_proj", "model.layers.7.mlp.experts.11.up_proj", "model.layers.7.mlp.experts.12.up_proj", "model.layers.7.mlp.experts.13.up_proj", "model.layers.7.mlp.experts.14.up_proj", "model.layers.7.mlp.experts.15.up_proj", "model.layers.7.mlp.experts.16.up_proj", "model.layers.7.mlp.experts.17.up_proj", "model.layers.7.mlp.experts.18.up_proj", "model.layers.7.mlp.experts.19.up_proj", "model.layers.7.mlp.experts.20.up_proj", "model.layers.7.mlp.experts.21.up_proj", "model.layers.7.mlp.experts.22.up_proj", "model.layers.7.mlp.experts.23.up_proj", "model.layers.7.mlp.experts.24.up_proj", "model.layers.7.mlp.experts.25.up_proj", "model.layers.7.mlp.experts.26.up_proj", "model.layers.7.mlp.experts.27.up_proj", "model.layers.7.mlp.experts.28.up_proj", "model.layers.7.mlp.experts.29.up_proj", "model.layers.7.mlp.experts.30.up_proj", "model.layers.7.mlp.experts.31.up_proj", "model.layers.7.mlp.experts.32.up_proj", "model.layers.7.mlp.experts.33.up_proj", "model.layers.7.mlp.experts.34.up_proj", "model.layers.7.mlp.experts.35.up_proj", "model.layers.7.mlp.experts.36.up_proj", "model.layers.7.mlp.experts.37.up_proj", "model.layers.7.mlp.experts.38.up_proj", "model.layers.7.mlp.experts.39.up_proj", "model.layers.7.mlp.experts.40.up_proj", "model.layers.7.mlp.experts.41.up_proj", "model.layers.7.mlp.experts.42.up_proj", "model.layers.7.mlp.experts.43.up_proj", "model.layers.7.mlp.experts.44.up_proj", "model.layers.7.mlp.experts.45.up_proj", "model.layers.7.mlp.experts.46.up_proj", "model.layers.7.mlp.experts.47.up_proj", "model.layers.7.mlp.experts.48.up_proj", "model.layers.7.mlp.experts.49.up_proj", "model.layers.7.mlp.experts.50.up_proj", "model.layers.7.mlp.experts.51.up_proj", "model.layers.7.mlp.experts.52.up_proj", "model.layers.7.mlp.experts.53.up_proj", "model.layers.7.mlp.experts.54.up_proj", "model.layers.7.mlp.experts.55.up_proj", "model.layers.7.mlp.experts.56.up_proj", "model.layers.7.mlp.experts.57.up_proj", "model.layers.7.mlp.experts.58.up_proj", "model.layers.7.mlp.experts.59.up_proj", "model.layers.7.mlp.experts.60.up_proj", "model.layers.7.mlp.experts.61.up_proj", "model.layers.7.mlp.experts.62.up_proj", "model.layers.7.mlp.experts.63.up_proj", "model.layers.7.mlp.experts.64.up_proj", "model.layers.7.mlp.experts.65.up_proj", "model.layers.7.mlp.experts.66.up_proj", "model.layers.7.mlp.experts.67.up_proj", "model.layers.7.mlp.experts.68.up_proj", "model.layers.7.mlp.experts.69.up_proj", "model.layers.7.mlp.experts.70.up_proj", "model.layers.7.mlp.experts.71.up_proj", "model.layers.7.mlp.experts.72.up_proj", "model.layers.7.mlp.experts.73.up_proj", "model.layers.7.mlp.experts.74.up_proj", "model.layers.7.mlp.experts.75.up_proj", "model.layers.7.mlp.experts.76.up_proj", "model.layers.7.mlp.experts.77.up_proj", "model.layers.7.mlp.experts.78.up_proj", "model.layers.7.mlp.experts.79.up_proj", "model.layers.7.mlp.experts.80.up_proj", "model.layers.7.mlp.experts.81.up_proj", "model.layers.7.mlp.experts.82.up_proj", "model.layers.7.mlp.experts.83.up_proj", "model.layers.7.mlp.experts.84.up_proj", "model.layers.7.mlp.experts.85.up_proj", "model.layers.7.mlp.experts.86.up_proj", "model.layers.7.mlp.experts.87.up_proj", "model.layers.7.mlp.experts.88.up_proj", "model.layers.7.mlp.experts.89.up_proj", "model.layers.7.mlp.experts.90.up_proj", "model.layers.7.mlp.experts.91.up_proj", "model.layers.7.mlp.experts.92.up_proj", "model.layers.7.mlp.experts.93.up_proj", "model.layers.7.mlp.experts.94.up_proj", "model.layers.7.mlp.experts.95.up_proj", "model.layers.7.mlp.experts.96.up_proj", "model.layers.7.mlp.experts.97.up_proj", "model.layers.7.mlp.experts.98.up_proj", "model.layers.7.mlp.experts.99.up_proj", "model.layers.7.mlp.experts.100.up_proj", "model.layers.7.mlp.experts.101.up_proj", "model.layers.7.mlp.experts.102.up_proj", "model.layers.7.mlp.experts.103.up_proj", "model.layers.7.mlp.experts.104.up_proj", "model.layers.7.mlp.experts.105.up_proj", "model.layers.7.mlp.experts.106.up_proj", "model.layers.7.mlp.experts.107.up_proj", "model.layers.7.mlp.experts.108.up_proj", "model.layers.7.mlp.experts.109.up_proj", "model.layers.7.mlp.experts.110.up_proj", "model.layers.7.mlp.experts.111.up_proj", "model.layers.7.mlp.experts.112.up_proj", "model.layers.7.mlp.experts.113.up_proj", "model.layers.7.mlp.experts.114.up_proj", "model.layers.7.mlp.experts.115.up_proj", "model.layers.7.mlp.experts.116.up_proj", "model.layers.7.mlp.experts.117.up_proj", "model.layers.7.mlp.experts.118.up_proj", "model.layers.7.mlp.experts.119.up_proj", "model.layers.7.mlp.experts.120.up_proj", "model.layers.7.mlp.experts.121.up_proj", "model.layers.7.mlp.experts.122.up_proj", "model.layers.7.mlp.experts.123.up_proj", "model.layers.7.mlp.experts.124.up_proj", "model.layers.7.mlp.experts.125.up_proj", "model.layers.7.mlp.experts.126.up_proj", "model.layers.7.mlp.experts.127.up_proj", "model.layers.7.mlp.experts.128.up_proj", "model.layers.7.mlp.experts.129.up_proj", "model.layers.7.mlp.experts.130.up_proj", "model.layers.7.mlp.experts.131.up_proj", "model.layers.7.mlp.experts.132.up_proj", "model.layers.7.mlp.experts.133.up_proj", "model.layers.7.mlp.experts.134.up_proj", "model.layers.7.mlp.experts.135.up_proj", "model.layers.7.mlp.experts.136.up_proj", "model.layers.7.mlp.experts.137.up_proj", "model.layers.7.mlp.experts.138.up_proj", "model.layers.7.mlp.experts.139.up_proj", "model.layers.7.mlp.experts.140.up_proj", "model.layers.7.mlp.experts.141.up_proj", "model.layers.7.mlp.experts.142.up_proj", "model.layers.7.mlp.experts.143.up_proj", "model.layers.7.mlp.experts.144.up_proj", "model.layers.7.mlp.experts.145.up_proj", "model.layers.7.mlp.experts.146.up_proj", "model.layers.7.mlp.experts.147.up_proj", "model.layers.7.mlp.experts.148.up_proj", "model.layers.7.mlp.experts.149.up_proj", "model.layers.7.mlp.experts.150.up_proj", "model.layers.7.mlp.experts.151.up_proj", "model.layers.7.mlp.experts.152.up_proj", "model.layers.7.mlp.experts.153.up_proj", "model.layers.7.mlp.experts.154.up_proj", "model.layers.7.mlp.experts.155.up_proj", "model.layers.7.mlp.experts.156.up_proj", "model.layers.7.mlp.experts.157.up_proj", "model.layers.7.mlp.experts.158.up_proj", "model.layers.7.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.001084516942501057, "dbits": 2516582400 } ] }, { "idx": 44, "layers": [ "model.layers.7.mlp.experts.0.down_proj", "model.layers.7.mlp.experts.1.down_proj", "model.layers.7.mlp.experts.2.down_proj", "model.layers.7.mlp.experts.3.down_proj", "model.layers.7.mlp.experts.4.down_proj", "model.layers.7.mlp.experts.5.down_proj", "model.layers.7.mlp.experts.6.down_proj", "model.layers.7.mlp.experts.7.down_proj", "model.layers.7.mlp.experts.8.down_proj", "model.layers.7.mlp.experts.9.down_proj", "model.layers.7.mlp.experts.10.down_proj", "model.layers.7.mlp.experts.11.down_proj", "model.layers.7.mlp.experts.12.down_proj", "model.layers.7.mlp.experts.13.down_proj", "model.layers.7.mlp.experts.14.down_proj", "model.layers.7.mlp.experts.15.down_proj", "model.layers.7.mlp.experts.16.down_proj", "model.layers.7.mlp.experts.17.down_proj", "model.layers.7.mlp.experts.18.down_proj", "model.layers.7.mlp.experts.19.down_proj", "model.layers.7.mlp.experts.20.down_proj", "model.layers.7.mlp.experts.21.down_proj", "model.layers.7.mlp.experts.22.down_proj", "model.layers.7.mlp.experts.23.down_proj", "model.layers.7.mlp.experts.24.down_proj", "model.layers.7.mlp.experts.25.down_proj", "model.layers.7.mlp.experts.26.down_proj", "model.layers.7.mlp.experts.27.down_proj", "model.layers.7.mlp.experts.28.down_proj", "model.layers.7.mlp.experts.29.down_proj", "model.layers.7.mlp.experts.30.down_proj", "model.layers.7.mlp.experts.31.down_proj", "model.layers.7.mlp.experts.32.down_proj", "model.layers.7.mlp.experts.33.down_proj", "model.layers.7.mlp.experts.34.down_proj", "model.layers.7.mlp.experts.35.down_proj", "model.layers.7.mlp.experts.36.down_proj", "model.layers.7.mlp.experts.37.down_proj", "model.layers.7.mlp.experts.38.down_proj", "model.layers.7.mlp.experts.39.down_proj", "model.layers.7.mlp.experts.40.down_proj", "model.layers.7.mlp.experts.41.down_proj", "model.layers.7.mlp.experts.42.down_proj", "model.layers.7.mlp.experts.43.down_proj", "model.layers.7.mlp.experts.44.down_proj", "model.layers.7.mlp.experts.45.down_proj", "model.layers.7.mlp.experts.46.down_proj", "model.layers.7.mlp.experts.47.down_proj", "model.layers.7.mlp.experts.48.down_proj", "model.layers.7.mlp.experts.49.down_proj", "model.layers.7.mlp.experts.50.down_proj", "model.layers.7.mlp.experts.51.down_proj", "model.layers.7.mlp.experts.52.down_proj", "model.layers.7.mlp.experts.53.down_proj", "model.layers.7.mlp.experts.54.down_proj", "model.layers.7.mlp.experts.55.down_proj", "model.layers.7.mlp.experts.56.down_proj", "model.layers.7.mlp.experts.57.down_proj", "model.layers.7.mlp.experts.58.down_proj", "model.layers.7.mlp.experts.59.down_proj", "model.layers.7.mlp.experts.60.down_proj", "model.layers.7.mlp.experts.61.down_proj", "model.layers.7.mlp.experts.62.down_proj", "model.layers.7.mlp.experts.63.down_proj", "model.layers.7.mlp.experts.64.down_proj", "model.layers.7.mlp.experts.65.down_proj", "model.layers.7.mlp.experts.66.down_proj", "model.layers.7.mlp.experts.67.down_proj", "model.layers.7.mlp.experts.68.down_proj", "model.layers.7.mlp.experts.69.down_proj", "model.layers.7.mlp.experts.70.down_proj", "model.layers.7.mlp.experts.71.down_proj", "model.layers.7.mlp.experts.72.down_proj", "model.layers.7.mlp.experts.73.down_proj", "model.layers.7.mlp.experts.74.down_proj", "model.layers.7.mlp.experts.75.down_proj", "model.layers.7.mlp.experts.76.down_proj", "model.layers.7.mlp.experts.77.down_proj", "model.layers.7.mlp.experts.78.down_proj", "model.layers.7.mlp.experts.79.down_proj", "model.layers.7.mlp.experts.80.down_proj", "model.layers.7.mlp.experts.81.down_proj", "model.layers.7.mlp.experts.82.down_proj", "model.layers.7.mlp.experts.83.down_proj", "model.layers.7.mlp.experts.84.down_proj", "model.layers.7.mlp.experts.85.down_proj", "model.layers.7.mlp.experts.86.down_proj", "model.layers.7.mlp.experts.87.down_proj", "model.layers.7.mlp.experts.88.down_proj", "model.layers.7.mlp.experts.89.down_proj", "model.layers.7.mlp.experts.90.down_proj", "model.layers.7.mlp.experts.91.down_proj", "model.layers.7.mlp.experts.92.down_proj", "model.layers.7.mlp.experts.93.down_proj", "model.layers.7.mlp.experts.94.down_proj", "model.layers.7.mlp.experts.95.down_proj", "model.layers.7.mlp.experts.96.down_proj", "model.layers.7.mlp.experts.97.down_proj", "model.layers.7.mlp.experts.98.down_proj", "model.layers.7.mlp.experts.99.down_proj", "model.layers.7.mlp.experts.100.down_proj", "model.layers.7.mlp.experts.101.down_proj", "model.layers.7.mlp.experts.102.down_proj", "model.layers.7.mlp.experts.103.down_proj", "model.layers.7.mlp.experts.104.down_proj", "model.layers.7.mlp.experts.105.down_proj", "model.layers.7.mlp.experts.106.down_proj", "model.layers.7.mlp.experts.107.down_proj", "model.layers.7.mlp.experts.108.down_proj", "model.layers.7.mlp.experts.109.down_proj", "model.layers.7.mlp.experts.110.down_proj", "model.layers.7.mlp.experts.111.down_proj", "model.layers.7.mlp.experts.112.down_proj", "model.layers.7.mlp.experts.113.down_proj", "model.layers.7.mlp.experts.114.down_proj", "model.layers.7.mlp.experts.115.down_proj", "model.layers.7.mlp.experts.116.down_proj", "model.layers.7.mlp.experts.117.down_proj", "model.layers.7.mlp.experts.118.down_proj", "model.layers.7.mlp.experts.119.down_proj", "model.layers.7.mlp.experts.120.down_proj", "model.layers.7.mlp.experts.121.down_proj", "model.layers.7.mlp.experts.122.down_proj", "model.layers.7.mlp.experts.123.down_proj", "model.layers.7.mlp.experts.124.down_proj", "model.layers.7.mlp.experts.125.down_proj", "model.layers.7.mlp.experts.126.down_proj", "model.layers.7.mlp.experts.127.down_proj", "model.layers.7.mlp.experts.128.down_proj", "model.layers.7.mlp.experts.129.down_proj", "model.layers.7.mlp.experts.130.down_proj", "model.layers.7.mlp.experts.131.down_proj", "model.layers.7.mlp.experts.132.down_proj", "model.layers.7.mlp.experts.133.down_proj", "model.layers.7.mlp.experts.134.down_proj", "model.layers.7.mlp.experts.135.down_proj", "model.layers.7.mlp.experts.136.down_proj", "model.layers.7.mlp.experts.137.down_proj", "model.layers.7.mlp.experts.138.down_proj", "model.layers.7.mlp.experts.139.down_proj", "model.layers.7.mlp.experts.140.down_proj", "model.layers.7.mlp.experts.141.down_proj", "model.layers.7.mlp.experts.142.down_proj", "model.layers.7.mlp.experts.143.down_proj", "model.layers.7.mlp.experts.144.down_proj", "model.layers.7.mlp.experts.145.down_proj", "model.layers.7.mlp.experts.146.down_proj", "model.layers.7.mlp.experts.147.down_proj", "model.layers.7.mlp.experts.148.down_proj", "model.layers.7.mlp.experts.149.down_proj", "model.layers.7.mlp.experts.150.down_proj", "model.layers.7.mlp.experts.151.down_proj", "model.layers.7.mlp.experts.152.down_proj", "model.layers.7.mlp.experts.153.down_proj", "model.layers.7.mlp.experts.154.down_proj", "model.layers.7.mlp.experts.155.down_proj", "model.layers.7.mlp.experts.156.down_proj", "model.layers.7.mlp.experts.157.down_proj", "model.layers.7.mlp.experts.158.down_proj", "model.layers.7.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0014752179384231456, "dbits": 1258291200 } ] }, { "idx": 45, "layers": [ "model.layers.8.self_attn.q_proj" ], "candidates": [ { "dkld": -0.000656285881996177, "dbits": 62914560 } ] }, { "idx": 46, "layers": [ "model.layers.8.self_attn.k_proj", "model.layers.8.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0027504965662956127, "dbits": 10485760 } ] }, { "idx": 47, "layers": [ "model.layers.8.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0020506829023361206, "dbits": 62914560 } ] }, { "idx": 48, "layers": [ "model.layers.8.mlp.shared_experts.gate_proj", "model.layers.8.mlp.shared_experts.up_proj", "model.layers.8.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.001581658422946941, "dbits": 23592960 } ] }, { "idx": 49, "layers": [ "model.layers.8.mlp.experts.0.gate_proj", "model.layers.8.mlp.experts.1.gate_proj", "model.layers.8.mlp.experts.2.gate_proj", "model.layers.8.mlp.experts.3.gate_proj", "model.layers.8.mlp.experts.4.gate_proj", "model.layers.8.mlp.experts.5.gate_proj", "model.layers.8.mlp.experts.6.gate_proj", "model.layers.8.mlp.experts.7.gate_proj", "model.layers.8.mlp.experts.8.gate_proj", "model.layers.8.mlp.experts.9.gate_proj", "model.layers.8.mlp.experts.10.gate_proj", "model.layers.8.mlp.experts.11.gate_proj", "model.layers.8.mlp.experts.12.gate_proj", "model.layers.8.mlp.experts.13.gate_proj", "model.layers.8.mlp.experts.14.gate_proj", "model.layers.8.mlp.experts.15.gate_proj", "model.layers.8.mlp.experts.16.gate_proj", "model.layers.8.mlp.experts.17.gate_proj", "model.layers.8.mlp.experts.18.gate_proj", "model.layers.8.mlp.experts.19.gate_proj", "model.layers.8.mlp.experts.20.gate_proj", "model.layers.8.mlp.experts.21.gate_proj", "model.layers.8.mlp.experts.22.gate_proj", "model.layers.8.mlp.experts.23.gate_proj", "model.layers.8.mlp.experts.24.gate_proj", "model.layers.8.mlp.experts.25.gate_proj", "model.layers.8.mlp.experts.26.gate_proj", "model.layers.8.mlp.experts.27.gate_proj", "model.layers.8.mlp.experts.28.gate_proj", "model.layers.8.mlp.experts.29.gate_proj", "model.layers.8.mlp.experts.30.gate_proj", "model.layers.8.mlp.experts.31.gate_proj", "model.layers.8.mlp.experts.32.gate_proj", "model.layers.8.mlp.experts.33.gate_proj", "model.layers.8.mlp.experts.34.gate_proj", "model.layers.8.mlp.experts.35.gate_proj", "model.layers.8.mlp.experts.36.gate_proj", "model.layers.8.mlp.experts.37.gate_proj", "model.layers.8.mlp.experts.38.gate_proj", "model.layers.8.mlp.experts.39.gate_proj", "model.layers.8.mlp.experts.40.gate_proj", "model.layers.8.mlp.experts.41.gate_proj", "model.layers.8.mlp.experts.42.gate_proj", "model.layers.8.mlp.experts.43.gate_proj", "model.layers.8.mlp.experts.44.gate_proj", "model.layers.8.mlp.experts.45.gate_proj", "model.layers.8.mlp.experts.46.gate_proj", "model.layers.8.mlp.experts.47.gate_proj", "model.layers.8.mlp.experts.48.gate_proj", "model.layers.8.mlp.experts.49.gate_proj", "model.layers.8.mlp.experts.50.gate_proj", "model.layers.8.mlp.experts.51.gate_proj", "model.layers.8.mlp.experts.52.gate_proj", "model.layers.8.mlp.experts.53.gate_proj", "model.layers.8.mlp.experts.54.gate_proj", "model.layers.8.mlp.experts.55.gate_proj", "model.layers.8.mlp.experts.56.gate_proj", "model.layers.8.mlp.experts.57.gate_proj", "model.layers.8.mlp.experts.58.gate_proj", "model.layers.8.mlp.experts.59.gate_proj", "model.layers.8.mlp.experts.60.gate_proj", "model.layers.8.mlp.experts.61.gate_proj", "model.layers.8.mlp.experts.62.gate_proj", "model.layers.8.mlp.experts.63.gate_proj", "model.layers.8.mlp.experts.64.gate_proj", "model.layers.8.mlp.experts.65.gate_proj", "model.layers.8.mlp.experts.66.gate_proj", "model.layers.8.mlp.experts.67.gate_proj", "model.layers.8.mlp.experts.68.gate_proj", "model.layers.8.mlp.experts.69.gate_proj", "model.layers.8.mlp.experts.70.gate_proj", "model.layers.8.mlp.experts.71.gate_proj", "model.layers.8.mlp.experts.72.gate_proj", "model.layers.8.mlp.experts.73.gate_proj", "model.layers.8.mlp.experts.74.gate_proj", "model.layers.8.mlp.experts.75.gate_proj", "model.layers.8.mlp.experts.76.gate_proj", "model.layers.8.mlp.experts.77.gate_proj", "model.layers.8.mlp.experts.78.gate_proj", "model.layers.8.mlp.experts.79.gate_proj", "model.layers.8.mlp.experts.80.gate_proj", "model.layers.8.mlp.experts.81.gate_proj", "model.layers.8.mlp.experts.82.gate_proj", "model.layers.8.mlp.experts.83.gate_proj", "model.layers.8.mlp.experts.84.gate_proj", "model.layers.8.mlp.experts.85.gate_proj", "model.layers.8.mlp.experts.86.gate_proj", "model.layers.8.mlp.experts.87.gate_proj", "model.layers.8.mlp.experts.88.gate_proj", "model.layers.8.mlp.experts.89.gate_proj", "model.layers.8.mlp.experts.90.gate_proj", "model.layers.8.mlp.experts.91.gate_proj", "model.layers.8.mlp.experts.92.gate_proj", "model.layers.8.mlp.experts.93.gate_proj", "model.layers.8.mlp.experts.94.gate_proj", "model.layers.8.mlp.experts.95.gate_proj", "model.layers.8.mlp.experts.96.gate_proj", "model.layers.8.mlp.experts.97.gate_proj", "model.layers.8.mlp.experts.98.gate_proj", "model.layers.8.mlp.experts.99.gate_proj", "model.layers.8.mlp.experts.100.gate_proj", "model.layers.8.mlp.experts.101.gate_proj", "model.layers.8.mlp.experts.102.gate_proj", "model.layers.8.mlp.experts.103.gate_proj", "model.layers.8.mlp.experts.104.gate_proj", "model.layers.8.mlp.experts.105.gate_proj", "model.layers.8.mlp.experts.106.gate_proj", "model.layers.8.mlp.experts.107.gate_proj", "model.layers.8.mlp.experts.108.gate_proj", "model.layers.8.mlp.experts.109.gate_proj", "model.layers.8.mlp.experts.110.gate_proj", "model.layers.8.mlp.experts.111.gate_proj", "model.layers.8.mlp.experts.112.gate_proj", "model.layers.8.mlp.experts.113.gate_proj", "model.layers.8.mlp.experts.114.gate_proj", "model.layers.8.mlp.experts.115.gate_proj", "model.layers.8.mlp.experts.116.gate_proj", "model.layers.8.mlp.experts.117.gate_proj", "model.layers.8.mlp.experts.118.gate_proj", "model.layers.8.mlp.experts.119.gate_proj", "model.layers.8.mlp.experts.120.gate_proj", "model.layers.8.mlp.experts.121.gate_proj", "model.layers.8.mlp.experts.122.gate_proj", "model.layers.8.mlp.experts.123.gate_proj", "model.layers.8.mlp.experts.124.gate_proj", "model.layers.8.mlp.experts.125.gate_proj", "model.layers.8.mlp.experts.126.gate_proj", "model.layers.8.mlp.experts.127.gate_proj", "model.layers.8.mlp.experts.128.gate_proj", "model.layers.8.mlp.experts.129.gate_proj", "model.layers.8.mlp.experts.130.gate_proj", "model.layers.8.mlp.experts.131.gate_proj", "model.layers.8.mlp.experts.132.gate_proj", "model.layers.8.mlp.experts.133.gate_proj", "model.layers.8.mlp.experts.134.gate_proj", "model.layers.8.mlp.experts.135.gate_proj", "model.layers.8.mlp.experts.136.gate_proj", "model.layers.8.mlp.experts.137.gate_proj", "model.layers.8.mlp.experts.138.gate_proj", "model.layers.8.mlp.experts.139.gate_proj", "model.layers.8.mlp.experts.140.gate_proj", "model.layers.8.mlp.experts.141.gate_proj", "model.layers.8.mlp.experts.142.gate_proj", "model.layers.8.mlp.experts.143.gate_proj", "model.layers.8.mlp.experts.144.gate_proj", "model.layers.8.mlp.experts.145.gate_proj", "model.layers.8.mlp.experts.146.gate_proj", "model.layers.8.mlp.experts.147.gate_proj", "model.layers.8.mlp.experts.148.gate_proj", "model.layers.8.mlp.experts.149.gate_proj", "model.layers.8.mlp.experts.150.gate_proj", "model.layers.8.mlp.experts.151.gate_proj", "model.layers.8.mlp.experts.152.gate_proj", "model.layers.8.mlp.experts.153.gate_proj", "model.layers.8.mlp.experts.154.gate_proj", "model.layers.8.mlp.experts.155.gate_proj", "model.layers.8.mlp.experts.156.gate_proj", "model.layers.8.mlp.experts.157.gate_proj", "model.layers.8.mlp.experts.158.gate_proj", "model.layers.8.mlp.experts.159.gate_proj", "model.layers.8.mlp.experts.0.up_proj", "model.layers.8.mlp.experts.1.up_proj", "model.layers.8.mlp.experts.2.up_proj", "model.layers.8.mlp.experts.3.up_proj", "model.layers.8.mlp.experts.4.up_proj", "model.layers.8.mlp.experts.5.up_proj", "model.layers.8.mlp.experts.6.up_proj", "model.layers.8.mlp.experts.7.up_proj", "model.layers.8.mlp.experts.8.up_proj", "model.layers.8.mlp.experts.9.up_proj", "model.layers.8.mlp.experts.10.up_proj", "model.layers.8.mlp.experts.11.up_proj", "model.layers.8.mlp.experts.12.up_proj", "model.layers.8.mlp.experts.13.up_proj", "model.layers.8.mlp.experts.14.up_proj", "model.layers.8.mlp.experts.15.up_proj", "model.layers.8.mlp.experts.16.up_proj", "model.layers.8.mlp.experts.17.up_proj", "model.layers.8.mlp.experts.18.up_proj", "model.layers.8.mlp.experts.19.up_proj", "model.layers.8.mlp.experts.20.up_proj", "model.layers.8.mlp.experts.21.up_proj", "model.layers.8.mlp.experts.22.up_proj", "model.layers.8.mlp.experts.23.up_proj", "model.layers.8.mlp.experts.24.up_proj", "model.layers.8.mlp.experts.25.up_proj", "model.layers.8.mlp.experts.26.up_proj", "model.layers.8.mlp.experts.27.up_proj", "model.layers.8.mlp.experts.28.up_proj", "model.layers.8.mlp.experts.29.up_proj", "model.layers.8.mlp.experts.30.up_proj", "model.layers.8.mlp.experts.31.up_proj", "model.layers.8.mlp.experts.32.up_proj", "model.layers.8.mlp.experts.33.up_proj", "model.layers.8.mlp.experts.34.up_proj", "model.layers.8.mlp.experts.35.up_proj", "model.layers.8.mlp.experts.36.up_proj", "model.layers.8.mlp.experts.37.up_proj", "model.layers.8.mlp.experts.38.up_proj", "model.layers.8.mlp.experts.39.up_proj", "model.layers.8.mlp.experts.40.up_proj", "model.layers.8.mlp.experts.41.up_proj", "model.layers.8.mlp.experts.42.up_proj", "model.layers.8.mlp.experts.43.up_proj", "model.layers.8.mlp.experts.44.up_proj", "model.layers.8.mlp.experts.45.up_proj", "model.layers.8.mlp.experts.46.up_proj", "model.layers.8.mlp.experts.47.up_proj", "model.layers.8.mlp.experts.48.up_proj", "model.layers.8.mlp.experts.49.up_proj", "model.layers.8.mlp.experts.50.up_proj", "model.layers.8.mlp.experts.51.up_proj", "model.layers.8.mlp.experts.52.up_proj", "model.layers.8.mlp.experts.53.up_proj", "model.layers.8.mlp.experts.54.up_proj", "model.layers.8.mlp.experts.55.up_proj", "model.layers.8.mlp.experts.56.up_proj", "model.layers.8.mlp.experts.57.up_proj", "model.layers.8.mlp.experts.58.up_proj", "model.layers.8.mlp.experts.59.up_proj", "model.layers.8.mlp.experts.60.up_proj", "model.layers.8.mlp.experts.61.up_proj", "model.layers.8.mlp.experts.62.up_proj", "model.layers.8.mlp.experts.63.up_proj", "model.layers.8.mlp.experts.64.up_proj", "model.layers.8.mlp.experts.65.up_proj", "model.layers.8.mlp.experts.66.up_proj", "model.layers.8.mlp.experts.67.up_proj", "model.layers.8.mlp.experts.68.up_proj", "model.layers.8.mlp.experts.69.up_proj", "model.layers.8.mlp.experts.70.up_proj", "model.layers.8.mlp.experts.71.up_proj", "model.layers.8.mlp.experts.72.up_proj", "model.layers.8.mlp.experts.73.up_proj", "model.layers.8.mlp.experts.74.up_proj", "model.layers.8.mlp.experts.75.up_proj", "model.layers.8.mlp.experts.76.up_proj", "model.layers.8.mlp.experts.77.up_proj", "model.layers.8.mlp.experts.78.up_proj", "model.layers.8.mlp.experts.79.up_proj", "model.layers.8.mlp.experts.80.up_proj", "model.layers.8.mlp.experts.81.up_proj", "model.layers.8.mlp.experts.82.up_proj", "model.layers.8.mlp.experts.83.up_proj", "model.layers.8.mlp.experts.84.up_proj", "model.layers.8.mlp.experts.85.up_proj", "model.layers.8.mlp.experts.86.up_proj", "model.layers.8.mlp.experts.87.up_proj", "model.layers.8.mlp.experts.88.up_proj", "model.layers.8.mlp.experts.89.up_proj", "model.layers.8.mlp.experts.90.up_proj", "model.layers.8.mlp.experts.91.up_proj", "model.layers.8.mlp.experts.92.up_proj", "model.layers.8.mlp.experts.93.up_proj", "model.layers.8.mlp.experts.94.up_proj", "model.layers.8.mlp.experts.95.up_proj", "model.layers.8.mlp.experts.96.up_proj", "model.layers.8.mlp.experts.97.up_proj", "model.layers.8.mlp.experts.98.up_proj", "model.layers.8.mlp.experts.99.up_proj", "model.layers.8.mlp.experts.100.up_proj", "model.layers.8.mlp.experts.101.up_proj", "model.layers.8.mlp.experts.102.up_proj", "model.layers.8.mlp.experts.103.up_proj", "model.layers.8.mlp.experts.104.up_proj", "model.layers.8.mlp.experts.105.up_proj", "model.layers.8.mlp.experts.106.up_proj", "model.layers.8.mlp.experts.107.up_proj", "model.layers.8.mlp.experts.108.up_proj", "model.layers.8.mlp.experts.109.up_proj", "model.layers.8.mlp.experts.110.up_proj", "model.layers.8.mlp.experts.111.up_proj", "model.layers.8.mlp.experts.112.up_proj", "model.layers.8.mlp.experts.113.up_proj", "model.layers.8.mlp.experts.114.up_proj", "model.layers.8.mlp.experts.115.up_proj", "model.layers.8.mlp.experts.116.up_proj", "model.layers.8.mlp.experts.117.up_proj", "model.layers.8.mlp.experts.118.up_proj", "model.layers.8.mlp.experts.119.up_proj", "model.layers.8.mlp.experts.120.up_proj", "model.layers.8.mlp.experts.121.up_proj", "model.layers.8.mlp.experts.122.up_proj", "model.layers.8.mlp.experts.123.up_proj", "model.layers.8.mlp.experts.124.up_proj", "model.layers.8.mlp.experts.125.up_proj", "model.layers.8.mlp.experts.126.up_proj", "model.layers.8.mlp.experts.127.up_proj", "model.layers.8.mlp.experts.128.up_proj", "model.layers.8.mlp.experts.129.up_proj", "model.layers.8.mlp.experts.130.up_proj", "model.layers.8.mlp.experts.131.up_proj", "model.layers.8.mlp.experts.132.up_proj", "model.layers.8.mlp.experts.133.up_proj", "model.layers.8.mlp.experts.134.up_proj", "model.layers.8.mlp.experts.135.up_proj", "model.layers.8.mlp.experts.136.up_proj", "model.layers.8.mlp.experts.137.up_proj", "model.layers.8.mlp.experts.138.up_proj", "model.layers.8.mlp.experts.139.up_proj", "model.layers.8.mlp.experts.140.up_proj", "model.layers.8.mlp.experts.141.up_proj", "model.layers.8.mlp.experts.142.up_proj", "model.layers.8.mlp.experts.143.up_proj", "model.layers.8.mlp.experts.144.up_proj", "model.layers.8.mlp.experts.145.up_proj", "model.layers.8.mlp.experts.146.up_proj", "model.layers.8.mlp.experts.147.up_proj", "model.layers.8.mlp.experts.148.up_proj", "model.layers.8.mlp.experts.149.up_proj", "model.layers.8.mlp.experts.150.up_proj", "model.layers.8.mlp.experts.151.up_proj", "model.layers.8.mlp.experts.152.up_proj", "model.layers.8.mlp.experts.153.up_proj", "model.layers.8.mlp.experts.154.up_proj", "model.layers.8.mlp.experts.155.up_proj", "model.layers.8.mlp.experts.156.up_proj", "model.layers.8.mlp.experts.157.up_proj", "model.layers.8.mlp.experts.158.up_proj", "model.layers.8.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.001111799478530906, "dbits": 2516582400 } ] }, { "idx": 50, "layers": [ "model.layers.8.mlp.experts.0.down_proj", "model.layers.8.mlp.experts.1.down_proj", "model.layers.8.mlp.experts.2.down_proj", "model.layers.8.mlp.experts.3.down_proj", "model.layers.8.mlp.experts.4.down_proj", "model.layers.8.mlp.experts.5.down_proj", "model.layers.8.mlp.experts.6.down_proj", "model.layers.8.mlp.experts.7.down_proj", "model.layers.8.mlp.experts.8.down_proj", "model.layers.8.mlp.experts.9.down_proj", "model.layers.8.mlp.experts.10.down_proj", "model.layers.8.mlp.experts.11.down_proj", "model.layers.8.mlp.experts.12.down_proj", "model.layers.8.mlp.experts.13.down_proj", "model.layers.8.mlp.experts.14.down_proj", "model.layers.8.mlp.experts.15.down_proj", "model.layers.8.mlp.experts.16.down_proj", "model.layers.8.mlp.experts.17.down_proj", "model.layers.8.mlp.experts.18.down_proj", "model.layers.8.mlp.experts.19.down_proj", "model.layers.8.mlp.experts.20.down_proj", "model.layers.8.mlp.experts.21.down_proj", "model.layers.8.mlp.experts.22.down_proj", "model.layers.8.mlp.experts.23.down_proj", "model.layers.8.mlp.experts.24.down_proj", "model.layers.8.mlp.experts.25.down_proj", "model.layers.8.mlp.experts.26.down_proj", "model.layers.8.mlp.experts.27.down_proj", "model.layers.8.mlp.experts.28.down_proj", "model.layers.8.mlp.experts.29.down_proj", "model.layers.8.mlp.experts.30.down_proj", "model.layers.8.mlp.experts.31.down_proj", "model.layers.8.mlp.experts.32.down_proj", "model.layers.8.mlp.experts.33.down_proj", "model.layers.8.mlp.experts.34.down_proj", "model.layers.8.mlp.experts.35.down_proj", "model.layers.8.mlp.experts.36.down_proj", "model.layers.8.mlp.experts.37.down_proj", "model.layers.8.mlp.experts.38.down_proj", "model.layers.8.mlp.experts.39.down_proj", "model.layers.8.mlp.experts.40.down_proj", "model.layers.8.mlp.experts.41.down_proj", "model.layers.8.mlp.experts.42.down_proj", "model.layers.8.mlp.experts.43.down_proj", "model.layers.8.mlp.experts.44.down_proj", "model.layers.8.mlp.experts.45.down_proj", "model.layers.8.mlp.experts.46.down_proj", "model.layers.8.mlp.experts.47.down_proj", "model.layers.8.mlp.experts.48.down_proj", "model.layers.8.mlp.experts.49.down_proj", "model.layers.8.mlp.experts.50.down_proj", "model.layers.8.mlp.experts.51.down_proj", "model.layers.8.mlp.experts.52.down_proj", "model.layers.8.mlp.experts.53.down_proj", "model.layers.8.mlp.experts.54.down_proj", "model.layers.8.mlp.experts.55.down_proj", "model.layers.8.mlp.experts.56.down_proj", "model.layers.8.mlp.experts.57.down_proj", "model.layers.8.mlp.experts.58.down_proj", "model.layers.8.mlp.experts.59.down_proj", "model.layers.8.mlp.experts.60.down_proj", "model.layers.8.mlp.experts.61.down_proj", "model.layers.8.mlp.experts.62.down_proj", "model.layers.8.mlp.experts.63.down_proj", "model.layers.8.mlp.experts.64.down_proj", "model.layers.8.mlp.experts.65.down_proj", "model.layers.8.mlp.experts.66.down_proj", "model.layers.8.mlp.experts.67.down_proj", "model.layers.8.mlp.experts.68.down_proj", "model.layers.8.mlp.experts.69.down_proj", "model.layers.8.mlp.experts.70.down_proj", "model.layers.8.mlp.experts.71.down_proj", "model.layers.8.mlp.experts.72.down_proj", "model.layers.8.mlp.experts.73.down_proj", "model.layers.8.mlp.experts.74.down_proj", "model.layers.8.mlp.experts.75.down_proj", "model.layers.8.mlp.experts.76.down_proj", "model.layers.8.mlp.experts.77.down_proj", "model.layers.8.mlp.experts.78.down_proj", "model.layers.8.mlp.experts.79.down_proj", "model.layers.8.mlp.experts.80.down_proj", "model.layers.8.mlp.experts.81.down_proj", "model.layers.8.mlp.experts.82.down_proj", "model.layers.8.mlp.experts.83.down_proj", "model.layers.8.mlp.experts.84.down_proj", "model.layers.8.mlp.experts.85.down_proj", "model.layers.8.mlp.experts.86.down_proj", "model.layers.8.mlp.experts.87.down_proj", "model.layers.8.mlp.experts.88.down_proj", "model.layers.8.mlp.experts.89.down_proj", "model.layers.8.mlp.experts.90.down_proj", "model.layers.8.mlp.experts.91.down_proj", "model.layers.8.mlp.experts.92.down_proj", "model.layers.8.mlp.experts.93.down_proj", "model.layers.8.mlp.experts.94.down_proj", "model.layers.8.mlp.experts.95.down_proj", "model.layers.8.mlp.experts.96.down_proj", "model.layers.8.mlp.experts.97.down_proj", "model.layers.8.mlp.experts.98.down_proj", "model.layers.8.mlp.experts.99.down_proj", "model.layers.8.mlp.experts.100.down_proj", "model.layers.8.mlp.experts.101.down_proj", "model.layers.8.mlp.experts.102.down_proj", "model.layers.8.mlp.experts.103.down_proj", "model.layers.8.mlp.experts.104.down_proj", "model.layers.8.mlp.experts.105.down_proj", "model.layers.8.mlp.experts.106.down_proj", "model.layers.8.mlp.experts.107.down_proj", "model.layers.8.mlp.experts.108.down_proj", "model.layers.8.mlp.experts.109.down_proj", "model.layers.8.mlp.experts.110.down_proj", "model.layers.8.mlp.experts.111.down_proj", "model.layers.8.mlp.experts.112.down_proj", "model.layers.8.mlp.experts.113.down_proj", "model.layers.8.mlp.experts.114.down_proj", "model.layers.8.mlp.experts.115.down_proj", "model.layers.8.mlp.experts.116.down_proj", "model.layers.8.mlp.experts.117.down_proj", "model.layers.8.mlp.experts.118.down_proj", "model.layers.8.mlp.experts.119.down_proj", "model.layers.8.mlp.experts.120.down_proj", "model.layers.8.mlp.experts.121.down_proj", "model.layers.8.mlp.experts.122.down_proj", "model.layers.8.mlp.experts.123.down_proj", "model.layers.8.mlp.experts.124.down_proj", "model.layers.8.mlp.experts.125.down_proj", "model.layers.8.mlp.experts.126.down_proj", "model.layers.8.mlp.experts.127.down_proj", "model.layers.8.mlp.experts.128.down_proj", "model.layers.8.mlp.experts.129.down_proj", "model.layers.8.mlp.experts.130.down_proj", "model.layers.8.mlp.experts.131.down_proj", "model.layers.8.mlp.experts.132.down_proj", "model.layers.8.mlp.experts.133.down_proj", "model.layers.8.mlp.experts.134.down_proj", "model.layers.8.mlp.experts.135.down_proj", "model.layers.8.mlp.experts.136.down_proj", "model.layers.8.mlp.experts.137.down_proj", "model.layers.8.mlp.experts.138.down_proj", "model.layers.8.mlp.experts.139.down_proj", "model.layers.8.mlp.experts.140.down_proj", "model.layers.8.mlp.experts.141.down_proj", "model.layers.8.mlp.experts.142.down_proj", "model.layers.8.mlp.experts.143.down_proj", "model.layers.8.mlp.experts.144.down_proj", "model.layers.8.mlp.experts.145.down_proj", "model.layers.8.mlp.experts.146.down_proj", "model.layers.8.mlp.experts.147.down_proj", "model.layers.8.mlp.experts.148.down_proj", "model.layers.8.mlp.experts.149.down_proj", "model.layers.8.mlp.experts.150.down_proj", "model.layers.8.mlp.experts.151.down_proj", "model.layers.8.mlp.experts.152.down_proj", "model.layers.8.mlp.experts.153.down_proj", "model.layers.8.mlp.experts.154.down_proj", "model.layers.8.mlp.experts.155.down_proj", "model.layers.8.mlp.experts.156.down_proj", "model.layers.8.mlp.experts.157.down_proj", "model.layers.8.mlp.experts.158.down_proj", "model.layers.8.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0010914742946624978, "dbits": 1258291200 } ] }, { "idx": 51, "layers": [ "model.layers.9.self_attn.q_proj" ], "candidates": [ { "dkld": 0.000827284902334191, "dbits": 62914560 } ] }, { "idx": 52, "layers": [ "model.layers.9.self_attn.k_proj", "model.layers.9.self_attn.v_proj" ], "candidates": [ { "dkld": 0.002393518388271343, "dbits": 10485760 } ] }, { "idx": 53, "layers": [ "model.layers.9.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00044346824288371556, "dbits": 62914560 } ] }, { "idx": 54, "layers": [ "model.layers.9.mlp.shared_experts.gate_proj", "model.layers.9.mlp.shared_experts.up_proj", "model.layers.9.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0007160231471061707, "dbits": 23592960 } ] }, { "idx": 55, "layers": [ "model.layers.9.mlp.experts.0.gate_proj", "model.layers.9.mlp.experts.1.gate_proj", "model.layers.9.mlp.experts.2.gate_proj", "model.layers.9.mlp.experts.3.gate_proj", "model.layers.9.mlp.experts.4.gate_proj", "model.layers.9.mlp.experts.5.gate_proj", "model.layers.9.mlp.experts.6.gate_proj", "model.layers.9.mlp.experts.7.gate_proj", "model.layers.9.mlp.experts.8.gate_proj", "model.layers.9.mlp.experts.9.gate_proj", "model.layers.9.mlp.experts.10.gate_proj", "model.layers.9.mlp.experts.11.gate_proj", "model.layers.9.mlp.experts.12.gate_proj", "model.layers.9.mlp.experts.13.gate_proj", "model.layers.9.mlp.experts.14.gate_proj", "model.layers.9.mlp.experts.15.gate_proj", "model.layers.9.mlp.experts.16.gate_proj", "model.layers.9.mlp.experts.17.gate_proj", "model.layers.9.mlp.experts.18.gate_proj", "model.layers.9.mlp.experts.19.gate_proj", "model.layers.9.mlp.experts.20.gate_proj", "model.layers.9.mlp.experts.21.gate_proj", "model.layers.9.mlp.experts.22.gate_proj", "model.layers.9.mlp.experts.23.gate_proj", "model.layers.9.mlp.experts.24.gate_proj", "model.layers.9.mlp.experts.25.gate_proj", "model.layers.9.mlp.experts.26.gate_proj", "model.layers.9.mlp.experts.27.gate_proj", "model.layers.9.mlp.experts.28.gate_proj", "model.layers.9.mlp.experts.29.gate_proj", "model.layers.9.mlp.experts.30.gate_proj", "model.layers.9.mlp.experts.31.gate_proj", "model.layers.9.mlp.experts.32.gate_proj", "model.layers.9.mlp.experts.33.gate_proj", "model.layers.9.mlp.experts.34.gate_proj", "model.layers.9.mlp.experts.35.gate_proj", "model.layers.9.mlp.experts.36.gate_proj", "model.layers.9.mlp.experts.37.gate_proj", "model.layers.9.mlp.experts.38.gate_proj", "model.layers.9.mlp.experts.39.gate_proj", "model.layers.9.mlp.experts.40.gate_proj", "model.layers.9.mlp.experts.41.gate_proj", "model.layers.9.mlp.experts.42.gate_proj", "model.layers.9.mlp.experts.43.gate_proj", "model.layers.9.mlp.experts.44.gate_proj", "model.layers.9.mlp.experts.45.gate_proj", "model.layers.9.mlp.experts.46.gate_proj", "model.layers.9.mlp.experts.47.gate_proj", "model.layers.9.mlp.experts.48.gate_proj", "model.layers.9.mlp.experts.49.gate_proj", "model.layers.9.mlp.experts.50.gate_proj", "model.layers.9.mlp.experts.51.gate_proj", "model.layers.9.mlp.experts.52.gate_proj", "model.layers.9.mlp.experts.53.gate_proj", "model.layers.9.mlp.experts.54.gate_proj", "model.layers.9.mlp.experts.55.gate_proj", "model.layers.9.mlp.experts.56.gate_proj", "model.layers.9.mlp.experts.57.gate_proj", "model.layers.9.mlp.experts.58.gate_proj", "model.layers.9.mlp.experts.59.gate_proj", "model.layers.9.mlp.experts.60.gate_proj", "model.layers.9.mlp.experts.61.gate_proj", "model.layers.9.mlp.experts.62.gate_proj", "model.layers.9.mlp.experts.63.gate_proj", "model.layers.9.mlp.experts.64.gate_proj", "model.layers.9.mlp.experts.65.gate_proj", "model.layers.9.mlp.experts.66.gate_proj", "model.layers.9.mlp.experts.67.gate_proj", "model.layers.9.mlp.experts.68.gate_proj", "model.layers.9.mlp.experts.69.gate_proj", "model.layers.9.mlp.experts.70.gate_proj", "model.layers.9.mlp.experts.71.gate_proj", "model.layers.9.mlp.experts.72.gate_proj", "model.layers.9.mlp.experts.73.gate_proj", "model.layers.9.mlp.experts.74.gate_proj", "model.layers.9.mlp.experts.75.gate_proj", "model.layers.9.mlp.experts.76.gate_proj", "model.layers.9.mlp.experts.77.gate_proj", "model.layers.9.mlp.experts.78.gate_proj", "model.layers.9.mlp.experts.79.gate_proj", "model.layers.9.mlp.experts.80.gate_proj", "model.layers.9.mlp.experts.81.gate_proj", "model.layers.9.mlp.experts.82.gate_proj", "model.layers.9.mlp.experts.83.gate_proj", "model.layers.9.mlp.experts.84.gate_proj", "model.layers.9.mlp.experts.85.gate_proj", "model.layers.9.mlp.experts.86.gate_proj", "model.layers.9.mlp.experts.87.gate_proj", "model.layers.9.mlp.experts.88.gate_proj", "model.layers.9.mlp.experts.89.gate_proj", "model.layers.9.mlp.experts.90.gate_proj", "model.layers.9.mlp.experts.91.gate_proj", "model.layers.9.mlp.experts.92.gate_proj", "model.layers.9.mlp.experts.93.gate_proj", "model.layers.9.mlp.experts.94.gate_proj", "model.layers.9.mlp.experts.95.gate_proj", "model.layers.9.mlp.experts.96.gate_proj", "model.layers.9.mlp.experts.97.gate_proj", "model.layers.9.mlp.experts.98.gate_proj", "model.layers.9.mlp.experts.99.gate_proj", "model.layers.9.mlp.experts.100.gate_proj", "model.layers.9.mlp.experts.101.gate_proj", "model.layers.9.mlp.experts.102.gate_proj", "model.layers.9.mlp.experts.103.gate_proj", "model.layers.9.mlp.experts.104.gate_proj", "model.layers.9.mlp.experts.105.gate_proj", "model.layers.9.mlp.experts.106.gate_proj", "model.layers.9.mlp.experts.107.gate_proj", "model.layers.9.mlp.experts.108.gate_proj", "model.layers.9.mlp.experts.109.gate_proj", "model.layers.9.mlp.experts.110.gate_proj", "model.layers.9.mlp.experts.111.gate_proj", "model.layers.9.mlp.experts.112.gate_proj", "model.layers.9.mlp.experts.113.gate_proj", "model.layers.9.mlp.experts.114.gate_proj", "model.layers.9.mlp.experts.115.gate_proj", "model.layers.9.mlp.experts.116.gate_proj", "model.layers.9.mlp.experts.117.gate_proj", "model.layers.9.mlp.experts.118.gate_proj", "model.layers.9.mlp.experts.119.gate_proj", "model.layers.9.mlp.experts.120.gate_proj", "model.layers.9.mlp.experts.121.gate_proj", "model.layers.9.mlp.experts.122.gate_proj", "model.layers.9.mlp.experts.123.gate_proj", "model.layers.9.mlp.experts.124.gate_proj", "model.layers.9.mlp.experts.125.gate_proj", "model.layers.9.mlp.experts.126.gate_proj", "model.layers.9.mlp.experts.127.gate_proj", "model.layers.9.mlp.experts.128.gate_proj", "model.layers.9.mlp.experts.129.gate_proj", "model.layers.9.mlp.experts.130.gate_proj", "model.layers.9.mlp.experts.131.gate_proj", "model.layers.9.mlp.experts.132.gate_proj", "model.layers.9.mlp.experts.133.gate_proj", "model.layers.9.mlp.experts.134.gate_proj", "model.layers.9.mlp.experts.135.gate_proj", "model.layers.9.mlp.experts.136.gate_proj", "model.layers.9.mlp.experts.137.gate_proj", "model.layers.9.mlp.experts.138.gate_proj", "model.layers.9.mlp.experts.139.gate_proj", "model.layers.9.mlp.experts.140.gate_proj", "model.layers.9.mlp.experts.141.gate_proj", "model.layers.9.mlp.experts.142.gate_proj", "model.layers.9.mlp.experts.143.gate_proj", "model.layers.9.mlp.experts.144.gate_proj", "model.layers.9.mlp.experts.145.gate_proj", "model.layers.9.mlp.experts.146.gate_proj", "model.layers.9.mlp.experts.147.gate_proj", "model.layers.9.mlp.experts.148.gate_proj", "model.layers.9.mlp.experts.149.gate_proj", "model.layers.9.mlp.experts.150.gate_proj", "model.layers.9.mlp.experts.151.gate_proj", "model.layers.9.mlp.experts.152.gate_proj", "model.layers.9.mlp.experts.153.gate_proj", "model.layers.9.mlp.experts.154.gate_proj", "model.layers.9.mlp.experts.155.gate_proj", "model.layers.9.mlp.experts.156.gate_proj", "model.layers.9.mlp.experts.157.gate_proj", "model.layers.9.mlp.experts.158.gate_proj", "model.layers.9.mlp.experts.159.gate_proj", "model.layers.9.mlp.experts.0.up_proj", "model.layers.9.mlp.experts.1.up_proj", "model.layers.9.mlp.experts.2.up_proj", "model.layers.9.mlp.experts.3.up_proj", "model.layers.9.mlp.experts.4.up_proj", "model.layers.9.mlp.experts.5.up_proj", "model.layers.9.mlp.experts.6.up_proj", "model.layers.9.mlp.experts.7.up_proj", "model.layers.9.mlp.experts.8.up_proj", "model.layers.9.mlp.experts.9.up_proj", "model.layers.9.mlp.experts.10.up_proj", "model.layers.9.mlp.experts.11.up_proj", "model.layers.9.mlp.experts.12.up_proj", "model.layers.9.mlp.experts.13.up_proj", "model.layers.9.mlp.experts.14.up_proj", "model.layers.9.mlp.experts.15.up_proj", "model.layers.9.mlp.experts.16.up_proj", "model.layers.9.mlp.experts.17.up_proj", "model.layers.9.mlp.experts.18.up_proj", "model.layers.9.mlp.experts.19.up_proj", "model.layers.9.mlp.experts.20.up_proj", "model.layers.9.mlp.experts.21.up_proj", "model.layers.9.mlp.experts.22.up_proj", "model.layers.9.mlp.experts.23.up_proj", "model.layers.9.mlp.experts.24.up_proj", "model.layers.9.mlp.experts.25.up_proj", "model.layers.9.mlp.experts.26.up_proj", "model.layers.9.mlp.experts.27.up_proj", "model.layers.9.mlp.experts.28.up_proj", "model.layers.9.mlp.experts.29.up_proj", "model.layers.9.mlp.experts.30.up_proj", "model.layers.9.mlp.experts.31.up_proj", "model.layers.9.mlp.experts.32.up_proj", "model.layers.9.mlp.experts.33.up_proj", "model.layers.9.mlp.experts.34.up_proj", "model.layers.9.mlp.experts.35.up_proj", "model.layers.9.mlp.experts.36.up_proj", "model.layers.9.mlp.experts.37.up_proj", "model.layers.9.mlp.experts.38.up_proj", "model.layers.9.mlp.experts.39.up_proj", "model.layers.9.mlp.experts.40.up_proj", "model.layers.9.mlp.experts.41.up_proj", "model.layers.9.mlp.experts.42.up_proj", "model.layers.9.mlp.experts.43.up_proj", "model.layers.9.mlp.experts.44.up_proj", "model.layers.9.mlp.experts.45.up_proj", "model.layers.9.mlp.experts.46.up_proj", "model.layers.9.mlp.experts.47.up_proj", "model.layers.9.mlp.experts.48.up_proj", "model.layers.9.mlp.experts.49.up_proj", "model.layers.9.mlp.experts.50.up_proj", "model.layers.9.mlp.experts.51.up_proj", "model.layers.9.mlp.experts.52.up_proj", "model.layers.9.mlp.experts.53.up_proj", "model.layers.9.mlp.experts.54.up_proj", "model.layers.9.mlp.experts.55.up_proj", "model.layers.9.mlp.experts.56.up_proj", "model.layers.9.mlp.experts.57.up_proj", "model.layers.9.mlp.experts.58.up_proj", "model.layers.9.mlp.experts.59.up_proj", "model.layers.9.mlp.experts.60.up_proj", "model.layers.9.mlp.experts.61.up_proj", "model.layers.9.mlp.experts.62.up_proj", "model.layers.9.mlp.experts.63.up_proj", "model.layers.9.mlp.experts.64.up_proj", "model.layers.9.mlp.experts.65.up_proj", "model.layers.9.mlp.experts.66.up_proj", "model.layers.9.mlp.experts.67.up_proj", "model.layers.9.mlp.experts.68.up_proj", "model.layers.9.mlp.experts.69.up_proj", "model.layers.9.mlp.experts.70.up_proj", "model.layers.9.mlp.experts.71.up_proj", "model.layers.9.mlp.experts.72.up_proj", "model.layers.9.mlp.experts.73.up_proj", "model.layers.9.mlp.experts.74.up_proj", "model.layers.9.mlp.experts.75.up_proj", "model.layers.9.mlp.experts.76.up_proj", "model.layers.9.mlp.experts.77.up_proj", "model.layers.9.mlp.experts.78.up_proj", "model.layers.9.mlp.experts.79.up_proj", "model.layers.9.mlp.experts.80.up_proj", "model.layers.9.mlp.experts.81.up_proj", "model.layers.9.mlp.experts.82.up_proj", "model.layers.9.mlp.experts.83.up_proj", "model.layers.9.mlp.experts.84.up_proj", "model.layers.9.mlp.experts.85.up_proj", "model.layers.9.mlp.experts.86.up_proj", "model.layers.9.mlp.experts.87.up_proj", "model.layers.9.mlp.experts.88.up_proj", "model.layers.9.mlp.experts.89.up_proj", "model.layers.9.mlp.experts.90.up_proj", "model.layers.9.mlp.experts.91.up_proj", "model.layers.9.mlp.experts.92.up_proj", "model.layers.9.mlp.experts.93.up_proj", "model.layers.9.mlp.experts.94.up_proj", "model.layers.9.mlp.experts.95.up_proj", "model.layers.9.mlp.experts.96.up_proj", "model.layers.9.mlp.experts.97.up_proj", "model.layers.9.mlp.experts.98.up_proj", "model.layers.9.mlp.experts.99.up_proj", "model.layers.9.mlp.experts.100.up_proj", "model.layers.9.mlp.experts.101.up_proj", "model.layers.9.mlp.experts.102.up_proj", "model.layers.9.mlp.experts.103.up_proj", "model.layers.9.mlp.experts.104.up_proj", "model.layers.9.mlp.experts.105.up_proj", "model.layers.9.mlp.experts.106.up_proj", "model.layers.9.mlp.experts.107.up_proj", "model.layers.9.mlp.experts.108.up_proj", "model.layers.9.mlp.experts.109.up_proj", "model.layers.9.mlp.experts.110.up_proj", "model.layers.9.mlp.experts.111.up_proj", "model.layers.9.mlp.experts.112.up_proj", "model.layers.9.mlp.experts.113.up_proj", "model.layers.9.mlp.experts.114.up_proj", "model.layers.9.mlp.experts.115.up_proj", "model.layers.9.mlp.experts.116.up_proj", "model.layers.9.mlp.experts.117.up_proj", "model.layers.9.mlp.experts.118.up_proj", "model.layers.9.mlp.experts.119.up_proj", "model.layers.9.mlp.experts.120.up_proj", "model.layers.9.mlp.experts.121.up_proj", "model.layers.9.mlp.experts.122.up_proj", "model.layers.9.mlp.experts.123.up_proj", "model.layers.9.mlp.experts.124.up_proj", "model.layers.9.mlp.experts.125.up_proj", "model.layers.9.mlp.experts.126.up_proj", "model.layers.9.mlp.experts.127.up_proj", "model.layers.9.mlp.experts.128.up_proj", "model.layers.9.mlp.experts.129.up_proj", "model.layers.9.mlp.experts.130.up_proj", "model.layers.9.mlp.experts.131.up_proj", "model.layers.9.mlp.experts.132.up_proj", "model.layers.9.mlp.experts.133.up_proj", "model.layers.9.mlp.experts.134.up_proj", "model.layers.9.mlp.experts.135.up_proj", "model.layers.9.mlp.experts.136.up_proj", "model.layers.9.mlp.experts.137.up_proj", "model.layers.9.mlp.experts.138.up_proj", "model.layers.9.mlp.experts.139.up_proj", "model.layers.9.mlp.experts.140.up_proj", "model.layers.9.mlp.experts.141.up_proj", "model.layers.9.mlp.experts.142.up_proj", "model.layers.9.mlp.experts.143.up_proj", "model.layers.9.mlp.experts.144.up_proj", "model.layers.9.mlp.experts.145.up_proj", "model.layers.9.mlp.experts.146.up_proj", "model.layers.9.mlp.experts.147.up_proj", "model.layers.9.mlp.experts.148.up_proj", "model.layers.9.mlp.experts.149.up_proj", "model.layers.9.mlp.experts.150.up_proj", "model.layers.9.mlp.experts.151.up_proj", "model.layers.9.mlp.experts.152.up_proj", "model.layers.9.mlp.experts.153.up_proj", "model.layers.9.mlp.experts.154.up_proj", "model.layers.9.mlp.experts.155.up_proj", "model.layers.9.mlp.experts.156.up_proj", "model.layers.9.mlp.experts.157.up_proj", "model.layers.9.mlp.experts.158.up_proj", "model.layers.9.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0017846845090389585, "dbits": 2516582400 } ] }, { "idx": 56, "layers": [ "model.layers.9.mlp.experts.0.down_proj", "model.layers.9.mlp.experts.1.down_proj", "model.layers.9.mlp.experts.2.down_proj", "model.layers.9.mlp.experts.3.down_proj", "model.layers.9.mlp.experts.4.down_proj", "model.layers.9.mlp.experts.5.down_proj", "model.layers.9.mlp.experts.6.down_proj", "model.layers.9.mlp.experts.7.down_proj", "model.layers.9.mlp.experts.8.down_proj", "model.layers.9.mlp.experts.9.down_proj", "model.layers.9.mlp.experts.10.down_proj", "model.layers.9.mlp.experts.11.down_proj", "model.layers.9.mlp.experts.12.down_proj", "model.layers.9.mlp.experts.13.down_proj", "model.layers.9.mlp.experts.14.down_proj", "model.layers.9.mlp.experts.15.down_proj", "model.layers.9.mlp.experts.16.down_proj", "model.layers.9.mlp.experts.17.down_proj", "model.layers.9.mlp.experts.18.down_proj", "model.layers.9.mlp.experts.19.down_proj", "model.layers.9.mlp.experts.20.down_proj", "model.layers.9.mlp.experts.21.down_proj", "model.layers.9.mlp.experts.22.down_proj", "model.layers.9.mlp.experts.23.down_proj", "model.layers.9.mlp.experts.24.down_proj", "model.layers.9.mlp.experts.25.down_proj", "model.layers.9.mlp.experts.26.down_proj", "model.layers.9.mlp.experts.27.down_proj", "model.layers.9.mlp.experts.28.down_proj", "model.layers.9.mlp.experts.29.down_proj", "model.layers.9.mlp.experts.30.down_proj", "model.layers.9.mlp.experts.31.down_proj", "model.layers.9.mlp.experts.32.down_proj", "model.layers.9.mlp.experts.33.down_proj", "model.layers.9.mlp.experts.34.down_proj", "model.layers.9.mlp.experts.35.down_proj", "model.layers.9.mlp.experts.36.down_proj", "model.layers.9.mlp.experts.37.down_proj", "model.layers.9.mlp.experts.38.down_proj", "model.layers.9.mlp.experts.39.down_proj", "model.layers.9.mlp.experts.40.down_proj", "model.layers.9.mlp.experts.41.down_proj", "model.layers.9.mlp.experts.42.down_proj", "model.layers.9.mlp.experts.43.down_proj", "model.layers.9.mlp.experts.44.down_proj", "model.layers.9.mlp.experts.45.down_proj", "model.layers.9.mlp.experts.46.down_proj", "model.layers.9.mlp.experts.47.down_proj", "model.layers.9.mlp.experts.48.down_proj", "model.layers.9.mlp.experts.49.down_proj", "model.layers.9.mlp.experts.50.down_proj", "model.layers.9.mlp.experts.51.down_proj", "model.layers.9.mlp.experts.52.down_proj", "model.layers.9.mlp.experts.53.down_proj", "model.layers.9.mlp.experts.54.down_proj", "model.layers.9.mlp.experts.55.down_proj", "model.layers.9.mlp.experts.56.down_proj", "model.layers.9.mlp.experts.57.down_proj", "model.layers.9.mlp.experts.58.down_proj", "model.layers.9.mlp.experts.59.down_proj", "model.layers.9.mlp.experts.60.down_proj", "model.layers.9.mlp.experts.61.down_proj", "model.layers.9.mlp.experts.62.down_proj", "model.layers.9.mlp.experts.63.down_proj", "model.layers.9.mlp.experts.64.down_proj", "model.layers.9.mlp.experts.65.down_proj", "model.layers.9.mlp.experts.66.down_proj", "model.layers.9.mlp.experts.67.down_proj", "model.layers.9.mlp.experts.68.down_proj", "model.layers.9.mlp.experts.69.down_proj", "model.layers.9.mlp.experts.70.down_proj", "model.layers.9.mlp.experts.71.down_proj", "model.layers.9.mlp.experts.72.down_proj", "model.layers.9.mlp.experts.73.down_proj", "model.layers.9.mlp.experts.74.down_proj", "model.layers.9.mlp.experts.75.down_proj", "model.layers.9.mlp.experts.76.down_proj", "model.layers.9.mlp.experts.77.down_proj", "model.layers.9.mlp.experts.78.down_proj", "model.layers.9.mlp.experts.79.down_proj", "model.layers.9.mlp.experts.80.down_proj", "model.layers.9.mlp.experts.81.down_proj", "model.layers.9.mlp.experts.82.down_proj", "model.layers.9.mlp.experts.83.down_proj", "model.layers.9.mlp.experts.84.down_proj", "model.layers.9.mlp.experts.85.down_proj", "model.layers.9.mlp.experts.86.down_proj", "model.layers.9.mlp.experts.87.down_proj", "model.layers.9.mlp.experts.88.down_proj", "model.layers.9.mlp.experts.89.down_proj", "model.layers.9.mlp.experts.90.down_proj", "model.layers.9.mlp.experts.91.down_proj", "model.layers.9.mlp.experts.92.down_proj", "model.layers.9.mlp.experts.93.down_proj", "model.layers.9.mlp.experts.94.down_proj", "model.layers.9.mlp.experts.95.down_proj", "model.layers.9.mlp.experts.96.down_proj", "model.layers.9.mlp.experts.97.down_proj", "model.layers.9.mlp.experts.98.down_proj", "model.layers.9.mlp.experts.99.down_proj", "model.layers.9.mlp.experts.100.down_proj", "model.layers.9.mlp.experts.101.down_proj", "model.layers.9.mlp.experts.102.down_proj", "model.layers.9.mlp.experts.103.down_proj", "model.layers.9.mlp.experts.104.down_proj", "model.layers.9.mlp.experts.105.down_proj", "model.layers.9.mlp.experts.106.down_proj", "model.layers.9.mlp.experts.107.down_proj", "model.layers.9.mlp.experts.108.down_proj", "model.layers.9.mlp.experts.109.down_proj", "model.layers.9.mlp.experts.110.down_proj", "model.layers.9.mlp.experts.111.down_proj", "model.layers.9.mlp.experts.112.down_proj", "model.layers.9.mlp.experts.113.down_proj", "model.layers.9.mlp.experts.114.down_proj", "model.layers.9.mlp.experts.115.down_proj", "model.layers.9.mlp.experts.116.down_proj", "model.layers.9.mlp.experts.117.down_proj", "model.layers.9.mlp.experts.118.down_proj", "model.layers.9.mlp.experts.119.down_proj", "model.layers.9.mlp.experts.120.down_proj", "model.layers.9.mlp.experts.121.down_proj", "model.layers.9.mlp.experts.122.down_proj", "model.layers.9.mlp.experts.123.down_proj", "model.layers.9.mlp.experts.124.down_proj", "model.layers.9.mlp.experts.125.down_proj", "model.layers.9.mlp.experts.126.down_proj", "model.layers.9.mlp.experts.127.down_proj", "model.layers.9.mlp.experts.128.down_proj", "model.layers.9.mlp.experts.129.down_proj", "model.layers.9.mlp.experts.130.down_proj", "model.layers.9.mlp.experts.131.down_proj", "model.layers.9.mlp.experts.132.down_proj", "model.layers.9.mlp.experts.133.down_proj", "model.layers.9.mlp.experts.134.down_proj", "model.layers.9.mlp.experts.135.down_proj", "model.layers.9.mlp.experts.136.down_proj", "model.layers.9.mlp.experts.137.down_proj", "model.layers.9.mlp.experts.138.down_proj", "model.layers.9.mlp.experts.139.down_proj", "model.layers.9.mlp.experts.140.down_proj", "model.layers.9.mlp.experts.141.down_proj", "model.layers.9.mlp.experts.142.down_proj", "model.layers.9.mlp.experts.143.down_proj", "model.layers.9.mlp.experts.144.down_proj", "model.layers.9.mlp.experts.145.down_proj", "model.layers.9.mlp.experts.146.down_proj", "model.layers.9.mlp.experts.147.down_proj", "model.layers.9.mlp.experts.148.down_proj", "model.layers.9.mlp.experts.149.down_proj", "model.layers.9.mlp.experts.150.down_proj", "model.layers.9.mlp.experts.151.down_proj", "model.layers.9.mlp.experts.152.down_proj", "model.layers.9.mlp.experts.153.down_proj", "model.layers.9.mlp.experts.154.down_proj", "model.layers.9.mlp.experts.155.down_proj", "model.layers.9.mlp.experts.156.down_proj", "model.layers.9.mlp.experts.157.down_proj", "model.layers.9.mlp.experts.158.down_proj", "model.layers.9.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0014043860137462616, "dbits": 1258291200 } ] }, { "idx": 57, "layers": [ "model.layers.10.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0005592077970504872, "dbits": 62914560 } ] }, { "idx": 58, "layers": [ "model.layers.10.self_attn.k_proj", "model.layers.10.self_attn.v_proj" ], "candidates": [ { "dkld": 8.181929588318981e-05, "dbits": 10485760 } ] }, { "idx": 59, "layers": [ "model.layers.10.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0026438407599925773, "dbits": 62914560 } ] }, { "idx": 60, "layers": [ "model.layers.10.mlp.shared_experts.gate_proj", "model.layers.10.mlp.shared_experts.up_proj", "model.layers.10.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0023736536502838024, "dbits": 23592960 } ] }, { "idx": 61, "layers": [ "model.layers.10.mlp.experts.0.gate_proj", "model.layers.10.mlp.experts.1.gate_proj", "model.layers.10.mlp.experts.2.gate_proj", "model.layers.10.mlp.experts.3.gate_proj", "model.layers.10.mlp.experts.4.gate_proj", "model.layers.10.mlp.experts.5.gate_proj", "model.layers.10.mlp.experts.6.gate_proj", "model.layers.10.mlp.experts.7.gate_proj", "model.layers.10.mlp.experts.8.gate_proj", "model.layers.10.mlp.experts.9.gate_proj", "model.layers.10.mlp.experts.10.gate_proj", "model.layers.10.mlp.experts.11.gate_proj", "model.layers.10.mlp.experts.12.gate_proj", "model.layers.10.mlp.experts.13.gate_proj", "model.layers.10.mlp.experts.14.gate_proj", "model.layers.10.mlp.experts.15.gate_proj", "model.layers.10.mlp.experts.16.gate_proj", "model.layers.10.mlp.experts.17.gate_proj", "model.layers.10.mlp.experts.18.gate_proj", "model.layers.10.mlp.experts.19.gate_proj", "model.layers.10.mlp.experts.20.gate_proj", "model.layers.10.mlp.experts.21.gate_proj", "model.layers.10.mlp.experts.22.gate_proj", "model.layers.10.mlp.experts.23.gate_proj", "model.layers.10.mlp.experts.24.gate_proj", "model.layers.10.mlp.experts.25.gate_proj", "model.layers.10.mlp.experts.26.gate_proj", "model.layers.10.mlp.experts.27.gate_proj", "model.layers.10.mlp.experts.28.gate_proj", "model.layers.10.mlp.experts.29.gate_proj", "model.layers.10.mlp.experts.30.gate_proj", "model.layers.10.mlp.experts.31.gate_proj", "model.layers.10.mlp.experts.32.gate_proj", "model.layers.10.mlp.experts.33.gate_proj", "model.layers.10.mlp.experts.34.gate_proj", "model.layers.10.mlp.experts.35.gate_proj", "model.layers.10.mlp.experts.36.gate_proj", "model.layers.10.mlp.experts.37.gate_proj", "model.layers.10.mlp.experts.38.gate_proj", "model.layers.10.mlp.experts.39.gate_proj", "model.layers.10.mlp.experts.40.gate_proj", "model.layers.10.mlp.experts.41.gate_proj", "model.layers.10.mlp.experts.42.gate_proj", "model.layers.10.mlp.experts.43.gate_proj", "model.layers.10.mlp.experts.44.gate_proj", "model.layers.10.mlp.experts.45.gate_proj", "model.layers.10.mlp.experts.46.gate_proj", "model.layers.10.mlp.experts.47.gate_proj", "model.layers.10.mlp.experts.48.gate_proj", "model.layers.10.mlp.experts.49.gate_proj", "model.layers.10.mlp.experts.50.gate_proj", "model.layers.10.mlp.experts.51.gate_proj", "model.layers.10.mlp.experts.52.gate_proj", "model.layers.10.mlp.experts.53.gate_proj", "model.layers.10.mlp.experts.54.gate_proj", "model.layers.10.mlp.experts.55.gate_proj", "model.layers.10.mlp.experts.56.gate_proj", "model.layers.10.mlp.experts.57.gate_proj", "model.layers.10.mlp.experts.58.gate_proj", "model.layers.10.mlp.experts.59.gate_proj", "model.layers.10.mlp.experts.60.gate_proj", "model.layers.10.mlp.experts.61.gate_proj", "model.layers.10.mlp.experts.62.gate_proj", "model.layers.10.mlp.experts.63.gate_proj", "model.layers.10.mlp.experts.64.gate_proj", "model.layers.10.mlp.experts.65.gate_proj", "model.layers.10.mlp.experts.66.gate_proj", "model.layers.10.mlp.experts.67.gate_proj", "model.layers.10.mlp.experts.68.gate_proj", "model.layers.10.mlp.experts.69.gate_proj", "model.layers.10.mlp.experts.70.gate_proj", "model.layers.10.mlp.experts.71.gate_proj", "model.layers.10.mlp.experts.72.gate_proj", "model.layers.10.mlp.experts.73.gate_proj", "model.layers.10.mlp.experts.74.gate_proj", "model.layers.10.mlp.experts.75.gate_proj", "model.layers.10.mlp.experts.76.gate_proj", "model.layers.10.mlp.experts.77.gate_proj", "model.layers.10.mlp.experts.78.gate_proj", "model.layers.10.mlp.experts.79.gate_proj", "model.layers.10.mlp.experts.80.gate_proj", "model.layers.10.mlp.experts.81.gate_proj", "model.layers.10.mlp.experts.82.gate_proj", "model.layers.10.mlp.experts.83.gate_proj", "model.layers.10.mlp.experts.84.gate_proj", "model.layers.10.mlp.experts.85.gate_proj", "model.layers.10.mlp.experts.86.gate_proj", "model.layers.10.mlp.experts.87.gate_proj", "model.layers.10.mlp.experts.88.gate_proj", "model.layers.10.mlp.experts.89.gate_proj", "model.layers.10.mlp.experts.90.gate_proj", "model.layers.10.mlp.experts.91.gate_proj", "model.layers.10.mlp.experts.92.gate_proj", "model.layers.10.mlp.experts.93.gate_proj", "model.layers.10.mlp.experts.94.gate_proj", "model.layers.10.mlp.experts.95.gate_proj", "model.layers.10.mlp.experts.96.gate_proj", "model.layers.10.mlp.experts.97.gate_proj", "model.layers.10.mlp.experts.98.gate_proj", "model.layers.10.mlp.experts.99.gate_proj", "model.layers.10.mlp.experts.100.gate_proj", "model.layers.10.mlp.experts.101.gate_proj", "model.layers.10.mlp.experts.102.gate_proj", "model.layers.10.mlp.experts.103.gate_proj", "model.layers.10.mlp.experts.104.gate_proj", "model.layers.10.mlp.experts.105.gate_proj", "model.layers.10.mlp.experts.106.gate_proj", "model.layers.10.mlp.experts.107.gate_proj", "model.layers.10.mlp.experts.108.gate_proj", "model.layers.10.mlp.experts.109.gate_proj", "model.layers.10.mlp.experts.110.gate_proj", "model.layers.10.mlp.experts.111.gate_proj", "model.layers.10.mlp.experts.112.gate_proj", "model.layers.10.mlp.experts.113.gate_proj", "model.layers.10.mlp.experts.114.gate_proj", "model.layers.10.mlp.experts.115.gate_proj", "model.layers.10.mlp.experts.116.gate_proj", "model.layers.10.mlp.experts.117.gate_proj", "model.layers.10.mlp.experts.118.gate_proj", "model.layers.10.mlp.experts.119.gate_proj", "model.layers.10.mlp.experts.120.gate_proj", "model.layers.10.mlp.experts.121.gate_proj", "model.layers.10.mlp.experts.122.gate_proj", "model.layers.10.mlp.experts.123.gate_proj", "model.layers.10.mlp.experts.124.gate_proj", "model.layers.10.mlp.experts.125.gate_proj", "model.layers.10.mlp.experts.126.gate_proj", "model.layers.10.mlp.experts.127.gate_proj", "model.layers.10.mlp.experts.128.gate_proj", "model.layers.10.mlp.experts.129.gate_proj", "model.layers.10.mlp.experts.130.gate_proj", "model.layers.10.mlp.experts.131.gate_proj", "model.layers.10.mlp.experts.132.gate_proj", "model.layers.10.mlp.experts.133.gate_proj", "model.layers.10.mlp.experts.134.gate_proj", "model.layers.10.mlp.experts.135.gate_proj", "model.layers.10.mlp.experts.136.gate_proj", "model.layers.10.mlp.experts.137.gate_proj", "model.layers.10.mlp.experts.138.gate_proj", "model.layers.10.mlp.experts.139.gate_proj", "model.layers.10.mlp.experts.140.gate_proj", "model.layers.10.mlp.experts.141.gate_proj", "model.layers.10.mlp.experts.142.gate_proj", "model.layers.10.mlp.experts.143.gate_proj", "model.layers.10.mlp.experts.144.gate_proj", "model.layers.10.mlp.experts.145.gate_proj", "model.layers.10.mlp.experts.146.gate_proj", "model.layers.10.mlp.experts.147.gate_proj", "model.layers.10.mlp.experts.148.gate_proj", "model.layers.10.mlp.experts.149.gate_proj", "model.layers.10.mlp.experts.150.gate_proj", "model.layers.10.mlp.experts.151.gate_proj", "model.layers.10.mlp.experts.152.gate_proj", "model.layers.10.mlp.experts.153.gate_proj", "model.layers.10.mlp.experts.154.gate_proj", "model.layers.10.mlp.experts.155.gate_proj", "model.layers.10.mlp.experts.156.gate_proj", "model.layers.10.mlp.experts.157.gate_proj", "model.layers.10.mlp.experts.158.gate_proj", "model.layers.10.mlp.experts.159.gate_proj", "model.layers.10.mlp.experts.0.up_proj", "model.layers.10.mlp.experts.1.up_proj", "model.layers.10.mlp.experts.2.up_proj", "model.layers.10.mlp.experts.3.up_proj", "model.layers.10.mlp.experts.4.up_proj", "model.layers.10.mlp.experts.5.up_proj", "model.layers.10.mlp.experts.6.up_proj", "model.layers.10.mlp.experts.7.up_proj", "model.layers.10.mlp.experts.8.up_proj", "model.layers.10.mlp.experts.9.up_proj", "model.layers.10.mlp.experts.10.up_proj", "model.layers.10.mlp.experts.11.up_proj", "model.layers.10.mlp.experts.12.up_proj", "model.layers.10.mlp.experts.13.up_proj", "model.layers.10.mlp.experts.14.up_proj", "model.layers.10.mlp.experts.15.up_proj", "model.layers.10.mlp.experts.16.up_proj", "model.layers.10.mlp.experts.17.up_proj", "model.layers.10.mlp.experts.18.up_proj", "model.layers.10.mlp.experts.19.up_proj", "model.layers.10.mlp.experts.20.up_proj", "model.layers.10.mlp.experts.21.up_proj", "model.layers.10.mlp.experts.22.up_proj", "model.layers.10.mlp.experts.23.up_proj", "model.layers.10.mlp.experts.24.up_proj", "model.layers.10.mlp.experts.25.up_proj", "model.layers.10.mlp.experts.26.up_proj", "model.layers.10.mlp.experts.27.up_proj", "model.layers.10.mlp.experts.28.up_proj", "model.layers.10.mlp.experts.29.up_proj", "model.layers.10.mlp.experts.30.up_proj", "model.layers.10.mlp.experts.31.up_proj", "model.layers.10.mlp.experts.32.up_proj", "model.layers.10.mlp.experts.33.up_proj", "model.layers.10.mlp.experts.34.up_proj", "model.layers.10.mlp.experts.35.up_proj", "model.layers.10.mlp.experts.36.up_proj", "model.layers.10.mlp.experts.37.up_proj", "model.layers.10.mlp.experts.38.up_proj", "model.layers.10.mlp.experts.39.up_proj", "model.layers.10.mlp.experts.40.up_proj", "model.layers.10.mlp.experts.41.up_proj", "model.layers.10.mlp.experts.42.up_proj", "model.layers.10.mlp.experts.43.up_proj", "model.layers.10.mlp.experts.44.up_proj", "model.layers.10.mlp.experts.45.up_proj", "model.layers.10.mlp.experts.46.up_proj", "model.layers.10.mlp.experts.47.up_proj", "model.layers.10.mlp.experts.48.up_proj", "model.layers.10.mlp.experts.49.up_proj", "model.layers.10.mlp.experts.50.up_proj", "model.layers.10.mlp.experts.51.up_proj", "model.layers.10.mlp.experts.52.up_proj", "model.layers.10.mlp.experts.53.up_proj", "model.layers.10.mlp.experts.54.up_proj", "model.layers.10.mlp.experts.55.up_proj", "model.layers.10.mlp.experts.56.up_proj", "model.layers.10.mlp.experts.57.up_proj", "model.layers.10.mlp.experts.58.up_proj", "model.layers.10.mlp.experts.59.up_proj", "model.layers.10.mlp.experts.60.up_proj", "model.layers.10.mlp.experts.61.up_proj", "model.layers.10.mlp.experts.62.up_proj", "model.layers.10.mlp.experts.63.up_proj", "model.layers.10.mlp.experts.64.up_proj", "model.layers.10.mlp.experts.65.up_proj", "model.layers.10.mlp.experts.66.up_proj", "model.layers.10.mlp.experts.67.up_proj", "model.layers.10.mlp.experts.68.up_proj", "model.layers.10.mlp.experts.69.up_proj", "model.layers.10.mlp.experts.70.up_proj", "model.layers.10.mlp.experts.71.up_proj", "model.layers.10.mlp.experts.72.up_proj", "model.layers.10.mlp.experts.73.up_proj", "model.layers.10.mlp.experts.74.up_proj", "model.layers.10.mlp.experts.75.up_proj", "model.layers.10.mlp.experts.76.up_proj", "model.layers.10.mlp.experts.77.up_proj", "model.layers.10.mlp.experts.78.up_proj", "model.layers.10.mlp.experts.79.up_proj", "model.layers.10.mlp.experts.80.up_proj", "model.layers.10.mlp.experts.81.up_proj", "model.layers.10.mlp.experts.82.up_proj", "model.layers.10.mlp.experts.83.up_proj", "model.layers.10.mlp.experts.84.up_proj", "model.layers.10.mlp.experts.85.up_proj", "model.layers.10.mlp.experts.86.up_proj", "model.layers.10.mlp.experts.87.up_proj", "model.layers.10.mlp.experts.88.up_proj", "model.layers.10.mlp.experts.89.up_proj", "model.layers.10.mlp.experts.90.up_proj", "model.layers.10.mlp.experts.91.up_proj", "model.layers.10.mlp.experts.92.up_proj", "model.layers.10.mlp.experts.93.up_proj", "model.layers.10.mlp.experts.94.up_proj", "model.layers.10.mlp.experts.95.up_proj", "model.layers.10.mlp.experts.96.up_proj", "model.layers.10.mlp.experts.97.up_proj", "model.layers.10.mlp.experts.98.up_proj", "model.layers.10.mlp.experts.99.up_proj", "model.layers.10.mlp.experts.100.up_proj", "model.layers.10.mlp.experts.101.up_proj", "model.layers.10.mlp.experts.102.up_proj", "model.layers.10.mlp.experts.103.up_proj", "model.layers.10.mlp.experts.104.up_proj", "model.layers.10.mlp.experts.105.up_proj", "model.layers.10.mlp.experts.106.up_proj", "model.layers.10.mlp.experts.107.up_proj", "model.layers.10.mlp.experts.108.up_proj", "model.layers.10.mlp.experts.109.up_proj", "model.layers.10.mlp.experts.110.up_proj", "model.layers.10.mlp.experts.111.up_proj", "model.layers.10.mlp.experts.112.up_proj", "model.layers.10.mlp.experts.113.up_proj", "model.layers.10.mlp.experts.114.up_proj", "model.layers.10.mlp.experts.115.up_proj", "model.layers.10.mlp.experts.116.up_proj", "model.layers.10.mlp.experts.117.up_proj", "model.layers.10.mlp.experts.118.up_proj", "model.layers.10.mlp.experts.119.up_proj", "model.layers.10.mlp.experts.120.up_proj", "model.layers.10.mlp.experts.121.up_proj", "model.layers.10.mlp.experts.122.up_proj", "model.layers.10.mlp.experts.123.up_proj", "model.layers.10.mlp.experts.124.up_proj", "model.layers.10.mlp.experts.125.up_proj", "model.layers.10.mlp.experts.126.up_proj", "model.layers.10.mlp.experts.127.up_proj", "model.layers.10.mlp.experts.128.up_proj", "model.layers.10.mlp.experts.129.up_proj", "model.layers.10.mlp.experts.130.up_proj", "model.layers.10.mlp.experts.131.up_proj", "model.layers.10.mlp.experts.132.up_proj", "model.layers.10.mlp.experts.133.up_proj", "model.layers.10.mlp.experts.134.up_proj", "model.layers.10.mlp.experts.135.up_proj", "model.layers.10.mlp.experts.136.up_proj", "model.layers.10.mlp.experts.137.up_proj", "model.layers.10.mlp.experts.138.up_proj", "model.layers.10.mlp.experts.139.up_proj", "model.layers.10.mlp.experts.140.up_proj", "model.layers.10.mlp.experts.141.up_proj", "model.layers.10.mlp.experts.142.up_proj", "model.layers.10.mlp.experts.143.up_proj", "model.layers.10.mlp.experts.144.up_proj", "model.layers.10.mlp.experts.145.up_proj", "model.layers.10.mlp.experts.146.up_proj", "model.layers.10.mlp.experts.147.up_proj", "model.layers.10.mlp.experts.148.up_proj", "model.layers.10.mlp.experts.149.up_proj", "model.layers.10.mlp.experts.150.up_proj", "model.layers.10.mlp.experts.151.up_proj", "model.layers.10.mlp.experts.152.up_proj", "model.layers.10.mlp.experts.153.up_proj", "model.layers.10.mlp.experts.154.up_proj", "model.layers.10.mlp.experts.155.up_proj", "model.layers.10.mlp.experts.156.up_proj", "model.layers.10.mlp.experts.157.up_proj", "model.layers.10.mlp.experts.158.up_proj", "model.layers.10.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.001174198091030143, "dbits": 2516582400 } ] }, { "idx": 62, "layers": [ "model.layers.10.mlp.experts.0.down_proj", "model.layers.10.mlp.experts.1.down_proj", "model.layers.10.mlp.experts.2.down_proj", "model.layers.10.mlp.experts.3.down_proj", "model.layers.10.mlp.experts.4.down_proj", "model.layers.10.mlp.experts.5.down_proj", "model.layers.10.mlp.experts.6.down_proj", "model.layers.10.mlp.experts.7.down_proj", "model.layers.10.mlp.experts.8.down_proj", "model.layers.10.mlp.experts.9.down_proj", "model.layers.10.mlp.experts.10.down_proj", "model.layers.10.mlp.experts.11.down_proj", "model.layers.10.mlp.experts.12.down_proj", "model.layers.10.mlp.experts.13.down_proj", "model.layers.10.mlp.experts.14.down_proj", "model.layers.10.mlp.experts.15.down_proj", "model.layers.10.mlp.experts.16.down_proj", "model.layers.10.mlp.experts.17.down_proj", "model.layers.10.mlp.experts.18.down_proj", "model.layers.10.mlp.experts.19.down_proj", "model.layers.10.mlp.experts.20.down_proj", "model.layers.10.mlp.experts.21.down_proj", "model.layers.10.mlp.experts.22.down_proj", "model.layers.10.mlp.experts.23.down_proj", "model.layers.10.mlp.experts.24.down_proj", "model.layers.10.mlp.experts.25.down_proj", "model.layers.10.mlp.experts.26.down_proj", "model.layers.10.mlp.experts.27.down_proj", "model.layers.10.mlp.experts.28.down_proj", "model.layers.10.mlp.experts.29.down_proj", "model.layers.10.mlp.experts.30.down_proj", "model.layers.10.mlp.experts.31.down_proj", "model.layers.10.mlp.experts.32.down_proj", "model.layers.10.mlp.experts.33.down_proj", "model.layers.10.mlp.experts.34.down_proj", "model.layers.10.mlp.experts.35.down_proj", "model.layers.10.mlp.experts.36.down_proj", "model.layers.10.mlp.experts.37.down_proj", "model.layers.10.mlp.experts.38.down_proj", "model.layers.10.mlp.experts.39.down_proj", "model.layers.10.mlp.experts.40.down_proj", "model.layers.10.mlp.experts.41.down_proj", "model.layers.10.mlp.experts.42.down_proj", "model.layers.10.mlp.experts.43.down_proj", "model.layers.10.mlp.experts.44.down_proj", "model.layers.10.mlp.experts.45.down_proj", "model.layers.10.mlp.experts.46.down_proj", "model.layers.10.mlp.experts.47.down_proj", "model.layers.10.mlp.experts.48.down_proj", "model.layers.10.mlp.experts.49.down_proj", "model.layers.10.mlp.experts.50.down_proj", "model.layers.10.mlp.experts.51.down_proj", "model.layers.10.mlp.experts.52.down_proj", "model.layers.10.mlp.experts.53.down_proj", "model.layers.10.mlp.experts.54.down_proj", "model.layers.10.mlp.experts.55.down_proj", "model.layers.10.mlp.experts.56.down_proj", "model.layers.10.mlp.experts.57.down_proj", "model.layers.10.mlp.experts.58.down_proj", "model.layers.10.mlp.experts.59.down_proj", "model.layers.10.mlp.experts.60.down_proj", "model.layers.10.mlp.experts.61.down_proj", "model.layers.10.mlp.experts.62.down_proj", "model.layers.10.mlp.experts.63.down_proj", "model.layers.10.mlp.experts.64.down_proj", "model.layers.10.mlp.experts.65.down_proj", "model.layers.10.mlp.experts.66.down_proj", "model.layers.10.mlp.experts.67.down_proj", "model.layers.10.mlp.experts.68.down_proj", "model.layers.10.mlp.experts.69.down_proj", "model.layers.10.mlp.experts.70.down_proj", "model.layers.10.mlp.experts.71.down_proj", "model.layers.10.mlp.experts.72.down_proj", "model.layers.10.mlp.experts.73.down_proj", "model.layers.10.mlp.experts.74.down_proj", "model.layers.10.mlp.experts.75.down_proj", "model.layers.10.mlp.experts.76.down_proj", "model.layers.10.mlp.experts.77.down_proj", "model.layers.10.mlp.experts.78.down_proj", "model.layers.10.mlp.experts.79.down_proj", "model.layers.10.mlp.experts.80.down_proj", "model.layers.10.mlp.experts.81.down_proj", "model.layers.10.mlp.experts.82.down_proj", "model.layers.10.mlp.experts.83.down_proj", "model.layers.10.mlp.experts.84.down_proj", "model.layers.10.mlp.experts.85.down_proj", "model.layers.10.mlp.experts.86.down_proj", "model.layers.10.mlp.experts.87.down_proj", "model.layers.10.mlp.experts.88.down_proj", "model.layers.10.mlp.experts.89.down_proj", "model.layers.10.mlp.experts.90.down_proj", "model.layers.10.mlp.experts.91.down_proj", "model.layers.10.mlp.experts.92.down_proj", "model.layers.10.mlp.experts.93.down_proj", "model.layers.10.mlp.experts.94.down_proj", "model.layers.10.mlp.experts.95.down_proj", "model.layers.10.mlp.experts.96.down_proj", "model.layers.10.mlp.experts.97.down_proj", "model.layers.10.mlp.experts.98.down_proj", "model.layers.10.mlp.experts.99.down_proj", "model.layers.10.mlp.experts.100.down_proj", "model.layers.10.mlp.experts.101.down_proj", "model.layers.10.mlp.experts.102.down_proj", "model.layers.10.mlp.experts.103.down_proj", "model.layers.10.mlp.experts.104.down_proj", "model.layers.10.mlp.experts.105.down_proj", "model.layers.10.mlp.experts.106.down_proj", "model.layers.10.mlp.experts.107.down_proj", "model.layers.10.mlp.experts.108.down_proj", "model.layers.10.mlp.experts.109.down_proj", "model.layers.10.mlp.experts.110.down_proj", "model.layers.10.mlp.experts.111.down_proj", "model.layers.10.mlp.experts.112.down_proj", "model.layers.10.mlp.experts.113.down_proj", "model.layers.10.mlp.experts.114.down_proj", "model.layers.10.mlp.experts.115.down_proj", "model.layers.10.mlp.experts.116.down_proj", "model.layers.10.mlp.experts.117.down_proj", "model.layers.10.mlp.experts.118.down_proj", "model.layers.10.mlp.experts.119.down_proj", "model.layers.10.mlp.experts.120.down_proj", "model.layers.10.mlp.experts.121.down_proj", "model.layers.10.mlp.experts.122.down_proj", "model.layers.10.mlp.experts.123.down_proj", "model.layers.10.mlp.experts.124.down_proj", "model.layers.10.mlp.experts.125.down_proj", "model.layers.10.mlp.experts.126.down_proj", "model.layers.10.mlp.experts.127.down_proj", "model.layers.10.mlp.experts.128.down_proj", "model.layers.10.mlp.experts.129.down_proj", "model.layers.10.mlp.experts.130.down_proj", "model.layers.10.mlp.experts.131.down_proj", "model.layers.10.mlp.experts.132.down_proj", "model.layers.10.mlp.experts.133.down_proj", "model.layers.10.mlp.experts.134.down_proj", "model.layers.10.mlp.experts.135.down_proj", "model.layers.10.mlp.experts.136.down_proj", "model.layers.10.mlp.experts.137.down_proj", "model.layers.10.mlp.experts.138.down_proj", "model.layers.10.mlp.experts.139.down_proj", "model.layers.10.mlp.experts.140.down_proj", "model.layers.10.mlp.experts.141.down_proj", "model.layers.10.mlp.experts.142.down_proj", "model.layers.10.mlp.experts.143.down_proj", "model.layers.10.mlp.experts.144.down_proj", "model.layers.10.mlp.experts.145.down_proj", "model.layers.10.mlp.experts.146.down_proj", "model.layers.10.mlp.experts.147.down_proj", "model.layers.10.mlp.experts.148.down_proj", "model.layers.10.mlp.experts.149.down_proj", "model.layers.10.mlp.experts.150.down_proj", "model.layers.10.mlp.experts.151.down_proj", "model.layers.10.mlp.experts.152.down_proj", "model.layers.10.mlp.experts.153.down_proj", "model.layers.10.mlp.experts.154.down_proj", "model.layers.10.mlp.experts.155.down_proj", "model.layers.10.mlp.experts.156.down_proj", "model.layers.10.mlp.experts.157.down_proj", "model.layers.10.mlp.experts.158.down_proj", "model.layers.10.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0019135236740112638, "dbits": 1258291200 } ] }, { "idx": 63, "layers": [ "model.layers.11.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0003782026469707489, "dbits": 62914560 } ] }, { "idx": 64, "layers": [ "model.layers.11.self_attn.k_proj", "model.layers.11.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0018980838358402252, "dbits": 10485760 } ] }, { "idx": 65, "layers": [ "model.layers.11.self_attn.o_proj" ], "candidates": [ { "dkld": -0.004504784941673279, "dbits": 62914560 } ] }, { "idx": 66, "layers": [ "model.layers.11.mlp.shared_experts.gate_proj", "model.layers.11.mlp.shared_experts.up_proj", "model.layers.11.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.003620959073305119, "dbits": 23592960 } ] }, { "idx": 67, "layers": [ "model.layers.11.mlp.experts.0.gate_proj", "model.layers.11.mlp.experts.1.gate_proj", "model.layers.11.mlp.experts.2.gate_proj", "model.layers.11.mlp.experts.3.gate_proj", "model.layers.11.mlp.experts.4.gate_proj", "model.layers.11.mlp.experts.5.gate_proj", "model.layers.11.mlp.experts.6.gate_proj", "model.layers.11.mlp.experts.7.gate_proj", "model.layers.11.mlp.experts.8.gate_proj", "model.layers.11.mlp.experts.9.gate_proj", "model.layers.11.mlp.experts.10.gate_proj", "model.layers.11.mlp.experts.11.gate_proj", "model.layers.11.mlp.experts.12.gate_proj", "model.layers.11.mlp.experts.13.gate_proj", "model.layers.11.mlp.experts.14.gate_proj", "model.layers.11.mlp.experts.15.gate_proj", "model.layers.11.mlp.experts.16.gate_proj", "model.layers.11.mlp.experts.17.gate_proj", "model.layers.11.mlp.experts.18.gate_proj", "model.layers.11.mlp.experts.19.gate_proj", "model.layers.11.mlp.experts.20.gate_proj", "model.layers.11.mlp.experts.21.gate_proj", "model.layers.11.mlp.experts.22.gate_proj", "model.layers.11.mlp.experts.23.gate_proj", "model.layers.11.mlp.experts.24.gate_proj", "model.layers.11.mlp.experts.25.gate_proj", "model.layers.11.mlp.experts.26.gate_proj", "model.layers.11.mlp.experts.27.gate_proj", "model.layers.11.mlp.experts.28.gate_proj", "model.layers.11.mlp.experts.29.gate_proj", "model.layers.11.mlp.experts.30.gate_proj", "model.layers.11.mlp.experts.31.gate_proj", "model.layers.11.mlp.experts.32.gate_proj", "model.layers.11.mlp.experts.33.gate_proj", "model.layers.11.mlp.experts.34.gate_proj", "model.layers.11.mlp.experts.35.gate_proj", "model.layers.11.mlp.experts.36.gate_proj", "model.layers.11.mlp.experts.37.gate_proj", "model.layers.11.mlp.experts.38.gate_proj", "model.layers.11.mlp.experts.39.gate_proj", "model.layers.11.mlp.experts.40.gate_proj", "model.layers.11.mlp.experts.41.gate_proj", "model.layers.11.mlp.experts.42.gate_proj", "model.layers.11.mlp.experts.43.gate_proj", "model.layers.11.mlp.experts.44.gate_proj", "model.layers.11.mlp.experts.45.gate_proj", "model.layers.11.mlp.experts.46.gate_proj", "model.layers.11.mlp.experts.47.gate_proj", "model.layers.11.mlp.experts.48.gate_proj", "model.layers.11.mlp.experts.49.gate_proj", "model.layers.11.mlp.experts.50.gate_proj", "model.layers.11.mlp.experts.51.gate_proj", "model.layers.11.mlp.experts.52.gate_proj", "model.layers.11.mlp.experts.53.gate_proj", "model.layers.11.mlp.experts.54.gate_proj", "model.layers.11.mlp.experts.55.gate_proj", "model.layers.11.mlp.experts.56.gate_proj", "model.layers.11.mlp.experts.57.gate_proj", "model.layers.11.mlp.experts.58.gate_proj", "model.layers.11.mlp.experts.59.gate_proj", "model.layers.11.mlp.experts.60.gate_proj", "model.layers.11.mlp.experts.61.gate_proj", "model.layers.11.mlp.experts.62.gate_proj", "model.layers.11.mlp.experts.63.gate_proj", "model.layers.11.mlp.experts.64.gate_proj", "model.layers.11.mlp.experts.65.gate_proj", "model.layers.11.mlp.experts.66.gate_proj", "model.layers.11.mlp.experts.67.gate_proj", "model.layers.11.mlp.experts.68.gate_proj", "model.layers.11.mlp.experts.69.gate_proj", "model.layers.11.mlp.experts.70.gate_proj", "model.layers.11.mlp.experts.71.gate_proj", "model.layers.11.mlp.experts.72.gate_proj", "model.layers.11.mlp.experts.73.gate_proj", "model.layers.11.mlp.experts.74.gate_proj", "model.layers.11.mlp.experts.75.gate_proj", "model.layers.11.mlp.experts.76.gate_proj", "model.layers.11.mlp.experts.77.gate_proj", "model.layers.11.mlp.experts.78.gate_proj", "model.layers.11.mlp.experts.79.gate_proj", "model.layers.11.mlp.experts.80.gate_proj", "model.layers.11.mlp.experts.81.gate_proj", "model.layers.11.mlp.experts.82.gate_proj", "model.layers.11.mlp.experts.83.gate_proj", "model.layers.11.mlp.experts.84.gate_proj", "model.layers.11.mlp.experts.85.gate_proj", "model.layers.11.mlp.experts.86.gate_proj", "model.layers.11.mlp.experts.87.gate_proj", "model.layers.11.mlp.experts.88.gate_proj", "model.layers.11.mlp.experts.89.gate_proj", "model.layers.11.mlp.experts.90.gate_proj", "model.layers.11.mlp.experts.91.gate_proj", "model.layers.11.mlp.experts.92.gate_proj", "model.layers.11.mlp.experts.93.gate_proj", "model.layers.11.mlp.experts.94.gate_proj", "model.layers.11.mlp.experts.95.gate_proj", "model.layers.11.mlp.experts.96.gate_proj", "model.layers.11.mlp.experts.97.gate_proj", "model.layers.11.mlp.experts.98.gate_proj", "model.layers.11.mlp.experts.99.gate_proj", "model.layers.11.mlp.experts.100.gate_proj", "model.layers.11.mlp.experts.101.gate_proj", "model.layers.11.mlp.experts.102.gate_proj", "model.layers.11.mlp.experts.103.gate_proj", "model.layers.11.mlp.experts.104.gate_proj", "model.layers.11.mlp.experts.105.gate_proj", "model.layers.11.mlp.experts.106.gate_proj", "model.layers.11.mlp.experts.107.gate_proj", "model.layers.11.mlp.experts.108.gate_proj", "model.layers.11.mlp.experts.109.gate_proj", "model.layers.11.mlp.experts.110.gate_proj", "model.layers.11.mlp.experts.111.gate_proj", "model.layers.11.mlp.experts.112.gate_proj", "model.layers.11.mlp.experts.113.gate_proj", "model.layers.11.mlp.experts.114.gate_proj", "model.layers.11.mlp.experts.115.gate_proj", "model.layers.11.mlp.experts.116.gate_proj", "model.layers.11.mlp.experts.117.gate_proj", "model.layers.11.mlp.experts.118.gate_proj", "model.layers.11.mlp.experts.119.gate_proj", "model.layers.11.mlp.experts.120.gate_proj", "model.layers.11.mlp.experts.121.gate_proj", "model.layers.11.mlp.experts.122.gate_proj", "model.layers.11.mlp.experts.123.gate_proj", "model.layers.11.mlp.experts.124.gate_proj", "model.layers.11.mlp.experts.125.gate_proj", "model.layers.11.mlp.experts.126.gate_proj", "model.layers.11.mlp.experts.127.gate_proj", "model.layers.11.mlp.experts.128.gate_proj", "model.layers.11.mlp.experts.129.gate_proj", "model.layers.11.mlp.experts.130.gate_proj", "model.layers.11.mlp.experts.131.gate_proj", "model.layers.11.mlp.experts.132.gate_proj", "model.layers.11.mlp.experts.133.gate_proj", "model.layers.11.mlp.experts.134.gate_proj", "model.layers.11.mlp.experts.135.gate_proj", "model.layers.11.mlp.experts.136.gate_proj", "model.layers.11.mlp.experts.137.gate_proj", "model.layers.11.mlp.experts.138.gate_proj", "model.layers.11.mlp.experts.139.gate_proj", "model.layers.11.mlp.experts.140.gate_proj", "model.layers.11.mlp.experts.141.gate_proj", "model.layers.11.mlp.experts.142.gate_proj", "model.layers.11.mlp.experts.143.gate_proj", "model.layers.11.mlp.experts.144.gate_proj", "model.layers.11.mlp.experts.145.gate_proj", "model.layers.11.mlp.experts.146.gate_proj", "model.layers.11.mlp.experts.147.gate_proj", "model.layers.11.mlp.experts.148.gate_proj", "model.layers.11.mlp.experts.149.gate_proj", "model.layers.11.mlp.experts.150.gate_proj", "model.layers.11.mlp.experts.151.gate_proj", "model.layers.11.mlp.experts.152.gate_proj", "model.layers.11.mlp.experts.153.gate_proj", "model.layers.11.mlp.experts.154.gate_proj", "model.layers.11.mlp.experts.155.gate_proj", "model.layers.11.mlp.experts.156.gate_proj", "model.layers.11.mlp.experts.157.gate_proj", "model.layers.11.mlp.experts.158.gate_proj", "model.layers.11.mlp.experts.159.gate_proj", "model.layers.11.mlp.experts.0.up_proj", "model.layers.11.mlp.experts.1.up_proj", "model.layers.11.mlp.experts.2.up_proj", "model.layers.11.mlp.experts.3.up_proj", "model.layers.11.mlp.experts.4.up_proj", "model.layers.11.mlp.experts.5.up_proj", "model.layers.11.mlp.experts.6.up_proj", "model.layers.11.mlp.experts.7.up_proj", "model.layers.11.mlp.experts.8.up_proj", "model.layers.11.mlp.experts.9.up_proj", "model.layers.11.mlp.experts.10.up_proj", "model.layers.11.mlp.experts.11.up_proj", "model.layers.11.mlp.experts.12.up_proj", "model.layers.11.mlp.experts.13.up_proj", "model.layers.11.mlp.experts.14.up_proj", "model.layers.11.mlp.experts.15.up_proj", "model.layers.11.mlp.experts.16.up_proj", "model.layers.11.mlp.experts.17.up_proj", "model.layers.11.mlp.experts.18.up_proj", "model.layers.11.mlp.experts.19.up_proj", "model.layers.11.mlp.experts.20.up_proj", "model.layers.11.mlp.experts.21.up_proj", "model.layers.11.mlp.experts.22.up_proj", "model.layers.11.mlp.experts.23.up_proj", "model.layers.11.mlp.experts.24.up_proj", "model.layers.11.mlp.experts.25.up_proj", "model.layers.11.mlp.experts.26.up_proj", "model.layers.11.mlp.experts.27.up_proj", "model.layers.11.mlp.experts.28.up_proj", "model.layers.11.mlp.experts.29.up_proj", "model.layers.11.mlp.experts.30.up_proj", "model.layers.11.mlp.experts.31.up_proj", "model.layers.11.mlp.experts.32.up_proj", "model.layers.11.mlp.experts.33.up_proj", "model.layers.11.mlp.experts.34.up_proj", "model.layers.11.mlp.experts.35.up_proj", "model.layers.11.mlp.experts.36.up_proj", "model.layers.11.mlp.experts.37.up_proj", "model.layers.11.mlp.experts.38.up_proj", "model.layers.11.mlp.experts.39.up_proj", "model.layers.11.mlp.experts.40.up_proj", "model.layers.11.mlp.experts.41.up_proj", "model.layers.11.mlp.experts.42.up_proj", "model.layers.11.mlp.experts.43.up_proj", "model.layers.11.mlp.experts.44.up_proj", "model.layers.11.mlp.experts.45.up_proj", "model.layers.11.mlp.experts.46.up_proj", "model.layers.11.mlp.experts.47.up_proj", "model.layers.11.mlp.experts.48.up_proj", "model.layers.11.mlp.experts.49.up_proj", "model.layers.11.mlp.experts.50.up_proj", "model.layers.11.mlp.experts.51.up_proj", "model.layers.11.mlp.experts.52.up_proj", "model.layers.11.mlp.experts.53.up_proj", "model.layers.11.mlp.experts.54.up_proj", "model.layers.11.mlp.experts.55.up_proj", "model.layers.11.mlp.experts.56.up_proj", "model.layers.11.mlp.experts.57.up_proj", "model.layers.11.mlp.experts.58.up_proj", "model.layers.11.mlp.experts.59.up_proj", "model.layers.11.mlp.experts.60.up_proj", "model.layers.11.mlp.experts.61.up_proj", "model.layers.11.mlp.experts.62.up_proj", "model.layers.11.mlp.experts.63.up_proj", "model.layers.11.mlp.experts.64.up_proj", "model.layers.11.mlp.experts.65.up_proj", "model.layers.11.mlp.experts.66.up_proj", "model.layers.11.mlp.experts.67.up_proj", "model.layers.11.mlp.experts.68.up_proj", "model.layers.11.mlp.experts.69.up_proj", "model.layers.11.mlp.experts.70.up_proj", "model.layers.11.mlp.experts.71.up_proj", "model.layers.11.mlp.experts.72.up_proj", "model.layers.11.mlp.experts.73.up_proj", "model.layers.11.mlp.experts.74.up_proj", "model.layers.11.mlp.experts.75.up_proj", "model.layers.11.mlp.experts.76.up_proj", "model.layers.11.mlp.experts.77.up_proj", "model.layers.11.mlp.experts.78.up_proj", "model.layers.11.mlp.experts.79.up_proj", "model.layers.11.mlp.experts.80.up_proj", "model.layers.11.mlp.experts.81.up_proj", "model.layers.11.mlp.experts.82.up_proj", "model.layers.11.mlp.experts.83.up_proj", "model.layers.11.mlp.experts.84.up_proj", "model.layers.11.mlp.experts.85.up_proj", "model.layers.11.mlp.experts.86.up_proj", "model.layers.11.mlp.experts.87.up_proj", "model.layers.11.mlp.experts.88.up_proj", "model.layers.11.mlp.experts.89.up_proj", "model.layers.11.mlp.experts.90.up_proj", "model.layers.11.mlp.experts.91.up_proj", "model.layers.11.mlp.experts.92.up_proj", "model.layers.11.mlp.experts.93.up_proj", "model.layers.11.mlp.experts.94.up_proj", "model.layers.11.mlp.experts.95.up_proj", "model.layers.11.mlp.experts.96.up_proj", "model.layers.11.mlp.experts.97.up_proj", "model.layers.11.mlp.experts.98.up_proj", "model.layers.11.mlp.experts.99.up_proj", "model.layers.11.mlp.experts.100.up_proj", "model.layers.11.mlp.experts.101.up_proj", "model.layers.11.mlp.experts.102.up_proj", "model.layers.11.mlp.experts.103.up_proj", "model.layers.11.mlp.experts.104.up_proj", "model.layers.11.mlp.experts.105.up_proj", "model.layers.11.mlp.experts.106.up_proj", "model.layers.11.mlp.experts.107.up_proj", "model.layers.11.mlp.experts.108.up_proj", "model.layers.11.mlp.experts.109.up_proj", "model.layers.11.mlp.experts.110.up_proj", "model.layers.11.mlp.experts.111.up_proj", "model.layers.11.mlp.experts.112.up_proj", "model.layers.11.mlp.experts.113.up_proj", "model.layers.11.mlp.experts.114.up_proj", "model.layers.11.mlp.experts.115.up_proj", "model.layers.11.mlp.experts.116.up_proj", "model.layers.11.mlp.experts.117.up_proj", "model.layers.11.mlp.experts.118.up_proj", "model.layers.11.mlp.experts.119.up_proj", "model.layers.11.mlp.experts.120.up_proj", "model.layers.11.mlp.experts.121.up_proj", "model.layers.11.mlp.experts.122.up_proj", "model.layers.11.mlp.experts.123.up_proj", "model.layers.11.mlp.experts.124.up_proj", "model.layers.11.mlp.experts.125.up_proj", "model.layers.11.mlp.experts.126.up_proj", "model.layers.11.mlp.experts.127.up_proj", "model.layers.11.mlp.experts.128.up_proj", "model.layers.11.mlp.experts.129.up_proj", "model.layers.11.mlp.experts.130.up_proj", "model.layers.11.mlp.experts.131.up_proj", "model.layers.11.mlp.experts.132.up_proj", "model.layers.11.mlp.experts.133.up_proj", "model.layers.11.mlp.experts.134.up_proj", "model.layers.11.mlp.experts.135.up_proj", "model.layers.11.mlp.experts.136.up_proj", "model.layers.11.mlp.experts.137.up_proj", "model.layers.11.mlp.experts.138.up_proj", "model.layers.11.mlp.experts.139.up_proj", "model.layers.11.mlp.experts.140.up_proj", "model.layers.11.mlp.experts.141.up_proj", "model.layers.11.mlp.experts.142.up_proj", "model.layers.11.mlp.experts.143.up_proj", "model.layers.11.mlp.experts.144.up_proj", "model.layers.11.mlp.experts.145.up_proj", "model.layers.11.mlp.experts.146.up_proj", "model.layers.11.mlp.experts.147.up_proj", "model.layers.11.mlp.experts.148.up_proj", "model.layers.11.mlp.experts.149.up_proj", "model.layers.11.mlp.experts.150.up_proj", "model.layers.11.mlp.experts.151.up_proj", "model.layers.11.mlp.experts.152.up_proj", "model.layers.11.mlp.experts.153.up_proj", "model.layers.11.mlp.experts.154.up_proj", "model.layers.11.mlp.experts.155.up_proj", "model.layers.11.mlp.experts.156.up_proj", "model.layers.11.mlp.experts.157.up_proj", "model.layers.11.mlp.experts.158.up_proj", "model.layers.11.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0027022607624531125, "dbits": 2516582400 } ] }, { "idx": 68, "layers": [ "model.layers.11.mlp.experts.0.down_proj", "model.layers.11.mlp.experts.1.down_proj", "model.layers.11.mlp.experts.2.down_proj", "model.layers.11.mlp.experts.3.down_proj", "model.layers.11.mlp.experts.4.down_proj", "model.layers.11.mlp.experts.5.down_proj", "model.layers.11.mlp.experts.6.down_proj", "model.layers.11.mlp.experts.7.down_proj", "model.layers.11.mlp.experts.8.down_proj", "model.layers.11.mlp.experts.9.down_proj", "model.layers.11.mlp.experts.10.down_proj", "model.layers.11.mlp.experts.11.down_proj", "model.layers.11.mlp.experts.12.down_proj", "model.layers.11.mlp.experts.13.down_proj", "model.layers.11.mlp.experts.14.down_proj", "model.layers.11.mlp.experts.15.down_proj", "model.layers.11.mlp.experts.16.down_proj", "model.layers.11.mlp.experts.17.down_proj", "model.layers.11.mlp.experts.18.down_proj", "model.layers.11.mlp.experts.19.down_proj", "model.layers.11.mlp.experts.20.down_proj", "model.layers.11.mlp.experts.21.down_proj", "model.layers.11.mlp.experts.22.down_proj", "model.layers.11.mlp.experts.23.down_proj", "model.layers.11.mlp.experts.24.down_proj", "model.layers.11.mlp.experts.25.down_proj", "model.layers.11.mlp.experts.26.down_proj", "model.layers.11.mlp.experts.27.down_proj", "model.layers.11.mlp.experts.28.down_proj", "model.layers.11.mlp.experts.29.down_proj", "model.layers.11.mlp.experts.30.down_proj", "model.layers.11.mlp.experts.31.down_proj", "model.layers.11.mlp.experts.32.down_proj", "model.layers.11.mlp.experts.33.down_proj", "model.layers.11.mlp.experts.34.down_proj", "model.layers.11.mlp.experts.35.down_proj", "model.layers.11.mlp.experts.36.down_proj", "model.layers.11.mlp.experts.37.down_proj", "model.layers.11.mlp.experts.38.down_proj", "model.layers.11.mlp.experts.39.down_proj", "model.layers.11.mlp.experts.40.down_proj", "model.layers.11.mlp.experts.41.down_proj", "model.layers.11.mlp.experts.42.down_proj", "model.layers.11.mlp.experts.43.down_proj", "model.layers.11.mlp.experts.44.down_proj", "model.layers.11.mlp.experts.45.down_proj", "model.layers.11.mlp.experts.46.down_proj", "model.layers.11.mlp.experts.47.down_proj", "model.layers.11.mlp.experts.48.down_proj", "model.layers.11.mlp.experts.49.down_proj", "model.layers.11.mlp.experts.50.down_proj", "model.layers.11.mlp.experts.51.down_proj", "model.layers.11.mlp.experts.52.down_proj", "model.layers.11.mlp.experts.53.down_proj", "model.layers.11.mlp.experts.54.down_proj", "model.layers.11.mlp.experts.55.down_proj", "model.layers.11.mlp.experts.56.down_proj", "model.layers.11.mlp.experts.57.down_proj", "model.layers.11.mlp.experts.58.down_proj", "model.layers.11.mlp.experts.59.down_proj", "model.layers.11.mlp.experts.60.down_proj", "model.layers.11.mlp.experts.61.down_proj", "model.layers.11.mlp.experts.62.down_proj", "model.layers.11.mlp.experts.63.down_proj", "model.layers.11.mlp.experts.64.down_proj", "model.layers.11.mlp.experts.65.down_proj", "model.layers.11.mlp.experts.66.down_proj", "model.layers.11.mlp.experts.67.down_proj", "model.layers.11.mlp.experts.68.down_proj", "model.layers.11.mlp.experts.69.down_proj", "model.layers.11.mlp.experts.70.down_proj", "model.layers.11.mlp.experts.71.down_proj", "model.layers.11.mlp.experts.72.down_proj", "model.layers.11.mlp.experts.73.down_proj", "model.layers.11.mlp.experts.74.down_proj", "model.layers.11.mlp.experts.75.down_proj", "model.layers.11.mlp.experts.76.down_proj", "model.layers.11.mlp.experts.77.down_proj", "model.layers.11.mlp.experts.78.down_proj", "model.layers.11.mlp.experts.79.down_proj", "model.layers.11.mlp.experts.80.down_proj", "model.layers.11.mlp.experts.81.down_proj", "model.layers.11.mlp.experts.82.down_proj", "model.layers.11.mlp.experts.83.down_proj", "model.layers.11.mlp.experts.84.down_proj", "model.layers.11.mlp.experts.85.down_proj", "model.layers.11.mlp.experts.86.down_proj", "model.layers.11.mlp.experts.87.down_proj", "model.layers.11.mlp.experts.88.down_proj", "model.layers.11.mlp.experts.89.down_proj", "model.layers.11.mlp.experts.90.down_proj", "model.layers.11.mlp.experts.91.down_proj", "model.layers.11.mlp.experts.92.down_proj", "model.layers.11.mlp.experts.93.down_proj", "model.layers.11.mlp.experts.94.down_proj", "model.layers.11.mlp.experts.95.down_proj", "model.layers.11.mlp.experts.96.down_proj", "model.layers.11.mlp.experts.97.down_proj", "model.layers.11.mlp.experts.98.down_proj", "model.layers.11.mlp.experts.99.down_proj", "model.layers.11.mlp.experts.100.down_proj", "model.layers.11.mlp.experts.101.down_proj", "model.layers.11.mlp.experts.102.down_proj", "model.layers.11.mlp.experts.103.down_proj", "model.layers.11.mlp.experts.104.down_proj", "model.layers.11.mlp.experts.105.down_proj", "model.layers.11.mlp.experts.106.down_proj", "model.layers.11.mlp.experts.107.down_proj", "model.layers.11.mlp.experts.108.down_proj", "model.layers.11.mlp.experts.109.down_proj", "model.layers.11.mlp.experts.110.down_proj", "model.layers.11.mlp.experts.111.down_proj", "model.layers.11.mlp.experts.112.down_proj", "model.layers.11.mlp.experts.113.down_proj", "model.layers.11.mlp.experts.114.down_proj", "model.layers.11.mlp.experts.115.down_proj", "model.layers.11.mlp.experts.116.down_proj", "model.layers.11.mlp.experts.117.down_proj", "model.layers.11.mlp.experts.118.down_proj", "model.layers.11.mlp.experts.119.down_proj", "model.layers.11.mlp.experts.120.down_proj", "model.layers.11.mlp.experts.121.down_proj", "model.layers.11.mlp.experts.122.down_proj", "model.layers.11.mlp.experts.123.down_proj", "model.layers.11.mlp.experts.124.down_proj", "model.layers.11.mlp.experts.125.down_proj", "model.layers.11.mlp.experts.126.down_proj", "model.layers.11.mlp.experts.127.down_proj", "model.layers.11.mlp.experts.128.down_proj", "model.layers.11.mlp.experts.129.down_proj", "model.layers.11.mlp.experts.130.down_proj", "model.layers.11.mlp.experts.131.down_proj", "model.layers.11.mlp.experts.132.down_proj", "model.layers.11.mlp.experts.133.down_proj", "model.layers.11.mlp.experts.134.down_proj", "model.layers.11.mlp.experts.135.down_proj", "model.layers.11.mlp.experts.136.down_proj", "model.layers.11.mlp.experts.137.down_proj", "model.layers.11.mlp.experts.138.down_proj", "model.layers.11.mlp.experts.139.down_proj", "model.layers.11.mlp.experts.140.down_proj", "model.layers.11.mlp.experts.141.down_proj", "model.layers.11.mlp.experts.142.down_proj", "model.layers.11.mlp.experts.143.down_proj", "model.layers.11.mlp.experts.144.down_proj", "model.layers.11.mlp.experts.145.down_proj", "model.layers.11.mlp.experts.146.down_proj", "model.layers.11.mlp.experts.147.down_proj", "model.layers.11.mlp.experts.148.down_proj", "model.layers.11.mlp.experts.149.down_proj", "model.layers.11.mlp.experts.150.down_proj", "model.layers.11.mlp.experts.151.down_proj", "model.layers.11.mlp.experts.152.down_proj", "model.layers.11.mlp.experts.153.down_proj", "model.layers.11.mlp.experts.154.down_proj", "model.layers.11.mlp.experts.155.down_proj", "model.layers.11.mlp.experts.156.down_proj", "model.layers.11.mlp.experts.157.down_proj", "model.layers.11.mlp.experts.158.down_proj", "model.layers.11.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0028965532779693937, "dbits": 1258291200 } ] }, { "idx": 69, "layers": [ "model.layers.12.self_attn.q_proj" ], "candidates": [ { "dkld": 0.00017319843173024818, "dbits": 62914560 } ] }, { "idx": 70, "layers": [ "model.layers.12.self_attn.k_proj", "model.layers.12.self_attn.v_proj" ], "candidates": [ { "dkld": -0.006282854825258255, "dbits": 10485760 } ] }, { "idx": 71, "layers": [ "model.layers.12.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0007792942225933186, "dbits": 62914560 } ] }, { "idx": 72, "layers": [ "model.layers.12.mlp.shared_experts.gate_proj", "model.layers.12.mlp.shared_experts.up_proj", "model.layers.12.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0030272379517555237, "dbits": 23592960 } ] }, { "idx": 73, "layers": [ "model.layers.12.mlp.experts.0.gate_proj", "model.layers.12.mlp.experts.1.gate_proj", "model.layers.12.mlp.experts.2.gate_proj", "model.layers.12.mlp.experts.3.gate_proj", "model.layers.12.mlp.experts.4.gate_proj", "model.layers.12.mlp.experts.5.gate_proj", "model.layers.12.mlp.experts.6.gate_proj", "model.layers.12.mlp.experts.7.gate_proj", "model.layers.12.mlp.experts.8.gate_proj", "model.layers.12.mlp.experts.9.gate_proj", "model.layers.12.mlp.experts.10.gate_proj", "model.layers.12.mlp.experts.11.gate_proj", "model.layers.12.mlp.experts.12.gate_proj", "model.layers.12.mlp.experts.13.gate_proj", "model.layers.12.mlp.experts.14.gate_proj", "model.layers.12.mlp.experts.15.gate_proj", "model.layers.12.mlp.experts.16.gate_proj", "model.layers.12.mlp.experts.17.gate_proj", "model.layers.12.mlp.experts.18.gate_proj", "model.layers.12.mlp.experts.19.gate_proj", "model.layers.12.mlp.experts.20.gate_proj", "model.layers.12.mlp.experts.21.gate_proj", "model.layers.12.mlp.experts.22.gate_proj", "model.layers.12.mlp.experts.23.gate_proj", "model.layers.12.mlp.experts.24.gate_proj", "model.layers.12.mlp.experts.25.gate_proj", "model.layers.12.mlp.experts.26.gate_proj", "model.layers.12.mlp.experts.27.gate_proj", "model.layers.12.mlp.experts.28.gate_proj", "model.layers.12.mlp.experts.29.gate_proj", "model.layers.12.mlp.experts.30.gate_proj", "model.layers.12.mlp.experts.31.gate_proj", "model.layers.12.mlp.experts.32.gate_proj", "model.layers.12.mlp.experts.33.gate_proj", "model.layers.12.mlp.experts.34.gate_proj", "model.layers.12.mlp.experts.35.gate_proj", "model.layers.12.mlp.experts.36.gate_proj", "model.layers.12.mlp.experts.37.gate_proj", "model.layers.12.mlp.experts.38.gate_proj", "model.layers.12.mlp.experts.39.gate_proj", "model.layers.12.mlp.experts.40.gate_proj", "model.layers.12.mlp.experts.41.gate_proj", "model.layers.12.mlp.experts.42.gate_proj", "model.layers.12.mlp.experts.43.gate_proj", "model.layers.12.mlp.experts.44.gate_proj", "model.layers.12.mlp.experts.45.gate_proj", "model.layers.12.mlp.experts.46.gate_proj", "model.layers.12.mlp.experts.47.gate_proj", "model.layers.12.mlp.experts.48.gate_proj", "model.layers.12.mlp.experts.49.gate_proj", "model.layers.12.mlp.experts.50.gate_proj", "model.layers.12.mlp.experts.51.gate_proj", "model.layers.12.mlp.experts.52.gate_proj", "model.layers.12.mlp.experts.53.gate_proj", "model.layers.12.mlp.experts.54.gate_proj", "model.layers.12.mlp.experts.55.gate_proj", "model.layers.12.mlp.experts.56.gate_proj", "model.layers.12.mlp.experts.57.gate_proj", "model.layers.12.mlp.experts.58.gate_proj", "model.layers.12.mlp.experts.59.gate_proj", "model.layers.12.mlp.experts.60.gate_proj", "model.layers.12.mlp.experts.61.gate_proj", "model.layers.12.mlp.experts.62.gate_proj", "model.layers.12.mlp.experts.63.gate_proj", "model.layers.12.mlp.experts.64.gate_proj", "model.layers.12.mlp.experts.65.gate_proj", "model.layers.12.mlp.experts.66.gate_proj", "model.layers.12.mlp.experts.67.gate_proj", "model.layers.12.mlp.experts.68.gate_proj", "model.layers.12.mlp.experts.69.gate_proj", "model.layers.12.mlp.experts.70.gate_proj", "model.layers.12.mlp.experts.71.gate_proj", "model.layers.12.mlp.experts.72.gate_proj", "model.layers.12.mlp.experts.73.gate_proj", "model.layers.12.mlp.experts.74.gate_proj", "model.layers.12.mlp.experts.75.gate_proj", "model.layers.12.mlp.experts.76.gate_proj", "model.layers.12.mlp.experts.77.gate_proj", "model.layers.12.mlp.experts.78.gate_proj", "model.layers.12.mlp.experts.79.gate_proj", "model.layers.12.mlp.experts.80.gate_proj", "model.layers.12.mlp.experts.81.gate_proj", "model.layers.12.mlp.experts.82.gate_proj", "model.layers.12.mlp.experts.83.gate_proj", "model.layers.12.mlp.experts.84.gate_proj", "model.layers.12.mlp.experts.85.gate_proj", "model.layers.12.mlp.experts.86.gate_proj", "model.layers.12.mlp.experts.87.gate_proj", "model.layers.12.mlp.experts.88.gate_proj", "model.layers.12.mlp.experts.89.gate_proj", "model.layers.12.mlp.experts.90.gate_proj", "model.layers.12.mlp.experts.91.gate_proj", "model.layers.12.mlp.experts.92.gate_proj", "model.layers.12.mlp.experts.93.gate_proj", "model.layers.12.mlp.experts.94.gate_proj", "model.layers.12.mlp.experts.95.gate_proj", "model.layers.12.mlp.experts.96.gate_proj", "model.layers.12.mlp.experts.97.gate_proj", "model.layers.12.mlp.experts.98.gate_proj", "model.layers.12.mlp.experts.99.gate_proj", "model.layers.12.mlp.experts.100.gate_proj", "model.layers.12.mlp.experts.101.gate_proj", "model.layers.12.mlp.experts.102.gate_proj", "model.layers.12.mlp.experts.103.gate_proj", "model.layers.12.mlp.experts.104.gate_proj", "model.layers.12.mlp.experts.105.gate_proj", "model.layers.12.mlp.experts.106.gate_proj", "model.layers.12.mlp.experts.107.gate_proj", "model.layers.12.mlp.experts.108.gate_proj", "model.layers.12.mlp.experts.109.gate_proj", "model.layers.12.mlp.experts.110.gate_proj", "model.layers.12.mlp.experts.111.gate_proj", "model.layers.12.mlp.experts.112.gate_proj", "model.layers.12.mlp.experts.113.gate_proj", "model.layers.12.mlp.experts.114.gate_proj", "model.layers.12.mlp.experts.115.gate_proj", "model.layers.12.mlp.experts.116.gate_proj", "model.layers.12.mlp.experts.117.gate_proj", "model.layers.12.mlp.experts.118.gate_proj", "model.layers.12.mlp.experts.119.gate_proj", "model.layers.12.mlp.experts.120.gate_proj", "model.layers.12.mlp.experts.121.gate_proj", "model.layers.12.mlp.experts.122.gate_proj", "model.layers.12.mlp.experts.123.gate_proj", "model.layers.12.mlp.experts.124.gate_proj", "model.layers.12.mlp.experts.125.gate_proj", "model.layers.12.mlp.experts.126.gate_proj", "model.layers.12.mlp.experts.127.gate_proj", "model.layers.12.mlp.experts.128.gate_proj", "model.layers.12.mlp.experts.129.gate_proj", "model.layers.12.mlp.experts.130.gate_proj", "model.layers.12.mlp.experts.131.gate_proj", "model.layers.12.mlp.experts.132.gate_proj", "model.layers.12.mlp.experts.133.gate_proj", "model.layers.12.mlp.experts.134.gate_proj", "model.layers.12.mlp.experts.135.gate_proj", "model.layers.12.mlp.experts.136.gate_proj", "model.layers.12.mlp.experts.137.gate_proj", "model.layers.12.mlp.experts.138.gate_proj", "model.layers.12.mlp.experts.139.gate_proj", "model.layers.12.mlp.experts.140.gate_proj", "model.layers.12.mlp.experts.141.gate_proj", "model.layers.12.mlp.experts.142.gate_proj", "model.layers.12.mlp.experts.143.gate_proj", "model.layers.12.mlp.experts.144.gate_proj", "model.layers.12.mlp.experts.145.gate_proj", "model.layers.12.mlp.experts.146.gate_proj", "model.layers.12.mlp.experts.147.gate_proj", "model.layers.12.mlp.experts.148.gate_proj", "model.layers.12.mlp.experts.149.gate_proj", "model.layers.12.mlp.experts.150.gate_proj", "model.layers.12.mlp.experts.151.gate_proj", "model.layers.12.mlp.experts.152.gate_proj", "model.layers.12.mlp.experts.153.gate_proj", "model.layers.12.mlp.experts.154.gate_proj", "model.layers.12.mlp.experts.155.gate_proj", "model.layers.12.mlp.experts.156.gate_proj", "model.layers.12.mlp.experts.157.gate_proj", "model.layers.12.mlp.experts.158.gate_proj", "model.layers.12.mlp.experts.159.gate_proj", "model.layers.12.mlp.experts.0.up_proj", "model.layers.12.mlp.experts.1.up_proj", "model.layers.12.mlp.experts.2.up_proj", "model.layers.12.mlp.experts.3.up_proj", "model.layers.12.mlp.experts.4.up_proj", "model.layers.12.mlp.experts.5.up_proj", "model.layers.12.mlp.experts.6.up_proj", "model.layers.12.mlp.experts.7.up_proj", "model.layers.12.mlp.experts.8.up_proj", "model.layers.12.mlp.experts.9.up_proj", "model.layers.12.mlp.experts.10.up_proj", "model.layers.12.mlp.experts.11.up_proj", "model.layers.12.mlp.experts.12.up_proj", "model.layers.12.mlp.experts.13.up_proj", "model.layers.12.mlp.experts.14.up_proj", "model.layers.12.mlp.experts.15.up_proj", "model.layers.12.mlp.experts.16.up_proj", "model.layers.12.mlp.experts.17.up_proj", "model.layers.12.mlp.experts.18.up_proj", "model.layers.12.mlp.experts.19.up_proj", "model.layers.12.mlp.experts.20.up_proj", "model.layers.12.mlp.experts.21.up_proj", "model.layers.12.mlp.experts.22.up_proj", "model.layers.12.mlp.experts.23.up_proj", "model.layers.12.mlp.experts.24.up_proj", "model.layers.12.mlp.experts.25.up_proj", "model.layers.12.mlp.experts.26.up_proj", "model.layers.12.mlp.experts.27.up_proj", "model.layers.12.mlp.experts.28.up_proj", "model.layers.12.mlp.experts.29.up_proj", "model.layers.12.mlp.experts.30.up_proj", "model.layers.12.mlp.experts.31.up_proj", "model.layers.12.mlp.experts.32.up_proj", "model.layers.12.mlp.experts.33.up_proj", "model.layers.12.mlp.experts.34.up_proj", "model.layers.12.mlp.experts.35.up_proj", "model.layers.12.mlp.experts.36.up_proj", "model.layers.12.mlp.experts.37.up_proj", "model.layers.12.mlp.experts.38.up_proj", "model.layers.12.mlp.experts.39.up_proj", "model.layers.12.mlp.experts.40.up_proj", "model.layers.12.mlp.experts.41.up_proj", "model.layers.12.mlp.experts.42.up_proj", "model.layers.12.mlp.experts.43.up_proj", "model.layers.12.mlp.experts.44.up_proj", "model.layers.12.mlp.experts.45.up_proj", "model.layers.12.mlp.experts.46.up_proj", "model.layers.12.mlp.experts.47.up_proj", "model.layers.12.mlp.experts.48.up_proj", "model.layers.12.mlp.experts.49.up_proj", "model.layers.12.mlp.experts.50.up_proj", "model.layers.12.mlp.experts.51.up_proj", "model.layers.12.mlp.experts.52.up_proj", "model.layers.12.mlp.experts.53.up_proj", "model.layers.12.mlp.experts.54.up_proj", "model.layers.12.mlp.experts.55.up_proj", "model.layers.12.mlp.experts.56.up_proj", "model.layers.12.mlp.experts.57.up_proj", "model.layers.12.mlp.experts.58.up_proj", "model.layers.12.mlp.experts.59.up_proj", "model.layers.12.mlp.experts.60.up_proj", "model.layers.12.mlp.experts.61.up_proj", "model.layers.12.mlp.experts.62.up_proj", "model.layers.12.mlp.experts.63.up_proj", "model.layers.12.mlp.experts.64.up_proj", "model.layers.12.mlp.experts.65.up_proj", "model.layers.12.mlp.experts.66.up_proj", "model.layers.12.mlp.experts.67.up_proj", "model.layers.12.mlp.experts.68.up_proj", "model.layers.12.mlp.experts.69.up_proj", "model.layers.12.mlp.experts.70.up_proj", "model.layers.12.mlp.experts.71.up_proj", "model.layers.12.mlp.experts.72.up_proj", "model.layers.12.mlp.experts.73.up_proj", "model.layers.12.mlp.experts.74.up_proj", "model.layers.12.mlp.experts.75.up_proj", "model.layers.12.mlp.experts.76.up_proj", "model.layers.12.mlp.experts.77.up_proj", "model.layers.12.mlp.experts.78.up_proj", "model.layers.12.mlp.experts.79.up_proj", "model.layers.12.mlp.experts.80.up_proj", "model.layers.12.mlp.experts.81.up_proj", "model.layers.12.mlp.experts.82.up_proj", "model.layers.12.mlp.experts.83.up_proj", "model.layers.12.mlp.experts.84.up_proj", "model.layers.12.mlp.experts.85.up_proj", "model.layers.12.mlp.experts.86.up_proj", "model.layers.12.mlp.experts.87.up_proj", "model.layers.12.mlp.experts.88.up_proj", "model.layers.12.mlp.experts.89.up_proj", "model.layers.12.mlp.experts.90.up_proj", "model.layers.12.mlp.experts.91.up_proj", "model.layers.12.mlp.experts.92.up_proj", "model.layers.12.mlp.experts.93.up_proj", "model.layers.12.mlp.experts.94.up_proj", "model.layers.12.mlp.experts.95.up_proj", "model.layers.12.mlp.experts.96.up_proj", "model.layers.12.mlp.experts.97.up_proj", "model.layers.12.mlp.experts.98.up_proj", "model.layers.12.mlp.experts.99.up_proj", "model.layers.12.mlp.experts.100.up_proj", "model.layers.12.mlp.experts.101.up_proj", "model.layers.12.mlp.experts.102.up_proj", "model.layers.12.mlp.experts.103.up_proj", "model.layers.12.mlp.experts.104.up_proj", "model.layers.12.mlp.experts.105.up_proj", "model.layers.12.mlp.experts.106.up_proj", "model.layers.12.mlp.experts.107.up_proj", "model.layers.12.mlp.experts.108.up_proj", "model.layers.12.mlp.experts.109.up_proj", "model.layers.12.mlp.experts.110.up_proj", "model.layers.12.mlp.experts.111.up_proj", "model.layers.12.mlp.experts.112.up_proj", "model.layers.12.mlp.experts.113.up_proj", "model.layers.12.mlp.experts.114.up_proj", "model.layers.12.mlp.experts.115.up_proj", "model.layers.12.mlp.experts.116.up_proj", "model.layers.12.mlp.experts.117.up_proj", "model.layers.12.mlp.experts.118.up_proj", "model.layers.12.mlp.experts.119.up_proj", "model.layers.12.mlp.experts.120.up_proj", "model.layers.12.mlp.experts.121.up_proj", "model.layers.12.mlp.experts.122.up_proj", "model.layers.12.mlp.experts.123.up_proj", "model.layers.12.mlp.experts.124.up_proj", "model.layers.12.mlp.experts.125.up_proj", "model.layers.12.mlp.experts.126.up_proj", "model.layers.12.mlp.experts.127.up_proj", "model.layers.12.mlp.experts.128.up_proj", "model.layers.12.mlp.experts.129.up_proj", "model.layers.12.mlp.experts.130.up_proj", "model.layers.12.mlp.experts.131.up_proj", "model.layers.12.mlp.experts.132.up_proj", "model.layers.12.mlp.experts.133.up_proj", "model.layers.12.mlp.experts.134.up_proj", "model.layers.12.mlp.experts.135.up_proj", "model.layers.12.mlp.experts.136.up_proj", "model.layers.12.mlp.experts.137.up_proj", "model.layers.12.mlp.experts.138.up_proj", "model.layers.12.mlp.experts.139.up_proj", "model.layers.12.mlp.experts.140.up_proj", "model.layers.12.mlp.experts.141.up_proj", "model.layers.12.mlp.experts.142.up_proj", "model.layers.12.mlp.experts.143.up_proj", "model.layers.12.mlp.experts.144.up_proj", "model.layers.12.mlp.experts.145.up_proj", "model.layers.12.mlp.experts.146.up_proj", "model.layers.12.mlp.experts.147.up_proj", "model.layers.12.mlp.experts.148.up_proj", "model.layers.12.mlp.experts.149.up_proj", "model.layers.12.mlp.experts.150.up_proj", "model.layers.12.mlp.experts.151.up_proj", "model.layers.12.mlp.experts.152.up_proj", "model.layers.12.mlp.experts.153.up_proj", "model.layers.12.mlp.experts.154.up_proj", "model.layers.12.mlp.experts.155.up_proj", "model.layers.12.mlp.experts.156.up_proj", "model.layers.12.mlp.experts.157.up_proj", "model.layers.12.mlp.experts.158.up_proj", "model.layers.12.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.004180979728698742, "dbits": 2516582400 } ] }, { "idx": 74, "layers": [ "model.layers.12.mlp.experts.0.down_proj", "model.layers.12.mlp.experts.1.down_proj", "model.layers.12.mlp.experts.2.down_proj", "model.layers.12.mlp.experts.3.down_proj", "model.layers.12.mlp.experts.4.down_proj", "model.layers.12.mlp.experts.5.down_proj", "model.layers.12.mlp.experts.6.down_proj", "model.layers.12.mlp.experts.7.down_proj", "model.layers.12.mlp.experts.8.down_proj", "model.layers.12.mlp.experts.9.down_proj", "model.layers.12.mlp.experts.10.down_proj", "model.layers.12.mlp.experts.11.down_proj", "model.layers.12.mlp.experts.12.down_proj", "model.layers.12.mlp.experts.13.down_proj", "model.layers.12.mlp.experts.14.down_proj", "model.layers.12.mlp.experts.15.down_proj", "model.layers.12.mlp.experts.16.down_proj", "model.layers.12.mlp.experts.17.down_proj", "model.layers.12.mlp.experts.18.down_proj", "model.layers.12.mlp.experts.19.down_proj", "model.layers.12.mlp.experts.20.down_proj", "model.layers.12.mlp.experts.21.down_proj", "model.layers.12.mlp.experts.22.down_proj", "model.layers.12.mlp.experts.23.down_proj", "model.layers.12.mlp.experts.24.down_proj", "model.layers.12.mlp.experts.25.down_proj", "model.layers.12.mlp.experts.26.down_proj", "model.layers.12.mlp.experts.27.down_proj", "model.layers.12.mlp.experts.28.down_proj", "model.layers.12.mlp.experts.29.down_proj", "model.layers.12.mlp.experts.30.down_proj", "model.layers.12.mlp.experts.31.down_proj", "model.layers.12.mlp.experts.32.down_proj", "model.layers.12.mlp.experts.33.down_proj", "model.layers.12.mlp.experts.34.down_proj", "model.layers.12.mlp.experts.35.down_proj", "model.layers.12.mlp.experts.36.down_proj", "model.layers.12.mlp.experts.37.down_proj", "model.layers.12.mlp.experts.38.down_proj", "model.layers.12.mlp.experts.39.down_proj", "model.layers.12.mlp.experts.40.down_proj", "model.layers.12.mlp.experts.41.down_proj", "model.layers.12.mlp.experts.42.down_proj", "model.layers.12.mlp.experts.43.down_proj", "model.layers.12.mlp.experts.44.down_proj", "model.layers.12.mlp.experts.45.down_proj", "model.layers.12.mlp.experts.46.down_proj", "model.layers.12.mlp.experts.47.down_proj", "model.layers.12.mlp.experts.48.down_proj", "model.layers.12.mlp.experts.49.down_proj", "model.layers.12.mlp.experts.50.down_proj", "model.layers.12.mlp.experts.51.down_proj", "model.layers.12.mlp.experts.52.down_proj", "model.layers.12.mlp.experts.53.down_proj", "model.layers.12.mlp.experts.54.down_proj", "model.layers.12.mlp.experts.55.down_proj", "model.layers.12.mlp.experts.56.down_proj", "model.layers.12.mlp.experts.57.down_proj", "model.layers.12.mlp.experts.58.down_proj", "model.layers.12.mlp.experts.59.down_proj", "model.layers.12.mlp.experts.60.down_proj", "model.layers.12.mlp.experts.61.down_proj", "model.layers.12.mlp.experts.62.down_proj", "model.layers.12.mlp.experts.63.down_proj", "model.layers.12.mlp.experts.64.down_proj", "model.layers.12.mlp.experts.65.down_proj", "model.layers.12.mlp.experts.66.down_proj", "model.layers.12.mlp.experts.67.down_proj", "model.layers.12.mlp.experts.68.down_proj", "model.layers.12.mlp.experts.69.down_proj", "model.layers.12.mlp.experts.70.down_proj", "model.layers.12.mlp.experts.71.down_proj", "model.layers.12.mlp.experts.72.down_proj", "model.layers.12.mlp.experts.73.down_proj", "model.layers.12.mlp.experts.74.down_proj", "model.layers.12.mlp.experts.75.down_proj", "model.layers.12.mlp.experts.76.down_proj", "model.layers.12.mlp.experts.77.down_proj", "model.layers.12.mlp.experts.78.down_proj", "model.layers.12.mlp.experts.79.down_proj", "model.layers.12.mlp.experts.80.down_proj", "model.layers.12.mlp.experts.81.down_proj", "model.layers.12.mlp.experts.82.down_proj", "model.layers.12.mlp.experts.83.down_proj", "model.layers.12.mlp.experts.84.down_proj", "model.layers.12.mlp.experts.85.down_proj", "model.layers.12.mlp.experts.86.down_proj", "model.layers.12.mlp.experts.87.down_proj", "model.layers.12.mlp.experts.88.down_proj", "model.layers.12.mlp.experts.89.down_proj", "model.layers.12.mlp.experts.90.down_proj", "model.layers.12.mlp.experts.91.down_proj", "model.layers.12.mlp.experts.92.down_proj", "model.layers.12.mlp.experts.93.down_proj", "model.layers.12.mlp.experts.94.down_proj", "model.layers.12.mlp.experts.95.down_proj", "model.layers.12.mlp.experts.96.down_proj", "model.layers.12.mlp.experts.97.down_proj", "model.layers.12.mlp.experts.98.down_proj", "model.layers.12.mlp.experts.99.down_proj", "model.layers.12.mlp.experts.100.down_proj", "model.layers.12.mlp.experts.101.down_proj", "model.layers.12.mlp.experts.102.down_proj", "model.layers.12.mlp.experts.103.down_proj", "model.layers.12.mlp.experts.104.down_proj", "model.layers.12.mlp.experts.105.down_proj", "model.layers.12.mlp.experts.106.down_proj", "model.layers.12.mlp.experts.107.down_proj", "model.layers.12.mlp.experts.108.down_proj", "model.layers.12.mlp.experts.109.down_proj", "model.layers.12.mlp.experts.110.down_proj", "model.layers.12.mlp.experts.111.down_proj", "model.layers.12.mlp.experts.112.down_proj", "model.layers.12.mlp.experts.113.down_proj", "model.layers.12.mlp.experts.114.down_proj", "model.layers.12.mlp.experts.115.down_proj", "model.layers.12.mlp.experts.116.down_proj", "model.layers.12.mlp.experts.117.down_proj", "model.layers.12.mlp.experts.118.down_proj", "model.layers.12.mlp.experts.119.down_proj", "model.layers.12.mlp.experts.120.down_proj", "model.layers.12.mlp.experts.121.down_proj", "model.layers.12.mlp.experts.122.down_proj", "model.layers.12.mlp.experts.123.down_proj", "model.layers.12.mlp.experts.124.down_proj", "model.layers.12.mlp.experts.125.down_proj", "model.layers.12.mlp.experts.126.down_proj", "model.layers.12.mlp.experts.127.down_proj", "model.layers.12.mlp.experts.128.down_proj", "model.layers.12.mlp.experts.129.down_proj", "model.layers.12.mlp.experts.130.down_proj", "model.layers.12.mlp.experts.131.down_proj", "model.layers.12.mlp.experts.132.down_proj", "model.layers.12.mlp.experts.133.down_proj", "model.layers.12.mlp.experts.134.down_proj", "model.layers.12.mlp.experts.135.down_proj", "model.layers.12.mlp.experts.136.down_proj", "model.layers.12.mlp.experts.137.down_proj", "model.layers.12.mlp.experts.138.down_proj", "model.layers.12.mlp.experts.139.down_proj", "model.layers.12.mlp.experts.140.down_proj", "model.layers.12.mlp.experts.141.down_proj", "model.layers.12.mlp.experts.142.down_proj", "model.layers.12.mlp.experts.143.down_proj", "model.layers.12.mlp.experts.144.down_proj", "model.layers.12.mlp.experts.145.down_proj", "model.layers.12.mlp.experts.146.down_proj", "model.layers.12.mlp.experts.147.down_proj", "model.layers.12.mlp.experts.148.down_proj", "model.layers.12.mlp.experts.149.down_proj", "model.layers.12.mlp.experts.150.down_proj", "model.layers.12.mlp.experts.151.down_proj", "model.layers.12.mlp.experts.152.down_proj", "model.layers.12.mlp.experts.153.down_proj", "model.layers.12.mlp.experts.154.down_proj", "model.layers.12.mlp.experts.155.down_proj", "model.layers.12.mlp.experts.156.down_proj", "model.layers.12.mlp.experts.157.down_proj", "model.layers.12.mlp.experts.158.down_proj", "model.layers.12.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.003991670161485661, "dbits": 1258291200 } ] }, { "idx": 75, "layers": [ "model.layers.13.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0013567559421062692, "dbits": 62914560 } ] }, { "idx": 76, "layers": [ "model.layers.13.self_attn.k_proj", "model.layers.13.self_attn.v_proj" ], "candidates": [ { "dkld": -0.004848945140838612, "dbits": 10485760 } ] }, { "idx": 77, "layers": [ "model.layers.13.self_attn.o_proj" ], "candidates": [ { "dkld": -0.003738044947385777, "dbits": 62914560 } ] }, { "idx": 78, "layers": [ "model.layers.13.mlp.shared_experts.gate_proj", "model.layers.13.mlp.shared_experts.up_proj", "model.layers.13.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.004032406955957413, "dbits": 23592960 } ] }, { "idx": 79, "layers": [ "model.layers.13.mlp.experts.0.gate_proj", "model.layers.13.mlp.experts.1.gate_proj", "model.layers.13.mlp.experts.2.gate_proj", "model.layers.13.mlp.experts.3.gate_proj", "model.layers.13.mlp.experts.4.gate_proj", "model.layers.13.mlp.experts.5.gate_proj", "model.layers.13.mlp.experts.6.gate_proj", "model.layers.13.mlp.experts.7.gate_proj", "model.layers.13.mlp.experts.8.gate_proj", "model.layers.13.mlp.experts.9.gate_proj", "model.layers.13.mlp.experts.10.gate_proj", "model.layers.13.mlp.experts.11.gate_proj", "model.layers.13.mlp.experts.12.gate_proj", "model.layers.13.mlp.experts.13.gate_proj", "model.layers.13.mlp.experts.14.gate_proj", "model.layers.13.mlp.experts.15.gate_proj", "model.layers.13.mlp.experts.16.gate_proj", "model.layers.13.mlp.experts.17.gate_proj", "model.layers.13.mlp.experts.18.gate_proj", "model.layers.13.mlp.experts.19.gate_proj", "model.layers.13.mlp.experts.20.gate_proj", "model.layers.13.mlp.experts.21.gate_proj", "model.layers.13.mlp.experts.22.gate_proj", "model.layers.13.mlp.experts.23.gate_proj", "model.layers.13.mlp.experts.24.gate_proj", "model.layers.13.mlp.experts.25.gate_proj", "model.layers.13.mlp.experts.26.gate_proj", "model.layers.13.mlp.experts.27.gate_proj", "model.layers.13.mlp.experts.28.gate_proj", "model.layers.13.mlp.experts.29.gate_proj", "model.layers.13.mlp.experts.30.gate_proj", "model.layers.13.mlp.experts.31.gate_proj", "model.layers.13.mlp.experts.32.gate_proj", "model.layers.13.mlp.experts.33.gate_proj", "model.layers.13.mlp.experts.34.gate_proj", "model.layers.13.mlp.experts.35.gate_proj", "model.layers.13.mlp.experts.36.gate_proj", "model.layers.13.mlp.experts.37.gate_proj", "model.layers.13.mlp.experts.38.gate_proj", "model.layers.13.mlp.experts.39.gate_proj", "model.layers.13.mlp.experts.40.gate_proj", "model.layers.13.mlp.experts.41.gate_proj", "model.layers.13.mlp.experts.42.gate_proj", "model.layers.13.mlp.experts.43.gate_proj", "model.layers.13.mlp.experts.44.gate_proj", "model.layers.13.mlp.experts.45.gate_proj", "model.layers.13.mlp.experts.46.gate_proj", "model.layers.13.mlp.experts.47.gate_proj", "model.layers.13.mlp.experts.48.gate_proj", "model.layers.13.mlp.experts.49.gate_proj", "model.layers.13.mlp.experts.50.gate_proj", "model.layers.13.mlp.experts.51.gate_proj", "model.layers.13.mlp.experts.52.gate_proj", "model.layers.13.mlp.experts.53.gate_proj", "model.layers.13.mlp.experts.54.gate_proj", "model.layers.13.mlp.experts.55.gate_proj", "model.layers.13.mlp.experts.56.gate_proj", "model.layers.13.mlp.experts.57.gate_proj", "model.layers.13.mlp.experts.58.gate_proj", "model.layers.13.mlp.experts.59.gate_proj", "model.layers.13.mlp.experts.60.gate_proj", "model.layers.13.mlp.experts.61.gate_proj", "model.layers.13.mlp.experts.62.gate_proj", "model.layers.13.mlp.experts.63.gate_proj", "model.layers.13.mlp.experts.64.gate_proj", "model.layers.13.mlp.experts.65.gate_proj", "model.layers.13.mlp.experts.66.gate_proj", "model.layers.13.mlp.experts.67.gate_proj", "model.layers.13.mlp.experts.68.gate_proj", "model.layers.13.mlp.experts.69.gate_proj", "model.layers.13.mlp.experts.70.gate_proj", "model.layers.13.mlp.experts.71.gate_proj", "model.layers.13.mlp.experts.72.gate_proj", "model.layers.13.mlp.experts.73.gate_proj", "model.layers.13.mlp.experts.74.gate_proj", "model.layers.13.mlp.experts.75.gate_proj", "model.layers.13.mlp.experts.76.gate_proj", "model.layers.13.mlp.experts.77.gate_proj", "model.layers.13.mlp.experts.78.gate_proj", "model.layers.13.mlp.experts.79.gate_proj", "model.layers.13.mlp.experts.80.gate_proj", "model.layers.13.mlp.experts.81.gate_proj", "model.layers.13.mlp.experts.82.gate_proj", "model.layers.13.mlp.experts.83.gate_proj", "model.layers.13.mlp.experts.84.gate_proj", "model.layers.13.mlp.experts.85.gate_proj", "model.layers.13.mlp.experts.86.gate_proj", "model.layers.13.mlp.experts.87.gate_proj", "model.layers.13.mlp.experts.88.gate_proj", "model.layers.13.mlp.experts.89.gate_proj", "model.layers.13.mlp.experts.90.gate_proj", "model.layers.13.mlp.experts.91.gate_proj", "model.layers.13.mlp.experts.92.gate_proj", "model.layers.13.mlp.experts.93.gate_proj", "model.layers.13.mlp.experts.94.gate_proj", "model.layers.13.mlp.experts.95.gate_proj", "model.layers.13.mlp.experts.96.gate_proj", "model.layers.13.mlp.experts.97.gate_proj", "model.layers.13.mlp.experts.98.gate_proj", "model.layers.13.mlp.experts.99.gate_proj", "model.layers.13.mlp.experts.100.gate_proj", "model.layers.13.mlp.experts.101.gate_proj", "model.layers.13.mlp.experts.102.gate_proj", "model.layers.13.mlp.experts.103.gate_proj", "model.layers.13.mlp.experts.104.gate_proj", "model.layers.13.mlp.experts.105.gate_proj", "model.layers.13.mlp.experts.106.gate_proj", "model.layers.13.mlp.experts.107.gate_proj", "model.layers.13.mlp.experts.108.gate_proj", "model.layers.13.mlp.experts.109.gate_proj", "model.layers.13.mlp.experts.110.gate_proj", "model.layers.13.mlp.experts.111.gate_proj", "model.layers.13.mlp.experts.112.gate_proj", "model.layers.13.mlp.experts.113.gate_proj", "model.layers.13.mlp.experts.114.gate_proj", "model.layers.13.mlp.experts.115.gate_proj", "model.layers.13.mlp.experts.116.gate_proj", "model.layers.13.mlp.experts.117.gate_proj", "model.layers.13.mlp.experts.118.gate_proj", "model.layers.13.mlp.experts.119.gate_proj", "model.layers.13.mlp.experts.120.gate_proj", "model.layers.13.mlp.experts.121.gate_proj", "model.layers.13.mlp.experts.122.gate_proj", "model.layers.13.mlp.experts.123.gate_proj", "model.layers.13.mlp.experts.124.gate_proj", "model.layers.13.mlp.experts.125.gate_proj", "model.layers.13.mlp.experts.126.gate_proj", "model.layers.13.mlp.experts.127.gate_proj", "model.layers.13.mlp.experts.128.gate_proj", "model.layers.13.mlp.experts.129.gate_proj", "model.layers.13.mlp.experts.130.gate_proj", "model.layers.13.mlp.experts.131.gate_proj", "model.layers.13.mlp.experts.132.gate_proj", "model.layers.13.mlp.experts.133.gate_proj", "model.layers.13.mlp.experts.134.gate_proj", "model.layers.13.mlp.experts.135.gate_proj", "model.layers.13.mlp.experts.136.gate_proj", "model.layers.13.mlp.experts.137.gate_proj", "model.layers.13.mlp.experts.138.gate_proj", "model.layers.13.mlp.experts.139.gate_proj", "model.layers.13.mlp.experts.140.gate_proj", "model.layers.13.mlp.experts.141.gate_proj", "model.layers.13.mlp.experts.142.gate_proj", "model.layers.13.mlp.experts.143.gate_proj", "model.layers.13.mlp.experts.144.gate_proj", "model.layers.13.mlp.experts.145.gate_proj", "model.layers.13.mlp.experts.146.gate_proj", "model.layers.13.mlp.experts.147.gate_proj", "model.layers.13.mlp.experts.148.gate_proj", "model.layers.13.mlp.experts.149.gate_proj", "model.layers.13.mlp.experts.150.gate_proj", "model.layers.13.mlp.experts.151.gate_proj", "model.layers.13.mlp.experts.152.gate_proj", "model.layers.13.mlp.experts.153.gate_proj", "model.layers.13.mlp.experts.154.gate_proj", "model.layers.13.mlp.experts.155.gate_proj", "model.layers.13.mlp.experts.156.gate_proj", "model.layers.13.mlp.experts.157.gate_proj", "model.layers.13.mlp.experts.158.gate_proj", "model.layers.13.mlp.experts.159.gate_proj", "model.layers.13.mlp.experts.0.up_proj", "model.layers.13.mlp.experts.1.up_proj", "model.layers.13.mlp.experts.2.up_proj", "model.layers.13.mlp.experts.3.up_proj", "model.layers.13.mlp.experts.4.up_proj", "model.layers.13.mlp.experts.5.up_proj", "model.layers.13.mlp.experts.6.up_proj", "model.layers.13.mlp.experts.7.up_proj", "model.layers.13.mlp.experts.8.up_proj", "model.layers.13.mlp.experts.9.up_proj", "model.layers.13.mlp.experts.10.up_proj", "model.layers.13.mlp.experts.11.up_proj", "model.layers.13.mlp.experts.12.up_proj", "model.layers.13.mlp.experts.13.up_proj", "model.layers.13.mlp.experts.14.up_proj", "model.layers.13.mlp.experts.15.up_proj", "model.layers.13.mlp.experts.16.up_proj", "model.layers.13.mlp.experts.17.up_proj", "model.layers.13.mlp.experts.18.up_proj", "model.layers.13.mlp.experts.19.up_proj", "model.layers.13.mlp.experts.20.up_proj", "model.layers.13.mlp.experts.21.up_proj", "model.layers.13.mlp.experts.22.up_proj", "model.layers.13.mlp.experts.23.up_proj", "model.layers.13.mlp.experts.24.up_proj", "model.layers.13.mlp.experts.25.up_proj", "model.layers.13.mlp.experts.26.up_proj", "model.layers.13.mlp.experts.27.up_proj", "model.layers.13.mlp.experts.28.up_proj", "model.layers.13.mlp.experts.29.up_proj", "model.layers.13.mlp.experts.30.up_proj", "model.layers.13.mlp.experts.31.up_proj", "model.layers.13.mlp.experts.32.up_proj", "model.layers.13.mlp.experts.33.up_proj", "model.layers.13.mlp.experts.34.up_proj", "model.layers.13.mlp.experts.35.up_proj", "model.layers.13.mlp.experts.36.up_proj", "model.layers.13.mlp.experts.37.up_proj", "model.layers.13.mlp.experts.38.up_proj", "model.layers.13.mlp.experts.39.up_proj", "model.layers.13.mlp.experts.40.up_proj", "model.layers.13.mlp.experts.41.up_proj", "model.layers.13.mlp.experts.42.up_proj", "model.layers.13.mlp.experts.43.up_proj", "model.layers.13.mlp.experts.44.up_proj", "model.layers.13.mlp.experts.45.up_proj", "model.layers.13.mlp.experts.46.up_proj", "model.layers.13.mlp.experts.47.up_proj", "model.layers.13.mlp.experts.48.up_proj", "model.layers.13.mlp.experts.49.up_proj", "model.layers.13.mlp.experts.50.up_proj", "model.layers.13.mlp.experts.51.up_proj", "model.layers.13.mlp.experts.52.up_proj", "model.layers.13.mlp.experts.53.up_proj", "model.layers.13.mlp.experts.54.up_proj", "model.layers.13.mlp.experts.55.up_proj", "model.layers.13.mlp.experts.56.up_proj", "model.layers.13.mlp.experts.57.up_proj", "model.layers.13.mlp.experts.58.up_proj", "model.layers.13.mlp.experts.59.up_proj", "model.layers.13.mlp.experts.60.up_proj", "model.layers.13.mlp.experts.61.up_proj", "model.layers.13.mlp.experts.62.up_proj", "model.layers.13.mlp.experts.63.up_proj", "model.layers.13.mlp.experts.64.up_proj", "model.layers.13.mlp.experts.65.up_proj", "model.layers.13.mlp.experts.66.up_proj", "model.layers.13.mlp.experts.67.up_proj", "model.layers.13.mlp.experts.68.up_proj", "model.layers.13.mlp.experts.69.up_proj", "model.layers.13.mlp.experts.70.up_proj", "model.layers.13.mlp.experts.71.up_proj", "model.layers.13.mlp.experts.72.up_proj", "model.layers.13.mlp.experts.73.up_proj", "model.layers.13.mlp.experts.74.up_proj", "model.layers.13.mlp.experts.75.up_proj", "model.layers.13.mlp.experts.76.up_proj", "model.layers.13.mlp.experts.77.up_proj", "model.layers.13.mlp.experts.78.up_proj", "model.layers.13.mlp.experts.79.up_proj", "model.layers.13.mlp.experts.80.up_proj", "model.layers.13.mlp.experts.81.up_proj", "model.layers.13.mlp.experts.82.up_proj", "model.layers.13.mlp.experts.83.up_proj", "model.layers.13.mlp.experts.84.up_proj", "model.layers.13.mlp.experts.85.up_proj", "model.layers.13.mlp.experts.86.up_proj", "model.layers.13.mlp.experts.87.up_proj", "model.layers.13.mlp.experts.88.up_proj", "model.layers.13.mlp.experts.89.up_proj", "model.layers.13.mlp.experts.90.up_proj", "model.layers.13.mlp.experts.91.up_proj", "model.layers.13.mlp.experts.92.up_proj", "model.layers.13.mlp.experts.93.up_proj", "model.layers.13.mlp.experts.94.up_proj", "model.layers.13.mlp.experts.95.up_proj", "model.layers.13.mlp.experts.96.up_proj", "model.layers.13.mlp.experts.97.up_proj", "model.layers.13.mlp.experts.98.up_proj", "model.layers.13.mlp.experts.99.up_proj", "model.layers.13.mlp.experts.100.up_proj", "model.layers.13.mlp.experts.101.up_proj", "model.layers.13.mlp.experts.102.up_proj", "model.layers.13.mlp.experts.103.up_proj", "model.layers.13.mlp.experts.104.up_proj", "model.layers.13.mlp.experts.105.up_proj", "model.layers.13.mlp.experts.106.up_proj", "model.layers.13.mlp.experts.107.up_proj", "model.layers.13.mlp.experts.108.up_proj", "model.layers.13.mlp.experts.109.up_proj", "model.layers.13.mlp.experts.110.up_proj", "model.layers.13.mlp.experts.111.up_proj", "model.layers.13.mlp.experts.112.up_proj", "model.layers.13.mlp.experts.113.up_proj", "model.layers.13.mlp.experts.114.up_proj", "model.layers.13.mlp.experts.115.up_proj", "model.layers.13.mlp.experts.116.up_proj", "model.layers.13.mlp.experts.117.up_proj", "model.layers.13.mlp.experts.118.up_proj", "model.layers.13.mlp.experts.119.up_proj", "model.layers.13.mlp.experts.120.up_proj", "model.layers.13.mlp.experts.121.up_proj", "model.layers.13.mlp.experts.122.up_proj", "model.layers.13.mlp.experts.123.up_proj", "model.layers.13.mlp.experts.124.up_proj", "model.layers.13.mlp.experts.125.up_proj", "model.layers.13.mlp.experts.126.up_proj", "model.layers.13.mlp.experts.127.up_proj", "model.layers.13.mlp.experts.128.up_proj", "model.layers.13.mlp.experts.129.up_proj", "model.layers.13.mlp.experts.130.up_proj", "model.layers.13.mlp.experts.131.up_proj", "model.layers.13.mlp.experts.132.up_proj", "model.layers.13.mlp.experts.133.up_proj", "model.layers.13.mlp.experts.134.up_proj", "model.layers.13.mlp.experts.135.up_proj", "model.layers.13.mlp.experts.136.up_proj", "model.layers.13.mlp.experts.137.up_proj", "model.layers.13.mlp.experts.138.up_proj", "model.layers.13.mlp.experts.139.up_proj", "model.layers.13.mlp.experts.140.up_proj", "model.layers.13.mlp.experts.141.up_proj", "model.layers.13.mlp.experts.142.up_proj", "model.layers.13.mlp.experts.143.up_proj", "model.layers.13.mlp.experts.144.up_proj", "model.layers.13.mlp.experts.145.up_proj", "model.layers.13.mlp.experts.146.up_proj", "model.layers.13.mlp.experts.147.up_proj", "model.layers.13.mlp.experts.148.up_proj", "model.layers.13.mlp.experts.149.up_proj", "model.layers.13.mlp.experts.150.up_proj", "model.layers.13.mlp.experts.151.up_proj", "model.layers.13.mlp.experts.152.up_proj", "model.layers.13.mlp.experts.153.up_proj", "model.layers.13.mlp.experts.154.up_proj", "model.layers.13.mlp.experts.155.up_proj", "model.layers.13.mlp.experts.156.up_proj", "model.layers.13.mlp.experts.157.up_proj", "model.layers.13.mlp.experts.158.up_proj", "model.layers.13.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.002537432312965382, "dbits": 2516582400 } ] }, { "idx": 80, "layers": [ "model.layers.13.mlp.experts.0.down_proj", "model.layers.13.mlp.experts.1.down_proj", "model.layers.13.mlp.experts.2.down_proj", "model.layers.13.mlp.experts.3.down_proj", "model.layers.13.mlp.experts.4.down_proj", "model.layers.13.mlp.experts.5.down_proj", "model.layers.13.mlp.experts.6.down_proj", "model.layers.13.mlp.experts.7.down_proj", "model.layers.13.mlp.experts.8.down_proj", "model.layers.13.mlp.experts.9.down_proj", "model.layers.13.mlp.experts.10.down_proj", "model.layers.13.mlp.experts.11.down_proj", "model.layers.13.mlp.experts.12.down_proj", "model.layers.13.mlp.experts.13.down_proj", "model.layers.13.mlp.experts.14.down_proj", "model.layers.13.mlp.experts.15.down_proj", "model.layers.13.mlp.experts.16.down_proj", "model.layers.13.mlp.experts.17.down_proj", "model.layers.13.mlp.experts.18.down_proj", "model.layers.13.mlp.experts.19.down_proj", "model.layers.13.mlp.experts.20.down_proj", "model.layers.13.mlp.experts.21.down_proj", "model.layers.13.mlp.experts.22.down_proj", "model.layers.13.mlp.experts.23.down_proj", "model.layers.13.mlp.experts.24.down_proj", "model.layers.13.mlp.experts.25.down_proj", "model.layers.13.mlp.experts.26.down_proj", "model.layers.13.mlp.experts.27.down_proj", "model.layers.13.mlp.experts.28.down_proj", "model.layers.13.mlp.experts.29.down_proj", "model.layers.13.mlp.experts.30.down_proj", "model.layers.13.mlp.experts.31.down_proj", "model.layers.13.mlp.experts.32.down_proj", "model.layers.13.mlp.experts.33.down_proj", "model.layers.13.mlp.experts.34.down_proj", "model.layers.13.mlp.experts.35.down_proj", "model.layers.13.mlp.experts.36.down_proj", "model.layers.13.mlp.experts.37.down_proj", "model.layers.13.mlp.experts.38.down_proj", "model.layers.13.mlp.experts.39.down_proj", "model.layers.13.mlp.experts.40.down_proj", "model.layers.13.mlp.experts.41.down_proj", "model.layers.13.mlp.experts.42.down_proj", "model.layers.13.mlp.experts.43.down_proj", "model.layers.13.mlp.experts.44.down_proj", "model.layers.13.mlp.experts.45.down_proj", "model.layers.13.mlp.experts.46.down_proj", "model.layers.13.mlp.experts.47.down_proj", "model.layers.13.mlp.experts.48.down_proj", "model.layers.13.mlp.experts.49.down_proj", "model.layers.13.mlp.experts.50.down_proj", "model.layers.13.mlp.experts.51.down_proj", "model.layers.13.mlp.experts.52.down_proj", "model.layers.13.mlp.experts.53.down_proj", "model.layers.13.mlp.experts.54.down_proj", "model.layers.13.mlp.experts.55.down_proj", "model.layers.13.mlp.experts.56.down_proj", "model.layers.13.mlp.experts.57.down_proj", "model.layers.13.mlp.experts.58.down_proj", "model.layers.13.mlp.experts.59.down_proj", "model.layers.13.mlp.experts.60.down_proj", "model.layers.13.mlp.experts.61.down_proj", "model.layers.13.mlp.experts.62.down_proj", "model.layers.13.mlp.experts.63.down_proj", "model.layers.13.mlp.experts.64.down_proj", "model.layers.13.mlp.experts.65.down_proj", "model.layers.13.mlp.experts.66.down_proj", "model.layers.13.mlp.experts.67.down_proj", "model.layers.13.mlp.experts.68.down_proj", "model.layers.13.mlp.experts.69.down_proj", "model.layers.13.mlp.experts.70.down_proj", "model.layers.13.mlp.experts.71.down_proj", "model.layers.13.mlp.experts.72.down_proj", "model.layers.13.mlp.experts.73.down_proj", "model.layers.13.mlp.experts.74.down_proj", "model.layers.13.mlp.experts.75.down_proj", "model.layers.13.mlp.experts.76.down_proj", "model.layers.13.mlp.experts.77.down_proj", "model.layers.13.mlp.experts.78.down_proj", "model.layers.13.mlp.experts.79.down_proj", "model.layers.13.mlp.experts.80.down_proj", "model.layers.13.mlp.experts.81.down_proj", "model.layers.13.mlp.experts.82.down_proj", "model.layers.13.mlp.experts.83.down_proj", "model.layers.13.mlp.experts.84.down_proj", "model.layers.13.mlp.experts.85.down_proj", "model.layers.13.mlp.experts.86.down_proj", "model.layers.13.mlp.experts.87.down_proj", "model.layers.13.mlp.experts.88.down_proj", "model.layers.13.mlp.experts.89.down_proj", "model.layers.13.mlp.experts.90.down_proj", "model.layers.13.mlp.experts.91.down_proj", "model.layers.13.mlp.experts.92.down_proj", "model.layers.13.mlp.experts.93.down_proj", "model.layers.13.mlp.experts.94.down_proj", "model.layers.13.mlp.experts.95.down_proj", "model.layers.13.mlp.experts.96.down_proj", "model.layers.13.mlp.experts.97.down_proj", "model.layers.13.mlp.experts.98.down_proj", "model.layers.13.mlp.experts.99.down_proj", "model.layers.13.mlp.experts.100.down_proj", "model.layers.13.mlp.experts.101.down_proj", "model.layers.13.mlp.experts.102.down_proj", "model.layers.13.mlp.experts.103.down_proj", "model.layers.13.mlp.experts.104.down_proj", "model.layers.13.mlp.experts.105.down_proj", "model.layers.13.mlp.experts.106.down_proj", "model.layers.13.mlp.experts.107.down_proj", "model.layers.13.mlp.experts.108.down_proj", "model.layers.13.mlp.experts.109.down_proj", "model.layers.13.mlp.experts.110.down_proj", "model.layers.13.mlp.experts.111.down_proj", "model.layers.13.mlp.experts.112.down_proj", "model.layers.13.mlp.experts.113.down_proj", "model.layers.13.mlp.experts.114.down_proj", "model.layers.13.mlp.experts.115.down_proj", "model.layers.13.mlp.experts.116.down_proj", "model.layers.13.mlp.experts.117.down_proj", "model.layers.13.mlp.experts.118.down_proj", "model.layers.13.mlp.experts.119.down_proj", "model.layers.13.mlp.experts.120.down_proj", "model.layers.13.mlp.experts.121.down_proj", "model.layers.13.mlp.experts.122.down_proj", "model.layers.13.mlp.experts.123.down_proj", "model.layers.13.mlp.experts.124.down_proj", "model.layers.13.mlp.experts.125.down_proj", "model.layers.13.mlp.experts.126.down_proj", "model.layers.13.mlp.experts.127.down_proj", "model.layers.13.mlp.experts.128.down_proj", "model.layers.13.mlp.experts.129.down_proj", "model.layers.13.mlp.experts.130.down_proj", "model.layers.13.mlp.experts.131.down_proj", "model.layers.13.mlp.experts.132.down_proj", "model.layers.13.mlp.experts.133.down_proj", "model.layers.13.mlp.experts.134.down_proj", "model.layers.13.mlp.experts.135.down_proj", "model.layers.13.mlp.experts.136.down_proj", "model.layers.13.mlp.experts.137.down_proj", "model.layers.13.mlp.experts.138.down_proj", "model.layers.13.mlp.experts.139.down_proj", "model.layers.13.mlp.experts.140.down_proj", "model.layers.13.mlp.experts.141.down_proj", "model.layers.13.mlp.experts.142.down_proj", "model.layers.13.mlp.experts.143.down_proj", "model.layers.13.mlp.experts.144.down_proj", "model.layers.13.mlp.experts.145.down_proj", "model.layers.13.mlp.experts.146.down_proj", "model.layers.13.mlp.experts.147.down_proj", "model.layers.13.mlp.experts.148.down_proj", "model.layers.13.mlp.experts.149.down_proj", "model.layers.13.mlp.experts.150.down_proj", "model.layers.13.mlp.experts.151.down_proj", "model.layers.13.mlp.experts.152.down_proj", "model.layers.13.mlp.experts.153.down_proj", "model.layers.13.mlp.experts.154.down_proj", "model.layers.13.mlp.experts.155.down_proj", "model.layers.13.mlp.experts.156.down_proj", "model.layers.13.mlp.experts.157.down_proj", "model.layers.13.mlp.experts.158.down_proj", "model.layers.13.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0024606779217720143, "dbits": 1258291200 } ] }, { "idx": 81, "layers": [ "model.layers.14.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0018081553280353546, "dbits": 62914560 } ] }, { "idx": 82, "layers": [ "model.layers.14.self_attn.k_proj", "model.layers.14.self_attn.v_proj" ], "candidates": [ { "dkld": 3.650262951848671e-05, "dbits": 10485760 } ] }, { "idx": 83, "layers": [ "model.layers.14.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0032398782670498005, "dbits": 62914560 } ] }, { "idx": 84, "layers": [ "model.layers.14.mlp.shared_experts.gate_proj", "model.layers.14.mlp.shared_experts.up_proj", "model.layers.14.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0003926776349544858, "dbits": 23592960 } ] }, { "idx": 85, "layers": [ "model.layers.14.mlp.experts.0.gate_proj", "model.layers.14.mlp.experts.1.gate_proj", "model.layers.14.mlp.experts.2.gate_proj", "model.layers.14.mlp.experts.3.gate_proj", "model.layers.14.mlp.experts.4.gate_proj", "model.layers.14.mlp.experts.5.gate_proj", "model.layers.14.mlp.experts.6.gate_proj", "model.layers.14.mlp.experts.7.gate_proj", "model.layers.14.mlp.experts.8.gate_proj", "model.layers.14.mlp.experts.9.gate_proj", "model.layers.14.mlp.experts.10.gate_proj", "model.layers.14.mlp.experts.11.gate_proj", "model.layers.14.mlp.experts.12.gate_proj", "model.layers.14.mlp.experts.13.gate_proj", "model.layers.14.mlp.experts.14.gate_proj", "model.layers.14.mlp.experts.15.gate_proj", "model.layers.14.mlp.experts.16.gate_proj", "model.layers.14.mlp.experts.17.gate_proj", "model.layers.14.mlp.experts.18.gate_proj", "model.layers.14.mlp.experts.19.gate_proj", "model.layers.14.mlp.experts.20.gate_proj", "model.layers.14.mlp.experts.21.gate_proj", "model.layers.14.mlp.experts.22.gate_proj", "model.layers.14.mlp.experts.23.gate_proj", "model.layers.14.mlp.experts.24.gate_proj", "model.layers.14.mlp.experts.25.gate_proj", "model.layers.14.mlp.experts.26.gate_proj", "model.layers.14.mlp.experts.27.gate_proj", "model.layers.14.mlp.experts.28.gate_proj", "model.layers.14.mlp.experts.29.gate_proj", "model.layers.14.mlp.experts.30.gate_proj", "model.layers.14.mlp.experts.31.gate_proj", "model.layers.14.mlp.experts.32.gate_proj", "model.layers.14.mlp.experts.33.gate_proj", "model.layers.14.mlp.experts.34.gate_proj", "model.layers.14.mlp.experts.35.gate_proj", "model.layers.14.mlp.experts.36.gate_proj", "model.layers.14.mlp.experts.37.gate_proj", "model.layers.14.mlp.experts.38.gate_proj", "model.layers.14.mlp.experts.39.gate_proj", "model.layers.14.mlp.experts.40.gate_proj", "model.layers.14.mlp.experts.41.gate_proj", "model.layers.14.mlp.experts.42.gate_proj", "model.layers.14.mlp.experts.43.gate_proj", "model.layers.14.mlp.experts.44.gate_proj", "model.layers.14.mlp.experts.45.gate_proj", "model.layers.14.mlp.experts.46.gate_proj", "model.layers.14.mlp.experts.47.gate_proj", "model.layers.14.mlp.experts.48.gate_proj", "model.layers.14.mlp.experts.49.gate_proj", "model.layers.14.mlp.experts.50.gate_proj", "model.layers.14.mlp.experts.51.gate_proj", "model.layers.14.mlp.experts.52.gate_proj", "model.layers.14.mlp.experts.53.gate_proj", "model.layers.14.mlp.experts.54.gate_proj", "model.layers.14.mlp.experts.55.gate_proj", "model.layers.14.mlp.experts.56.gate_proj", "model.layers.14.mlp.experts.57.gate_proj", "model.layers.14.mlp.experts.58.gate_proj", "model.layers.14.mlp.experts.59.gate_proj", "model.layers.14.mlp.experts.60.gate_proj", "model.layers.14.mlp.experts.61.gate_proj", "model.layers.14.mlp.experts.62.gate_proj", "model.layers.14.mlp.experts.63.gate_proj", "model.layers.14.mlp.experts.64.gate_proj", "model.layers.14.mlp.experts.65.gate_proj", "model.layers.14.mlp.experts.66.gate_proj", "model.layers.14.mlp.experts.67.gate_proj", "model.layers.14.mlp.experts.68.gate_proj", "model.layers.14.mlp.experts.69.gate_proj", "model.layers.14.mlp.experts.70.gate_proj", "model.layers.14.mlp.experts.71.gate_proj", "model.layers.14.mlp.experts.72.gate_proj", "model.layers.14.mlp.experts.73.gate_proj", "model.layers.14.mlp.experts.74.gate_proj", "model.layers.14.mlp.experts.75.gate_proj", "model.layers.14.mlp.experts.76.gate_proj", "model.layers.14.mlp.experts.77.gate_proj", "model.layers.14.mlp.experts.78.gate_proj", "model.layers.14.mlp.experts.79.gate_proj", "model.layers.14.mlp.experts.80.gate_proj", "model.layers.14.mlp.experts.81.gate_proj", "model.layers.14.mlp.experts.82.gate_proj", "model.layers.14.mlp.experts.83.gate_proj", "model.layers.14.mlp.experts.84.gate_proj", "model.layers.14.mlp.experts.85.gate_proj", "model.layers.14.mlp.experts.86.gate_proj", "model.layers.14.mlp.experts.87.gate_proj", "model.layers.14.mlp.experts.88.gate_proj", "model.layers.14.mlp.experts.89.gate_proj", "model.layers.14.mlp.experts.90.gate_proj", "model.layers.14.mlp.experts.91.gate_proj", "model.layers.14.mlp.experts.92.gate_proj", "model.layers.14.mlp.experts.93.gate_proj", "model.layers.14.mlp.experts.94.gate_proj", "model.layers.14.mlp.experts.95.gate_proj", "model.layers.14.mlp.experts.96.gate_proj", "model.layers.14.mlp.experts.97.gate_proj", "model.layers.14.mlp.experts.98.gate_proj", "model.layers.14.mlp.experts.99.gate_proj", "model.layers.14.mlp.experts.100.gate_proj", "model.layers.14.mlp.experts.101.gate_proj", "model.layers.14.mlp.experts.102.gate_proj", "model.layers.14.mlp.experts.103.gate_proj", "model.layers.14.mlp.experts.104.gate_proj", "model.layers.14.mlp.experts.105.gate_proj", "model.layers.14.mlp.experts.106.gate_proj", "model.layers.14.mlp.experts.107.gate_proj", "model.layers.14.mlp.experts.108.gate_proj", "model.layers.14.mlp.experts.109.gate_proj", "model.layers.14.mlp.experts.110.gate_proj", "model.layers.14.mlp.experts.111.gate_proj", "model.layers.14.mlp.experts.112.gate_proj", "model.layers.14.mlp.experts.113.gate_proj", "model.layers.14.mlp.experts.114.gate_proj", "model.layers.14.mlp.experts.115.gate_proj", "model.layers.14.mlp.experts.116.gate_proj", "model.layers.14.mlp.experts.117.gate_proj", "model.layers.14.mlp.experts.118.gate_proj", "model.layers.14.mlp.experts.119.gate_proj", "model.layers.14.mlp.experts.120.gate_proj", "model.layers.14.mlp.experts.121.gate_proj", "model.layers.14.mlp.experts.122.gate_proj", "model.layers.14.mlp.experts.123.gate_proj", "model.layers.14.mlp.experts.124.gate_proj", "model.layers.14.mlp.experts.125.gate_proj", "model.layers.14.mlp.experts.126.gate_proj", "model.layers.14.mlp.experts.127.gate_proj", "model.layers.14.mlp.experts.128.gate_proj", "model.layers.14.mlp.experts.129.gate_proj", "model.layers.14.mlp.experts.130.gate_proj", "model.layers.14.mlp.experts.131.gate_proj", "model.layers.14.mlp.experts.132.gate_proj", "model.layers.14.mlp.experts.133.gate_proj", "model.layers.14.mlp.experts.134.gate_proj", "model.layers.14.mlp.experts.135.gate_proj", "model.layers.14.mlp.experts.136.gate_proj", "model.layers.14.mlp.experts.137.gate_proj", "model.layers.14.mlp.experts.138.gate_proj", "model.layers.14.mlp.experts.139.gate_proj", "model.layers.14.mlp.experts.140.gate_proj", "model.layers.14.mlp.experts.141.gate_proj", "model.layers.14.mlp.experts.142.gate_proj", "model.layers.14.mlp.experts.143.gate_proj", "model.layers.14.mlp.experts.144.gate_proj", "model.layers.14.mlp.experts.145.gate_proj", "model.layers.14.mlp.experts.146.gate_proj", "model.layers.14.mlp.experts.147.gate_proj", "model.layers.14.mlp.experts.148.gate_proj", "model.layers.14.mlp.experts.149.gate_proj", "model.layers.14.mlp.experts.150.gate_proj", "model.layers.14.mlp.experts.151.gate_proj", "model.layers.14.mlp.experts.152.gate_proj", "model.layers.14.mlp.experts.153.gate_proj", "model.layers.14.mlp.experts.154.gate_proj", "model.layers.14.mlp.experts.155.gate_proj", "model.layers.14.mlp.experts.156.gate_proj", "model.layers.14.mlp.experts.157.gate_proj", "model.layers.14.mlp.experts.158.gate_proj", "model.layers.14.mlp.experts.159.gate_proj", "model.layers.14.mlp.experts.0.up_proj", "model.layers.14.mlp.experts.1.up_proj", "model.layers.14.mlp.experts.2.up_proj", "model.layers.14.mlp.experts.3.up_proj", "model.layers.14.mlp.experts.4.up_proj", "model.layers.14.mlp.experts.5.up_proj", "model.layers.14.mlp.experts.6.up_proj", "model.layers.14.mlp.experts.7.up_proj", "model.layers.14.mlp.experts.8.up_proj", "model.layers.14.mlp.experts.9.up_proj", "model.layers.14.mlp.experts.10.up_proj", "model.layers.14.mlp.experts.11.up_proj", "model.layers.14.mlp.experts.12.up_proj", "model.layers.14.mlp.experts.13.up_proj", "model.layers.14.mlp.experts.14.up_proj", "model.layers.14.mlp.experts.15.up_proj", "model.layers.14.mlp.experts.16.up_proj", "model.layers.14.mlp.experts.17.up_proj", "model.layers.14.mlp.experts.18.up_proj", "model.layers.14.mlp.experts.19.up_proj", "model.layers.14.mlp.experts.20.up_proj", "model.layers.14.mlp.experts.21.up_proj", "model.layers.14.mlp.experts.22.up_proj", "model.layers.14.mlp.experts.23.up_proj", "model.layers.14.mlp.experts.24.up_proj", "model.layers.14.mlp.experts.25.up_proj", "model.layers.14.mlp.experts.26.up_proj", "model.layers.14.mlp.experts.27.up_proj", "model.layers.14.mlp.experts.28.up_proj", "model.layers.14.mlp.experts.29.up_proj", "model.layers.14.mlp.experts.30.up_proj", "model.layers.14.mlp.experts.31.up_proj", "model.layers.14.mlp.experts.32.up_proj", "model.layers.14.mlp.experts.33.up_proj", "model.layers.14.mlp.experts.34.up_proj", "model.layers.14.mlp.experts.35.up_proj", "model.layers.14.mlp.experts.36.up_proj", "model.layers.14.mlp.experts.37.up_proj", "model.layers.14.mlp.experts.38.up_proj", "model.layers.14.mlp.experts.39.up_proj", "model.layers.14.mlp.experts.40.up_proj", "model.layers.14.mlp.experts.41.up_proj", "model.layers.14.mlp.experts.42.up_proj", "model.layers.14.mlp.experts.43.up_proj", "model.layers.14.mlp.experts.44.up_proj", "model.layers.14.mlp.experts.45.up_proj", "model.layers.14.mlp.experts.46.up_proj", "model.layers.14.mlp.experts.47.up_proj", "model.layers.14.mlp.experts.48.up_proj", "model.layers.14.mlp.experts.49.up_proj", "model.layers.14.mlp.experts.50.up_proj", "model.layers.14.mlp.experts.51.up_proj", "model.layers.14.mlp.experts.52.up_proj", "model.layers.14.mlp.experts.53.up_proj", "model.layers.14.mlp.experts.54.up_proj", "model.layers.14.mlp.experts.55.up_proj", "model.layers.14.mlp.experts.56.up_proj", "model.layers.14.mlp.experts.57.up_proj", "model.layers.14.mlp.experts.58.up_proj", "model.layers.14.mlp.experts.59.up_proj", "model.layers.14.mlp.experts.60.up_proj", "model.layers.14.mlp.experts.61.up_proj", "model.layers.14.mlp.experts.62.up_proj", "model.layers.14.mlp.experts.63.up_proj", "model.layers.14.mlp.experts.64.up_proj", "model.layers.14.mlp.experts.65.up_proj", "model.layers.14.mlp.experts.66.up_proj", "model.layers.14.mlp.experts.67.up_proj", "model.layers.14.mlp.experts.68.up_proj", "model.layers.14.mlp.experts.69.up_proj", "model.layers.14.mlp.experts.70.up_proj", "model.layers.14.mlp.experts.71.up_proj", "model.layers.14.mlp.experts.72.up_proj", "model.layers.14.mlp.experts.73.up_proj", "model.layers.14.mlp.experts.74.up_proj", "model.layers.14.mlp.experts.75.up_proj", "model.layers.14.mlp.experts.76.up_proj", "model.layers.14.mlp.experts.77.up_proj", "model.layers.14.mlp.experts.78.up_proj", "model.layers.14.mlp.experts.79.up_proj", "model.layers.14.mlp.experts.80.up_proj", "model.layers.14.mlp.experts.81.up_proj", "model.layers.14.mlp.experts.82.up_proj", "model.layers.14.mlp.experts.83.up_proj", "model.layers.14.mlp.experts.84.up_proj", "model.layers.14.mlp.experts.85.up_proj", "model.layers.14.mlp.experts.86.up_proj", "model.layers.14.mlp.experts.87.up_proj", "model.layers.14.mlp.experts.88.up_proj", "model.layers.14.mlp.experts.89.up_proj", "model.layers.14.mlp.experts.90.up_proj", "model.layers.14.mlp.experts.91.up_proj", "model.layers.14.mlp.experts.92.up_proj", "model.layers.14.mlp.experts.93.up_proj", "model.layers.14.mlp.experts.94.up_proj", "model.layers.14.mlp.experts.95.up_proj", "model.layers.14.mlp.experts.96.up_proj", "model.layers.14.mlp.experts.97.up_proj", "model.layers.14.mlp.experts.98.up_proj", "model.layers.14.mlp.experts.99.up_proj", "model.layers.14.mlp.experts.100.up_proj", "model.layers.14.mlp.experts.101.up_proj", "model.layers.14.mlp.experts.102.up_proj", "model.layers.14.mlp.experts.103.up_proj", "model.layers.14.mlp.experts.104.up_proj", "model.layers.14.mlp.experts.105.up_proj", "model.layers.14.mlp.experts.106.up_proj", "model.layers.14.mlp.experts.107.up_proj", "model.layers.14.mlp.experts.108.up_proj", "model.layers.14.mlp.experts.109.up_proj", "model.layers.14.mlp.experts.110.up_proj", "model.layers.14.mlp.experts.111.up_proj", "model.layers.14.mlp.experts.112.up_proj", "model.layers.14.mlp.experts.113.up_proj", "model.layers.14.mlp.experts.114.up_proj", "model.layers.14.mlp.experts.115.up_proj", "model.layers.14.mlp.experts.116.up_proj", "model.layers.14.mlp.experts.117.up_proj", "model.layers.14.mlp.experts.118.up_proj", "model.layers.14.mlp.experts.119.up_proj", "model.layers.14.mlp.experts.120.up_proj", "model.layers.14.mlp.experts.121.up_proj", "model.layers.14.mlp.experts.122.up_proj", "model.layers.14.mlp.experts.123.up_proj", "model.layers.14.mlp.experts.124.up_proj", "model.layers.14.mlp.experts.125.up_proj", "model.layers.14.mlp.experts.126.up_proj", "model.layers.14.mlp.experts.127.up_proj", "model.layers.14.mlp.experts.128.up_proj", "model.layers.14.mlp.experts.129.up_proj", "model.layers.14.mlp.experts.130.up_proj", "model.layers.14.mlp.experts.131.up_proj", "model.layers.14.mlp.experts.132.up_proj", "model.layers.14.mlp.experts.133.up_proj", "model.layers.14.mlp.experts.134.up_proj", "model.layers.14.mlp.experts.135.up_proj", "model.layers.14.mlp.experts.136.up_proj", "model.layers.14.mlp.experts.137.up_proj", "model.layers.14.mlp.experts.138.up_proj", "model.layers.14.mlp.experts.139.up_proj", "model.layers.14.mlp.experts.140.up_proj", "model.layers.14.mlp.experts.141.up_proj", "model.layers.14.mlp.experts.142.up_proj", "model.layers.14.mlp.experts.143.up_proj", "model.layers.14.mlp.experts.144.up_proj", "model.layers.14.mlp.experts.145.up_proj", "model.layers.14.mlp.experts.146.up_proj", "model.layers.14.mlp.experts.147.up_proj", "model.layers.14.mlp.experts.148.up_proj", "model.layers.14.mlp.experts.149.up_proj", "model.layers.14.mlp.experts.150.up_proj", "model.layers.14.mlp.experts.151.up_proj", "model.layers.14.mlp.experts.152.up_proj", "model.layers.14.mlp.experts.153.up_proj", "model.layers.14.mlp.experts.154.up_proj", "model.layers.14.mlp.experts.155.up_proj", "model.layers.14.mlp.experts.156.up_proj", "model.layers.14.mlp.experts.157.up_proj", "model.layers.14.mlp.experts.158.up_proj", "model.layers.14.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.003055129200220108, "dbits": 2516582400 } ] }, { "idx": 86, "layers": [ "model.layers.14.mlp.experts.0.down_proj", "model.layers.14.mlp.experts.1.down_proj", "model.layers.14.mlp.experts.2.down_proj", "model.layers.14.mlp.experts.3.down_proj", "model.layers.14.mlp.experts.4.down_proj", "model.layers.14.mlp.experts.5.down_proj", "model.layers.14.mlp.experts.6.down_proj", "model.layers.14.mlp.experts.7.down_proj", "model.layers.14.mlp.experts.8.down_proj", "model.layers.14.mlp.experts.9.down_proj", "model.layers.14.mlp.experts.10.down_proj", "model.layers.14.mlp.experts.11.down_proj", "model.layers.14.mlp.experts.12.down_proj", "model.layers.14.mlp.experts.13.down_proj", "model.layers.14.mlp.experts.14.down_proj", "model.layers.14.mlp.experts.15.down_proj", "model.layers.14.mlp.experts.16.down_proj", "model.layers.14.mlp.experts.17.down_proj", "model.layers.14.mlp.experts.18.down_proj", "model.layers.14.mlp.experts.19.down_proj", "model.layers.14.mlp.experts.20.down_proj", "model.layers.14.mlp.experts.21.down_proj", "model.layers.14.mlp.experts.22.down_proj", "model.layers.14.mlp.experts.23.down_proj", "model.layers.14.mlp.experts.24.down_proj", "model.layers.14.mlp.experts.25.down_proj", "model.layers.14.mlp.experts.26.down_proj", "model.layers.14.mlp.experts.27.down_proj", "model.layers.14.mlp.experts.28.down_proj", "model.layers.14.mlp.experts.29.down_proj", "model.layers.14.mlp.experts.30.down_proj", "model.layers.14.mlp.experts.31.down_proj", "model.layers.14.mlp.experts.32.down_proj", "model.layers.14.mlp.experts.33.down_proj", "model.layers.14.mlp.experts.34.down_proj", "model.layers.14.mlp.experts.35.down_proj", "model.layers.14.mlp.experts.36.down_proj", "model.layers.14.mlp.experts.37.down_proj", "model.layers.14.mlp.experts.38.down_proj", "model.layers.14.mlp.experts.39.down_proj", "model.layers.14.mlp.experts.40.down_proj", "model.layers.14.mlp.experts.41.down_proj", "model.layers.14.mlp.experts.42.down_proj", "model.layers.14.mlp.experts.43.down_proj", "model.layers.14.mlp.experts.44.down_proj", "model.layers.14.mlp.experts.45.down_proj", "model.layers.14.mlp.experts.46.down_proj", "model.layers.14.mlp.experts.47.down_proj", "model.layers.14.mlp.experts.48.down_proj", "model.layers.14.mlp.experts.49.down_proj", "model.layers.14.mlp.experts.50.down_proj", "model.layers.14.mlp.experts.51.down_proj", "model.layers.14.mlp.experts.52.down_proj", "model.layers.14.mlp.experts.53.down_proj", "model.layers.14.mlp.experts.54.down_proj", "model.layers.14.mlp.experts.55.down_proj", "model.layers.14.mlp.experts.56.down_proj", "model.layers.14.mlp.experts.57.down_proj", "model.layers.14.mlp.experts.58.down_proj", "model.layers.14.mlp.experts.59.down_proj", "model.layers.14.mlp.experts.60.down_proj", "model.layers.14.mlp.experts.61.down_proj", "model.layers.14.mlp.experts.62.down_proj", "model.layers.14.mlp.experts.63.down_proj", "model.layers.14.mlp.experts.64.down_proj", "model.layers.14.mlp.experts.65.down_proj", "model.layers.14.mlp.experts.66.down_proj", "model.layers.14.mlp.experts.67.down_proj", "model.layers.14.mlp.experts.68.down_proj", "model.layers.14.mlp.experts.69.down_proj", "model.layers.14.mlp.experts.70.down_proj", "model.layers.14.mlp.experts.71.down_proj", "model.layers.14.mlp.experts.72.down_proj", "model.layers.14.mlp.experts.73.down_proj", "model.layers.14.mlp.experts.74.down_proj", "model.layers.14.mlp.experts.75.down_proj", "model.layers.14.mlp.experts.76.down_proj", "model.layers.14.mlp.experts.77.down_proj", "model.layers.14.mlp.experts.78.down_proj", "model.layers.14.mlp.experts.79.down_proj", "model.layers.14.mlp.experts.80.down_proj", "model.layers.14.mlp.experts.81.down_proj", "model.layers.14.mlp.experts.82.down_proj", "model.layers.14.mlp.experts.83.down_proj", "model.layers.14.mlp.experts.84.down_proj", "model.layers.14.mlp.experts.85.down_proj", "model.layers.14.mlp.experts.86.down_proj", "model.layers.14.mlp.experts.87.down_proj", "model.layers.14.mlp.experts.88.down_proj", "model.layers.14.mlp.experts.89.down_proj", "model.layers.14.mlp.experts.90.down_proj", "model.layers.14.mlp.experts.91.down_proj", "model.layers.14.mlp.experts.92.down_proj", "model.layers.14.mlp.experts.93.down_proj", "model.layers.14.mlp.experts.94.down_proj", "model.layers.14.mlp.experts.95.down_proj", "model.layers.14.mlp.experts.96.down_proj", "model.layers.14.mlp.experts.97.down_proj", "model.layers.14.mlp.experts.98.down_proj", "model.layers.14.mlp.experts.99.down_proj", "model.layers.14.mlp.experts.100.down_proj", "model.layers.14.mlp.experts.101.down_proj", "model.layers.14.mlp.experts.102.down_proj", "model.layers.14.mlp.experts.103.down_proj", "model.layers.14.mlp.experts.104.down_proj", "model.layers.14.mlp.experts.105.down_proj", "model.layers.14.mlp.experts.106.down_proj", "model.layers.14.mlp.experts.107.down_proj", "model.layers.14.mlp.experts.108.down_proj", "model.layers.14.mlp.experts.109.down_proj", "model.layers.14.mlp.experts.110.down_proj", "model.layers.14.mlp.experts.111.down_proj", "model.layers.14.mlp.experts.112.down_proj", "model.layers.14.mlp.experts.113.down_proj", "model.layers.14.mlp.experts.114.down_proj", "model.layers.14.mlp.experts.115.down_proj", "model.layers.14.mlp.experts.116.down_proj", "model.layers.14.mlp.experts.117.down_proj", "model.layers.14.mlp.experts.118.down_proj", "model.layers.14.mlp.experts.119.down_proj", "model.layers.14.mlp.experts.120.down_proj", "model.layers.14.mlp.experts.121.down_proj", "model.layers.14.mlp.experts.122.down_proj", "model.layers.14.mlp.experts.123.down_proj", "model.layers.14.mlp.experts.124.down_proj", "model.layers.14.mlp.experts.125.down_proj", "model.layers.14.mlp.experts.126.down_proj", "model.layers.14.mlp.experts.127.down_proj", "model.layers.14.mlp.experts.128.down_proj", "model.layers.14.mlp.experts.129.down_proj", "model.layers.14.mlp.experts.130.down_proj", "model.layers.14.mlp.experts.131.down_proj", "model.layers.14.mlp.experts.132.down_proj", "model.layers.14.mlp.experts.133.down_proj", "model.layers.14.mlp.experts.134.down_proj", "model.layers.14.mlp.experts.135.down_proj", "model.layers.14.mlp.experts.136.down_proj", "model.layers.14.mlp.experts.137.down_proj", "model.layers.14.mlp.experts.138.down_proj", "model.layers.14.mlp.experts.139.down_proj", "model.layers.14.mlp.experts.140.down_proj", "model.layers.14.mlp.experts.141.down_proj", "model.layers.14.mlp.experts.142.down_proj", "model.layers.14.mlp.experts.143.down_proj", "model.layers.14.mlp.experts.144.down_proj", "model.layers.14.mlp.experts.145.down_proj", "model.layers.14.mlp.experts.146.down_proj", "model.layers.14.mlp.experts.147.down_proj", "model.layers.14.mlp.experts.148.down_proj", "model.layers.14.mlp.experts.149.down_proj", "model.layers.14.mlp.experts.150.down_proj", "model.layers.14.mlp.experts.151.down_proj", "model.layers.14.mlp.experts.152.down_proj", "model.layers.14.mlp.experts.153.down_proj", "model.layers.14.mlp.experts.154.down_proj", "model.layers.14.mlp.experts.155.down_proj", "model.layers.14.mlp.experts.156.down_proj", "model.layers.14.mlp.experts.157.down_proj", "model.layers.14.mlp.experts.158.down_proj", "model.layers.14.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002107186615467105, "dbits": 1258291200 } ] }, { "idx": 87, "layers": [ "model.layers.15.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0016474671661853568, "dbits": 62914560 } ] }, { "idx": 88, "layers": [ "model.layers.15.self_attn.k_proj", "model.layers.15.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00020568221807476528, "dbits": 10485760 } ] }, { "idx": 89, "layers": [ "model.layers.15.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0008925125002861023, "dbits": 62914560 } ] }, { "idx": 90, "layers": [ "model.layers.15.mlp.shared_experts.gate_proj", "model.layers.15.mlp.shared_experts.up_proj", "model.layers.15.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0018712520599365456, "dbits": 23592960 } ] }, { "idx": 91, "layers": [ "model.layers.15.mlp.experts.0.gate_proj", "model.layers.15.mlp.experts.1.gate_proj", "model.layers.15.mlp.experts.2.gate_proj", "model.layers.15.mlp.experts.3.gate_proj", "model.layers.15.mlp.experts.4.gate_proj", "model.layers.15.mlp.experts.5.gate_proj", "model.layers.15.mlp.experts.6.gate_proj", "model.layers.15.mlp.experts.7.gate_proj", "model.layers.15.mlp.experts.8.gate_proj", "model.layers.15.mlp.experts.9.gate_proj", "model.layers.15.mlp.experts.10.gate_proj", "model.layers.15.mlp.experts.11.gate_proj", "model.layers.15.mlp.experts.12.gate_proj", "model.layers.15.mlp.experts.13.gate_proj", "model.layers.15.mlp.experts.14.gate_proj", "model.layers.15.mlp.experts.15.gate_proj", "model.layers.15.mlp.experts.16.gate_proj", "model.layers.15.mlp.experts.17.gate_proj", "model.layers.15.mlp.experts.18.gate_proj", "model.layers.15.mlp.experts.19.gate_proj", "model.layers.15.mlp.experts.20.gate_proj", "model.layers.15.mlp.experts.21.gate_proj", "model.layers.15.mlp.experts.22.gate_proj", "model.layers.15.mlp.experts.23.gate_proj", "model.layers.15.mlp.experts.24.gate_proj", "model.layers.15.mlp.experts.25.gate_proj", "model.layers.15.mlp.experts.26.gate_proj", "model.layers.15.mlp.experts.27.gate_proj", "model.layers.15.mlp.experts.28.gate_proj", "model.layers.15.mlp.experts.29.gate_proj", "model.layers.15.mlp.experts.30.gate_proj", "model.layers.15.mlp.experts.31.gate_proj", "model.layers.15.mlp.experts.32.gate_proj", "model.layers.15.mlp.experts.33.gate_proj", "model.layers.15.mlp.experts.34.gate_proj", "model.layers.15.mlp.experts.35.gate_proj", "model.layers.15.mlp.experts.36.gate_proj", "model.layers.15.mlp.experts.37.gate_proj", "model.layers.15.mlp.experts.38.gate_proj", "model.layers.15.mlp.experts.39.gate_proj", "model.layers.15.mlp.experts.40.gate_proj", "model.layers.15.mlp.experts.41.gate_proj", "model.layers.15.mlp.experts.42.gate_proj", "model.layers.15.mlp.experts.43.gate_proj", "model.layers.15.mlp.experts.44.gate_proj", "model.layers.15.mlp.experts.45.gate_proj", "model.layers.15.mlp.experts.46.gate_proj", "model.layers.15.mlp.experts.47.gate_proj", "model.layers.15.mlp.experts.48.gate_proj", "model.layers.15.mlp.experts.49.gate_proj", "model.layers.15.mlp.experts.50.gate_proj", "model.layers.15.mlp.experts.51.gate_proj", "model.layers.15.mlp.experts.52.gate_proj", "model.layers.15.mlp.experts.53.gate_proj", "model.layers.15.mlp.experts.54.gate_proj", "model.layers.15.mlp.experts.55.gate_proj", "model.layers.15.mlp.experts.56.gate_proj", "model.layers.15.mlp.experts.57.gate_proj", "model.layers.15.mlp.experts.58.gate_proj", "model.layers.15.mlp.experts.59.gate_proj", "model.layers.15.mlp.experts.60.gate_proj", "model.layers.15.mlp.experts.61.gate_proj", "model.layers.15.mlp.experts.62.gate_proj", "model.layers.15.mlp.experts.63.gate_proj", "model.layers.15.mlp.experts.64.gate_proj", "model.layers.15.mlp.experts.65.gate_proj", "model.layers.15.mlp.experts.66.gate_proj", "model.layers.15.mlp.experts.67.gate_proj", "model.layers.15.mlp.experts.68.gate_proj", "model.layers.15.mlp.experts.69.gate_proj", "model.layers.15.mlp.experts.70.gate_proj", "model.layers.15.mlp.experts.71.gate_proj", "model.layers.15.mlp.experts.72.gate_proj", "model.layers.15.mlp.experts.73.gate_proj", "model.layers.15.mlp.experts.74.gate_proj", "model.layers.15.mlp.experts.75.gate_proj", "model.layers.15.mlp.experts.76.gate_proj", "model.layers.15.mlp.experts.77.gate_proj", "model.layers.15.mlp.experts.78.gate_proj", "model.layers.15.mlp.experts.79.gate_proj", "model.layers.15.mlp.experts.80.gate_proj", "model.layers.15.mlp.experts.81.gate_proj", "model.layers.15.mlp.experts.82.gate_proj", "model.layers.15.mlp.experts.83.gate_proj", "model.layers.15.mlp.experts.84.gate_proj", "model.layers.15.mlp.experts.85.gate_proj", "model.layers.15.mlp.experts.86.gate_proj", "model.layers.15.mlp.experts.87.gate_proj", "model.layers.15.mlp.experts.88.gate_proj", "model.layers.15.mlp.experts.89.gate_proj", "model.layers.15.mlp.experts.90.gate_proj", "model.layers.15.mlp.experts.91.gate_proj", "model.layers.15.mlp.experts.92.gate_proj", "model.layers.15.mlp.experts.93.gate_proj", "model.layers.15.mlp.experts.94.gate_proj", "model.layers.15.mlp.experts.95.gate_proj", "model.layers.15.mlp.experts.96.gate_proj", "model.layers.15.mlp.experts.97.gate_proj", "model.layers.15.mlp.experts.98.gate_proj", "model.layers.15.mlp.experts.99.gate_proj", "model.layers.15.mlp.experts.100.gate_proj", "model.layers.15.mlp.experts.101.gate_proj", "model.layers.15.mlp.experts.102.gate_proj", "model.layers.15.mlp.experts.103.gate_proj", "model.layers.15.mlp.experts.104.gate_proj", "model.layers.15.mlp.experts.105.gate_proj", "model.layers.15.mlp.experts.106.gate_proj", "model.layers.15.mlp.experts.107.gate_proj", "model.layers.15.mlp.experts.108.gate_proj", "model.layers.15.mlp.experts.109.gate_proj", "model.layers.15.mlp.experts.110.gate_proj", "model.layers.15.mlp.experts.111.gate_proj", "model.layers.15.mlp.experts.112.gate_proj", "model.layers.15.mlp.experts.113.gate_proj", "model.layers.15.mlp.experts.114.gate_proj", "model.layers.15.mlp.experts.115.gate_proj", "model.layers.15.mlp.experts.116.gate_proj", "model.layers.15.mlp.experts.117.gate_proj", "model.layers.15.mlp.experts.118.gate_proj", "model.layers.15.mlp.experts.119.gate_proj", "model.layers.15.mlp.experts.120.gate_proj", "model.layers.15.mlp.experts.121.gate_proj", "model.layers.15.mlp.experts.122.gate_proj", "model.layers.15.mlp.experts.123.gate_proj", "model.layers.15.mlp.experts.124.gate_proj", "model.layers.15.mlp.experts.125.gate_proj", "model.layers.15.mlp.experts.126.gate_proj", "model.layers.15.mlp.experts.127.gate_proj", "model.layers.15.mlp.experts.128.gate_proj", "model.layers.15.mlp.experts.129.gate_proj", "model.layers.15.mlp.experts.130.gate_proj", "model.layers.15.mlp.experts.131.gate_proj", "model.layers.15.mlp.experts.132.gate_proj", "model.layers.15.mlp.experts.133.gate_proj", "model.layers.15.mlp.experts.134.gate_proj", "model.layers.15.mlp.experts.135.gate_proj", "model.layers.15.mlp.experts.136.gate_proj", "model.layers.15.mlp.experts.137.gate_proj", "model.layers.15.mlp.experts.138.gate_proj", "model.layers.15.mlp.experts.139.gate_proj", "model.layers.15.mlp.experts.140.gate_proj", "model.layers.15.mlp.experts.141.gate_proj", "model.layers.15.mlp.experts.142.gate_proj", "model.layers.15.mlp.experts.143.gate_proj", "model.layers.15.mlp.experts.144.gate_proj", "model.layers.15.mlp.experts.145.gate_proj", "model.layers.15.mlp.experts.146.gate_proj", "model.layers.15.mlp.experts.147.gate_proj", "model.layers.15.mlp.experts.148.gate_proj", "model.layers.15.mlp.experts.149.gate_proj", "model.layers.15.mlp.experts.150.gate_proj", "model.layers.15.mlp.experts.151.gate_proj", "model.layers.15.mlp.experts.152.gate_proj", "model.layers.15.mlp.experts.153.gate_proj", "model.layers.15.mlp.experts.154.gate_proj", "model.layers.15.mlp.experts.155.gate_proj", "model.layers.15.mlp.experts.156.gate_proj", "model.layers.15.mlp.experts.157.gate_proj", "model.layers.15.mlp.experts.158.gate_proj", "model.layers.15.mlp.experts.159.gate_proj", "model.layers.15.mlp.experts.0.up_proj", "model.layers.15.mlp.experts.1.up_proj", "model.layers.15.mlp.experts.2.up_proj", "model.layers.15.mlp.experts.3.up_proj", "model.layers.15.mlp.experts.4.up_proj", "model.layers.15.mlp.experts.5.up_proj", "model.layers.15.mlp.experts.6.up_proj", "model.layers.15.mlp.experts.7.up_proj", "model.layers.15.mlp.experts.8.up_proj", "model.layers.15.mlp.experts.9.up_proj", "model.layers.15.mlp.experts.10.up_proj", "model.layers.15.mlp.experts.11.up_proj", "model.layers.15.mlp.experts.12.up_proj", "model.layers.15.mlp.experts.13.up_proj", "model.layers.15.mlp.experts.14.up_proj", "model.layers.15.mlp.experts.15.up_proj", "model.layers.15.mlp.experts.16.up_proj", "model.layers.15.mlp.experts.17.up_proj", "model.layers.15.mlp.experts.18.up_proj", "model.layers.15.mlp.experts.19.up_proj", "model.layers.15.mlp.experts.20.up_proj", "model.layers.15.mlp.experts.21.up_proj", "model.layers.15.mlp.experts.22.up_proj", "model.layers.15.mlp.experts.23.up_proj", "model.layers.15.mlp.experts.24.up_proj", "model.layers.15.mlp.experts.25.up_proj", "model.layers.15.mlp.experts.26.up_proj", "model.layers.15.mlp.experts.27.up_proj", "model.layers.15.mlp.experts.28.up_proj", "model.layers.15.mlp.experts.29.up_proj", "model.layers.15.mlp.experts.30.up_proj", "model.layers.15.mlp.experts.31.up_proj", "model.layers.15.mlp.experts.32.up_proj", "model.layers.15.mlp.experts.33.up_proj", "model.layers.15.mlp.experts.34.up_proj", "model.layers.15.mlp.experts.35.up_proj", "model.layers.15.mlp.experts.36.up_proj", "model.layers.15.mlp.experts.37.up_proj", "model.layers.15.mlp.experts.38.up_proj", "model.layers.15.mlp.experts.39.up_proj", "model.layers.15.mlp.experts.40.up_proj", "model.layers.15.mlp.experts.41.up_proj", "model.layers.15.mlp.experts.42.up_proj", "model.layers.15.mlp.experts.43.up_proj", "model.layers.15.mlp.experts.44.up_proj", "model.layers.15.mlp.experts.45.up_proj", "model.layers.15.mlp.experts.46.up_proj", "model.layers.15.mlp.experts.47.up_proj", "model.layers.15.mlp.experts.48.up_proj", "model.layers.15.mlp.experts.49.up_proj", "model.layers.15.mlp.experts.50.up_proj", "model.layers.15.mlp.experts.51.up_proj", "model.layers.15.mlp.experts.52.up_proj", "model.layers.15.mlp.experts.53.up_proj", "model.layers.15.mlp.experts.54.up_proj", "model.layers.15.mlp.experts.55.up_proj", "model.layers.15.mlp.experts.56.up_proj", "model.layers.15.mlp.experts.57.up_proj", "model.layers.15.mlp.experts.58.up_proj", "model.layers.15.mlp.experts.59.up_proj", "model.layers.15.mlp.experts.60.up_proj", "model.layers.15.mlp.experts.61.up_proj", "model.layers.15.mlp.experts.62.up_proj", "model.layers.15.mlp.experts.63.up_proj", "model.layers.15.mlp.experts.64.up_proj", "model.layers.15.mlp.experts.65.up_proj", "model.layers.15.mlp.experts.66.up_proj", "model.layers.15.mlp.experts.67.up_proj", "model.layers.15.mlp.experts.68.up_proj", "model.layers.15.mlp.experts.69.up_proj", "model.layers.15.mlp.experts.70.up_proj", "model.layers.15.mlp.experts.71.up_proj", "model.layers.15.mlp.experts.72.up_proj", "model.layers.15.mlp.experts.73.up_proj", "model.layers.15.mlp.experts.74.up_proj", "model.layers.15.mlp.experts.75.up_proj", "model.layers.15.mlp.experts.76.up_proj", "model.layers.15.mlp.experts.77.up_proj", "model.layers.15.mlp.experts.78.up_proj", "model.layers.15.mlp.experts.79.up_proj", "model.layers.15.mlp.experts.80.up_proj", "model.layers.15.mlp.experts.81.up_proj", "model.layers.15.mlp.experts.82.up_proj", "model.layers.15.mlp.experts.83.up_proj", "model.layers.15.mlp.experts.84.up_proj", "model.layers.15.mlp.experts.85.up_proj", "model.layers.15.mlp.experts.86.up_proj", "model.layers.15.mlp.experts.87.up_proj", "model.layers.15.mlp.experts.88.up_proj", "model.layers.15.mlp.experts.89.up_proj", "model.layers.15.mlp.experts.90.up_proj", "model.layers.15.mlp.experts.91.up_proj", "model.layers.15.mlp.experts.92.up_proj", "model.layers.15.mlp.experts.93.up_proj", "model.layers.15.mlp.experts.94.up_proj", "model.layers.15.mlp.experts.95.up_proj", "model.layers.15.mlp.experts.96.up_proj", "model.layers.15.mlp.experts.97.up_proj", "model.layers.15.mlp.experts.98.up_proj", "model.layers.15.mlp.experts.99.up_proj", "model.layers.15.mlp.experts.100.up_proj", "model.layers.15.mlp.experts.101.up_proj", "model.layers.15.mlp.experts.102.up_proj", "model.layers.15.mlp.experts.103.up_proj", "model.layers.15.mlp.experts.104.up_proj", "model.layers.15.mlp.experts.105.up_proj", "model.layers.15.mlp.experts.106.up_proj", "model.layers.15.mlp.experts.107.up_proj", "model.layers.15.mlp.experts.108.up_proj", "model.layers.15.mlp.experts.109.up_proj", "model.layers.15.mlp.experts.110.up_proj", "model.layers.15.mlp.experts.111.up_proj", "model.layers.15.mlp.experts.112.up_proj", "model.layers.15.mlp.experts.113.up_proj", "model.layers.15.mlp.experts.114.up_proj", "model.layers.15.mlp.experts.115.up_proj", "model.layers.15.mlp.experts.116.up_proj", "model.layers.15.mlp.experts.117.up_proj", "model.layers.15.mlp.experts.118.up_proj", "model.layers.15.mlp.experts.119.up_proj", "model.layers.15.mlp.experts.120.up_proj", "model.layers.15.mlp.experts.121.up_proj", "model.layers.15.mlp.experts.122.up_proj", "model.layers.15.mlp.experts.123.up_proj", "model.layers.15.mlp.experts.124.up_proj", "model.layers.15.mlp.experts.125.up_proj", "model.layers.15.mlp.experts.126.up_proj", "model.layers.15.mlp.experts.127.up_proj", "model.layers.15.mlp.experts.128.up_proj", "model.layers.15.mlp.experts.129.up_proj", "model.layers.15.mlp.experts.130.up_proj", "model.layers.15.mlp.experts.131.up_proj", "model.layers.15.mlp.experts.132.up_proj", "model.layers.15.mlp.experts.133.up_proj", "model.layers.15.mlp.experts.134.up_proj", "model.layers.15.mlp.experts.135.up_proj", "model.layers.15.mlp.experts.136.up_proj", "model.layers.15.mlp.experts.137.up_proj", "model.layers.15.mlp.experts.138.up_proj", "model.layers.15.mlp.experts.139.up_proj", "model.layers.15.mlp.experts.140.up_proj", "model.layers.15.mlp.experts.141.up_proj", "model.layers.15.mlp.experts.142.up_proj", "model.layers.15.mlp.experts.143.up_proj", "model.layers.15.mlp.experts.144.up_proj", "model.layers.15.mlp.experts.145.up_proj", "model.layers.15.mlp.experts.146.up_proj", "model.layers.15.mlp.experts.147.up_proj", "model.layers.15.mlp.experts.148.up_proj", "model.layers.15.mlp.experts.149.up_proj", "model.layers.15.mlp.experts.150.up_proj", "model.layers.15.mlp.experts.151.up_proj", "model.layers.15.mlp.experts.152.up_proj", "model.layers.15.mlp.experts.153.up_proj", "model.layers.15.mlp.experts.154.up_proj", "model.layers.15.mlp.experts.155.up_proj", "model.layers.15.mlp.experts.156.up_proj", "model.layers.15.mlp.experts.157.up_proj", "model.layers.15.mlp.experts.158.up_proj", "model.layers.15.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.002976980060338974, "dbits": 2516582400 } ] }, { "idx": 92, "layers": [ "model.layers.15.mlp.experts.0.down_proj", "model.layers.15.mlp.experts.1.down_proj", "model.layers.15.mlp.experts.2.down_proj", "model.layers.15.mlp.experts.3.down_proj", "model.layers.15.mlp.experts.4.down_proj", "model.layers.15.mlp.experts.5.down_proj", "model.layers.15.mlp.experts.6.down_proj", "model.layers.15.mlp.experts.7.down_proj", "model.layers.15.mlp.experts.8.down_proj", "model.layers.15.mlp.experts.9.down_proj", "model.layers.15.mlp.experts.10.down_proj", "model.layers.15.mlp.experts.11.down_proj", "model.layers.15.mlp.experts.12.down_proj", "model.layers.15.mlp.experts.13.down_proj", "model.layers.15.mlp.experts.14.down_proj", "model.layers.15.mlp.experts.15.down_proj", "model.layers.15.mlp.experts.16.down_proj", "model.layers.15.mlp.experts.17.down_proj", "model.layers.15.mlp.experts.18.down_proj", "model.layers.15.mlp.experts.19.down_proj", "model.layers.15.mlp.experts.20.down_proj", "model.layers.15.mlp.experts.21.down_proj", "model.layers.15.mlp.experts.22.down_proj", "model.layers.15.mlp.experts.23.down_proj", "model.layers.15.mlp.experts.24.down_proj", "model.layers.15.mlp.experts.25.down_proj", "model.layers.15.mlp.experts.26.down_proj", "model.layers.15.mlp.experts.27.down_proj", "model.layers.15.mlp.experts.28.down_proj", "model.layers.15.mlp.experts.29.down_proj", "model.layers.15.mlp.experts.30.down_proj", "model.layers.15.mlp.experts.31.down_proj", "model.layers.15.mlp.experts.32.down_proj", "model.layers.15.mlp.experts.33.down_proj", "model.layers.15.mlp.experts.34.down_proj", "model.layers.15.mlp.experts.35.down_proj", "model.layers.15.mlp.experts.36.down_proj", "model.layers.15.mlp.experts.37.down_proj", "model.layers.15.mlp.experts.38.down_proj", "model.layers.15.mlp.experts.39.down_proj", "model.layers.15.mlp.experts.40.down_proj", "model.layers.15.mlp.experts.41.down_proj", "model.layers.15.mlp.experts.42.down_proj", "model.layers.15.mlp.experts.43.down_proj", "model.layers.15.mlp.experts.44.down_proj", "model.layers.15.mlp.experts.45.down_proj", "model.layers.15.mlp.experts.46.down_proj", "model.layers.15.mlp.experts.47.down_proj", "model.layers.15.mlp.experts.48.down_proj", "model.layers.15.mlp.experts.49.down_proj", "model.layers.15.mlp.experts.50.down_proj", "model.layers.15.mlp.experts.51.down_proj", "model.layers.15.mlp.experts.52.down_proj", "model.layers.15.mlp.experts.53.down_proj", "model.layers.15.mlp.experts.54.down_proj", "model.layers.15.mlp.experts.55.down_proj", "model.layers.15.mlp.experts.56.down_proj", "model.layers.15.mlp.experts.57.down_proj", "model.layers.15.mlp.experts.58.down_proj", "model.layers.15.mlp.experts.59.down_proj", "model.layers.15.mlp.experts.60.down_proj", "model.layers.15.mlp.experts.61.down_proj", "model.layers.15.mlp.experts.62.down_proj", "model.layers.15.mlp.experts.63.down_proj", "model.layers.15.mlp.experts.64.down_proj", "model.layers.15.mlp.experts.65.down_proj", "model.layers.15.mlp.experts.66.down_proj", "model.layers.15.mlp.experts.67.down_proj", "model.layers.15.mlp.experts.68.down_proj", "model.layers.15.mlp.experts.69.down_proj", "model.layers.15.mlp.experts.70.down_proj", "model.layers.15.mlp.experts.71.down_proj", "model.layers.15.mlp.experts.72.down_proj", "model.layers.15.mlp.experts.73.down_proj", "model.layers.15.mlp.experts.74.down_proj", "model.layers.15.mlp.experts.75.down_proj", "model.layers.15.mlp.experts.76.down_proj", "model.layers.15.mlp.experts.77.down_proj", "model.layers.15.mlp.experts.78.down_proj", "model.layers.15.mlp.experts.79.down_proj", "model.layers.15.mlp.experts.80.down_proj", "model.layers.15.mlp.experts.81.down_proj", "model.layers.15.mlp.experts.82.down_proj", "model.layers.15.mlp.experts.83.down_proj", "model.layers.15.mlp.experts.84.down_proj", "model.layers.15.mlp.experts.85.down_proj", "model.layers.15.mlp.experts.86.down_proj", "model.layers.15.mlp.experts.87.down_proj", "model.layers.15.mlp.experts.88.down_proj", "model.layers.15.mlp.experts.89.down_proj", "model.layers.15.mlp.experts.90.down_proj", "model.layers.15.mlp.experts.91.down_proj", "model.layers.15.mlp.experts.92.down_proj", "model.layers.15.mlp.experts.93.down_proj", "model.layers.15.mlp.experts.94.down_proj", "model.layers.15.mlp.experts.95.down_proj", "model.layers.15.mlp.experts.96.down_proj", "model.layers.15.mlp.experts.97.down_proj", "model.layers.15.mlp.experts.98.down_proj", "model.layers.15.mlp.experts.99.down_proj", "model.layers.15.mlp.experts.100.down_proj", "model.layers.15.mlp.experts.101.down_proj", "model.layers.15.mlp.experts.102.down_proj", "model.layers.15.mlp.experts.103.down_proj", "model.layers.15.mlp.experts.104.down_proj", "model.layers.15.mlp.experts.105.down_proj", "model.layers.15.mlp.experts.106.down_proj", "model.layers.15.mlp.experts.107.down_proj", "model.layers.15.mlp.experts.108.down_proj", "model.layers.15.mlp.experts.109.down_proj", "model.layers.15.mlp.experts.110.down_proj", "model.layers.15.mlp.experts.111.down_proj", "model.layers.15.mlp.experts.112.down_proj", "model.layers.15.mlp.experts.113.down_proj", "model.layers.15.mlp.experts.114.down_proj", "model.layers.15.mlp.experts.115.down_proj", "model.layers.15.mlp.experts.116.down_proj", "model.layers.15.mlp.experts.117.down_proj", "model.layers.15.mlp.experts.118.down_proj", "model.layers.15.mlp.experts.119.down_proj", "model.layers.15.mlp.experts.120.down_proj", "model.layers.15.mlp.experts.121.down_proj", "model.layers.15.mlp.experts.122.down_proj", "model.layers.15.mlp.experts.123.down_proj", "model.layers.15.mlp.experts.124.down_proj", "model.layers.15.mlp.experts.125.down_proj", "model.layers.15.mlp.experts.126.down_proj", "model.layers.15.mlp.experts.127.down_proj", "model.layers.15.mlp.experts.128.down_proj", "model.layers.15.mlp.experts.129.down_proj", "model.layers.15.mlp.experts.130.down_proj", "model.layers.15.mlp.experts.131.down_proj", "model.layers.15.mlp.experts.132.down_proj", "model.layers.15.mlp.experts.133.down_proj", "model.layers.15.mlp.experts.134.down_proj", "model.layers.15.mlp.experts.135.down_proj", "model.layers.15.mlp.experts.136.down_proj", "model.layers.15.mlp.experts.137.down_proj", "model.layers.15.mlp.experts.138.down_proj", "model.layers.15.mlp.experts.139.down_proj", "model.layers.15.mlp.experts.140.down_proj", "model.layers.15.mlp.experts.141.down_proj", "model.layers.15.mlp.experts.142.down_proj", "model.layers.15.mlp.experts.143.down_proj", "model.layers.15.mlp.experts.144.down_proj", "model.layers.15.mlp.experts.145.down_proj", "model.layers.15.mlp.experts.146.down_proj", "model.layers.15.mlp.experts.147.down_proj", "model.layers.15.mlp.experts.148.down_proj", "model.layers.15.mlp.experts.149.down_proj", "model.layers.15.mlp.experts.150.down_proj", "model.layers.15.mlp.experts.151.down_proj", "model.layers.15.mlp.experts.152.down_proj", "model.layers.15.mlp.experts.153.down_proj", "model.layers.15.mlp.experts.154.down_proj", "model.layers.15.mlp.experts.155.down_proj", "model.layers.15.mlp.experts.156.down_proj", "model.layers.15.mlp.experts.157.down_proj", "model.layers.15.mlp.experts.158.down_proj", "model.layers.15.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0030275903642177915, "dbits": 1258291200 } ] }, { "idx": 93, "layers": [ "model.layers.16.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0005450904369354026, "dbits": 62914560 } ] }, { "idx": 94, "layers": [ "model.layers.16.self_attn.k_proj", "model.layers.16.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0004159390926360973, "dbits": 10485760 } ] }, { "idx": 95, "layers": [ "model.layers.16.self_attn.o_proj" ], "candidates": [ { "dkld": 0.000358256697654713, "dbits": 62914560 } ] }, { "idx": 96, "layers": [ "model.layers.16.mlp.shared_experts.gate_proj", "model.layers.16.mlp.shared_experts.up_proj", "model.layers.16.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0005298279225826596, "dbits": 23592960 } ] }, { "idx": 97, "layers": [ "model.layers.16.mlp.experts.0.gate_proj", "model.layers.16.mlp.experts.1.gate_proj", "model.layers.16.mlp.experts.2.gate_proj", "model.layers.16.mlp.experts.3.gate_proj", "model.layers.16.mlp.experts.4.gate_proj", "model.layers.16.mlp.experts.5.gate_proj", "model.layers.16.mlp.experts.6.gate_proj", "model.layers.16.mlp.experts.7.gate_proj", "model.layers.16.mlp.experts.8.gate_proj", "model.layers.16.mlp.experts.9.gate_proj", "model.layers.16.mlp.experts.10.gate_proj", "model.layers.16.mlp.experts.11.gate_proj", "model.layers.16.mlp.experts.12.gate_proj", "model.layers.16.mlp.experts.13.gate_proj", "model.layers.16.mlp.experts.14.gate_proj", "model.layers.16.mlp.experts.15.gate_proj", "model.layers.16.mlp.experts.16.gate_proj", "model.layers.16.mlp.experts.17.gate_proj", "model.layers.16.mlp.experts.18.gate_proj", "model.layers.16.mlp.experts.19.gate_proj", "model.layers.16.mlp.experts.20.gate_proj", "model.layers.16.mlp.experts.21.gate_proj", "model.layers.16.mlp.experts.22.gate_proj", "model.layers.16.mlp.experts.23.gate_proj", "model.layers.16.mlp.experts.24.gate_proj", "model.layers.16.mlp.experts.25.gate_proj", "model.layers.16.mlp.experts.26.gate_proj", "model.layers.16.mlp.experts.27.gate_proj", "model.layers.16.mlp.experts.28.gate_proj", "model.layers.16.mlp.experts.29.gate_proj", "model.layers.16.mlp.experts.30.gate_proj", "model.layers.16.mlp.experts.31.gate_proj", "model.layers.16.mlp.experts.32.gate_proj", "model.layers.16.mlp.experts.33.gate_proj", "model.layers.16.mlp.experts.34.gate_proj", "model.layers.16.mlp.experts.35.gate_proj", "model.layers.16.mlp.experts.36.gate_proj", "model.layers.16.mlp.experts.37.gate_proj", "model.layers.16.mlp.experts.38.gate_proj", "model.layers.16.mlp.experts.39.gate_proj", "model.layers.16.mlp.experts.40.gate_proj", "model.layers.16.mlp.experts.41.gate_proj", "model.layers.16.mlp.experts.42.gate_proj", "model.layers.16.mlp.experts.43.gate_proj", "model.layers.16.mlp.experts.44.gate_proj", "model.layers.16.mlp.experts.45.gate_proj", "model.layers.16.mlp.experts.46.gate_proj", "model.layers.16.mlp.experts.47.gate_proj", "model.layers.16.mlp.experts.48.gate_proj", "model.layers.16.mlp.experts.49.gate_proj", "model.layers.16.mlp.experts.50.gate_proj", "model.layers.16.mlp.experts.51.gate_proj", "model.layers.16.mlp.experts.52.gate_proj", "model.layers.16.mlp.experts.53.gate_proj", "model.layers.16.mlp.experts.54.gate_proj", "model.layers.16.mlp.experts.55.gate_proj", "model.layers.16.mlp.experts.56.gate_proj", "model.layers.16.mlp.experts.57.gate_proj", "model.layers.16.mlp.experts.58.gate_proj", "model.layers.16.mlp.experts.59.gate_proj", "model.layers.16.mlp.experts.60.gate_proj", "model.layers.16.mlp.experts.61.gate_proj", "model.layers.16.mlp.experts.62.gate_proj", "model.layers.16.mlp.experts.63.gate_proj", "model.layers.16.mlp.experts.64.gate_proj", "model.layers.16.mlp.experts.65.gate_proj", "model.layers.16.mlp.experts.66.gate_proj", "model.layers.16.mlp.experts.67.gate_proj", "model.layers.16.mlp.experts.68.gate_proj", "model.layers.16.mlp.experts.69.gate_proj", "model.layers.16.mlp.experts.70.gate_proj", "model.layers.16.mlp.experts.71.gate_proj", "model.layers.16.mlp.experts.72.gate_proj", "model.layers.16.mlp.experts.73.gate_proj", "model.layers.16.mlp.experts.74.gate_proj", "model.layers.16.mlp.experts.75.gate_proj", "model.layers.16.mlp.experts.76.gate_proj", "model.layers.16.mlp.experts.77.gate_proj", "model.layers.16.mlp.experts.78.gate_proj", "model.layers.16.mlp.experts.79.gate_proj", "model.layers.16.mlp.experts.80.gate_proj", "model.layers.16.mlp.experts.81.gate_proj", "model.layers.16.mlp.experts.82.gate_proj", "model.layers.16.mlp.experts.83.gate_proj", "model.layers.16.mlp.experts.84.gate_proj", "model.layers.16.mlp.experts.85.gate_proj", "model.layers.16.mlp.experts.86.gate_proj", "model.layers.16.mlp.experts.87.gate_proj", "model.layers.16.mlp.experts.88.gate_proj", "model.layers.16.mlp.experts.89.gate_proj", "model.layers.16.mlp.experts.90.gate_proj", "model.layers.16.mlp.experts.91.gate_proj", "model.layers.16.mlp.experts.92.gate_proj", "model.layers.16.mlp.experts.93.gate_proj", "model.layers.16.mlp.experts.94.gate_proj", "model.layers.16.mlp.experts.95.gate_proj", "model.layers.16.mlp.experts.96.gate_proj", "model.layers.16.mlp.experts.97.gate_proj", "model.layers.16.mlp.experts.98.gate_proj", "model.layers.16.mlp.experts.99.gate_proj", "model.layers.16.mlp.experts.100.gate_proj", "model.layers.16.mlp.experts.101.gate_proj", "model.layers.16.mlp.experts.102.gate_proj", "model.layers.16.mlp.experts.103.gate_proj", "model.layers.16.mlp.experts.104.gate_proj", "model.layers.16.mlp.experts.105.gate_proj", "model.layers.16.mlp.experts.106.gate_proj", "model.layers.16.mlp.experts.107.gate_proj", "model.layers.16.mlp.experts.108.gate_proj", "model.layers.16.mlp.experts.109.gate_proj", "model.layers.16.mlp.experts.110.gate_proj", "model.layers.16.mlp.experts.111.gate_proj", "model.layers.16.mlp.experts.112.gate_proj", "model.layers.16.mlp.experts.113.gate_proj", "model.layers.16.mlp.experts.114.gate_proj", "model.layers.16.mlp.experts.115.gate_proj", "model.layers.16.mlp.experts.116.gate_proj", "model.layers.16.mlp.experts.117.gate_proj", "model.layers.16.mlp.experts.118.gate_proj", "model.layers.16.mlp.experts.119.gate_proj", "model.layers.16.mlp.experts.120.gate_proj", "model.layers.16.mlp.experts.121.gate_proj", "model.layers.16.mlp.experts.122.gate_proj", "model.layers.16.mlp.experts.123.gate_proj", "model.layers.16.mlp.experts.124.gate_proj", "model.layers.16.mlp.experts.125.gate_proj", "model.layers.16.mlp.experts.126.gate_proj", "model.layers.16.mlp.experts.127.gate_proj", "model.layers.16.mlp.experts.128.gate_proj", "model.layers.16.mlp.experts.129.gate_proj", "model.layers.16.mlp.experts.130.gate_proj", "model.layers.16.mlp.experts.131.gate_proj", "model.layers.16.mlp.experts.132.gate_proj", "model.layers.16.mlp.experts.133.gate_proj", "model.layers.16.mlp.experts.134.gate_proj", "model.layers.16.mlp.experts.135.gate_proj", "model.layers.16.mlp.experts.136.gate_proj", "model.layers.16.mlp.experts.137.gate_proj", "model.layers.16.mlp.experts.138.gate_proj", "model.layers.16.mlp.experts.139.gate_proj", "model.layers.16.mlp.experts.140.gate_proj", "model.layers.16.mlp.experts.141.gate_proj", "model.layers.16.mlp.experts.142.gate_proj", "model.layers.16.mlp.experts.143.gate_proj", "model.layers.16.mlp.experts.144.gate_proj", "model.layers.16.mlp.experts.145.gate_proj", "model.layers.16.mlp.experts.146.gate_proj", "model.layers.16.mlp.experts.147.gate_proj", "model.layers.16.mlp.experts.148.gate_proj", "model.layers.16.mlp.experts.149.gate_proj", "model.layers.16.mlp.experts.150.gate_proj", "model.layers.16.mlp.experts.151.gate_proj", "model.layers.16.mlp.experts.152.gate_proj", "model.layers.16.mlp.experts.153.gate_proj", "model.layers.16.mlp.experts.154.gate_proj", "model.layers.16.mlp.experts.155.gate_proj", "model.layers.16.mlp.experts.156.gate_proj", "model.layers.16.mlp.experts.157.gate_proj", "model.layers.16.mlp.experts.158.gate_proj", "model.layers.16.mlp.experts.159.gate_proj", "model.layers.16.mlp.experts.0.up_proj", "model.layers.16.mlp.experts.1.up_proj", "model.layers.16.mlp.experts.2.up_proj", "model.layers.16.mlp.experts.3.up_proj", "model.layers.16.mlp.experts.4.up_proj", "model.layers.16.mlp.experts.5.up_proj", "model.layers.16.mlp.experts.6.up_proj", "model.layers.16.mlp.experts.7.up_proj", "model.layers.16.mlp.experts.8.up_proj", "model.layers.16.mlp.experts.9.up_proj", "model.layers.16.mlp.experts.10.up_proj", "model.layers.16.mlp.experts.11.up_proj", "model.layers.16.mlp.experts.12.up_proj", "model.layers.16.mlp.experts.13.up_proj", "model.layers.16.mlp.experts.14.up_proj", "model.layers.16.mlp.experts.15.up_proj", "model.layers.16.mlp.experts.16.up_proj", "model.layers.16.mlp.experts.17.up_proj", "model.layers.16.mlp.experts.18.up_proj", "model.layers.16.mlp.experts.19.up_proj", "model.layers.16.mlp.experts.20.up_proj", "model.layers.16.mlp.experts.21.up_proj", "model.layers.16.mlp.experts.22.up_proj", "model.layers.16.mlp.experts.23.up_proj", "model.layers.16.mlp.experts.24.up_proj", "model.layers.16.mlp.experts.25.up_proj", "model.layers.16.mlp.experts.26.up_proj", "model.layers.16.mlp.experts.27.up_proj", "model.layers.16.mlp.experts.28.up_proj", "model.layers.16.mlp.experts.29.up_proj", "model.layers.16.mlp.experts.30.up_proj", "model.layers.16.mlp.experts.31.up_proj", "model.layers.16.mlp.experts.32.up_proj", "model.layers.16.mlp.experts.33.up_proj", "model.layers.16.mlp.experts.34.up_proj", "model.layers.16.mlp.experts.35.up_proj", "model.layers.16.mlp.experts.36.up_proj", "model.layers.16.mlp.experts.37.up_proj", "model.layers.16.mlp.experts.38.up_proj", "model.layers.16.mlp.experts.39.up_proj", "model.layers.16.mlp.experts.40.up_proj", "model.layers.16.mlp.experts.41.up_proj", "model.layers.16.mlp.experts.42.up_proj", "model.layers.16.mlp.experts.43.up_proj", "model.layers.16.mlp.experts.44.up_proj", "model.layers.16.mlp.experts.45.up_proj", "model.layers.16.mlp.experts.46.up_proj", "model.layers.16.mlp.experts.47.up_proj", "model.layers.16.mlp.experts.48.up_proj", "model.layers.16.mlp.experts.49.up_proj", "model.layers.16.mlp.experts.50.up_proj", "model.layers.16.mlp.experts.51.up_proj", "model.layers.16.mlp.experts.52.up_proj", "model.layers.16.mlp.experts.53.up_proj", "model.layers.16.mlp.experts.54.up_proj", "model.layers.16.mlp.experts.55.up_proj", "model.layers.16.mlp.experts.56.up_proj", "model.layers.16.mlp.experts.57.up_proj", "model.layers.16.mlp.experts.58.up_proj", "model.layers.16.mlp.experts.59.up_proj", "model.layers.16.mlp.experts.60.up_proj", "model.layers.16.mlp.experts.61.up_proj", "model.layers.16.mlp.experts.62.up_proj", "model.layers.16.mlp.experts.63.up_proj", "model.layers.16.mlp.experts.64.up_proj", "model.layers.16.mlp.experts.65.up_proj", "model.layers.16.mlp.experts.66.up_proj", "model.layers.16.mlp.experts.67.up_proj", "model.layers.16.mlp.experts.68.up_proj", "model.layers.16.mlp.experts.69.up_proj", "model.layers.16.mlp.experts.70.up_proj", "model.layers.16.mlp.experts.71.up_proj", "model.layers.16.mlp.experts.72.up_proj", "model.layers.16.mlp.experts.73.up_proj", "model.layers.16.mlp.experts.74.up_proj", "model.layers.16.mlp.experts.75.up_proj", "model.layers.16.mlp.experts.76.up_proj", "model.layers.16.mlp.experts.77.up_proj", "model.layers.16.mlp.experts.78.up_proj", "model.layers.16.mlp.experts.79.up_proj", "model.layers.16.mlp.experts.80.up_proj", "model.layers.16.mlp.experts.81.up_proj", "model.layers.16.mlp.experts.82.up_proj", "model.layers.16.mlp.experts.83.up_proj", "model.layers.16.mlp.experts.84.up_proj", "model.layers.16.mlp.experts.85.up_proj", "model.layers.16.mlp.experts.86.up_proj", "model.layers.16.mlp.experts.87.up_proj", "model.layers.16.mlp.experts.88.up_proj", "model.layers.16.mlp.experts.89.up_proj", "model.layers.16.mlp.experts.90.up_proj", "model.layers.16.mlp.experts.91.up_proj", "model.layers.16.mlp.experts.92.up_proj", "model.layers.16.mlp.experts.93.up_proj", "model.layers.16.mlp.experts.94.up_proj", "model.layers.16.mlp.experts.95.up_proj", "model.layers.16.mlp.experts.96.up_proj", "model.layers.16.mlp.experts.97.up_proj", "model.layers.16.mlp.experts.98.up_proj", "model.layers.16.mlp.experts.99.up_proj", "model.layers.16.mlp.experts.100.up_proj", "model.layers.16.mlp.experts.101.up_proj", "model.layers.16.mlp.experts.102.up_proj", "model.layers.16.mlp.experts.103.up_proj", "model.layers.16.mlp.experts.104.up_proj", "model.layers.16.mlp.experts.105.up_proj", "model.layers.16.mlp.experts.106.up_proj", "model.layers.16.mlp.experts.107.up_proj", "model.layers.16.mlp.experts.108.up_proj", "model.layers.16.mlp.experts.109.up_proj", "model.layers.16.mlp.experts.110.up_proj", "model.layers.16.mlp.experts.111.up_proj", "model.layers.16.mlp.experts.112.up_proj", "model.layers.16.mlp.experts.113.up_proj", "model.layers.16.mlp.experts.114.up_proj", "model.layers.16.mlp.experts.115.up_proj", "model.layers.16.mlp.experts.116.up_proj", "model.layers.16.mlp.experts.117.up_proj", "model.layers.16.mlp.experts.118.up_proj", "model.layers.16.mlp.experts.119.up_proj", "model.layers.16.mlp.experts.120.up_proj", "model.layers.16.mlp.experts.121.up_proj", "model.layers.16.mlp.experts.122.up_proj", "model.layers.16.mlp.experts.123.up_proj", "model.layers.16.mlp.experts.124.up_proj", "model.layers.16.mlp.experts.125.up_proj", "model.layers.16.mlp.experts.126.up_proj", "model.layers.16.mlp.experts.127.up_proj", "model.layers.16.mlp.experts.128.up_proj", "model.layers.16.mlp.experts.129.up_proj", "model.layers.16.mlp.experts.130.up_proj", "model.layers.16.mlp.experts.131.up_proj", "model.layers.16.mlp.experts.132.up_proj", "model.layers.16.mlp.experts.133.up_proj", "model.layers.16.mlp.experts.134.up_proj", "model.layers.16.mlp.experts.135.up_proj", "model.layers.16.mlp.experts.136.up_proj", "model.layers.16.mlp.experts.137.up_proj", "model.layers.16.mlp.experts.138.up_proj", "model.layers.16.mlp.experts.139.up_proj", "model.layers.16.mlp.experts.140.up_proj", "model.layers.16.mlp.experts.141.up_proj", "model.layers.16.mlp.experts.142.up_proj", "model.layers.16.mlp.experts.143.up_proj", "model.layers.16.mlp.experts.144.up_proj", "model.layers.16.mlp.experts.145.up_proj", "model.layers.16.mlp.experts.146.up_proj", "model.layers.16.mlp.experts.147.up_proj", "model.layers.16.mlp.experts.148.up_proj", "model.layers.16.mlp.experts.149.up_proj", "model.layers.16.mlp.experts.150.up_proj", "model.layers.16.mlp.experts.151.up_proj", "model.layers.16.mlp.experts.152.up_proj", "model.layers.16.mlp.experts.153.up_proj", "model.layers.16.mlp.experts.154.up_proj", "model.layers.16.mlp.experts.155.up_proj", "model.layers.16.mlp.experts.156.up_proj", "model.layers.16.mlp.experts.157.up_proj", "model.layers.16.mlp.experts.158.up_proj", "model.layers.16.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0031852893531322812, "dbits": 2516582400 } ] }, { "idx": 98, "layers": [ "model.layers.16.mlp.experts.0.down_proj", "model.layers.16.mlp.experts.1.down_proj", "model.layers.16.mlp.experts.2.down_proj", "model.layers.16.mlp.experts.3.down_proj", "model.layers.16.mlp.experts.4.down_proj", "model.layers.16.mlp.experts.5.down_proj", "model.layers.16.mlp.experts.6.down_proj", "model.layers.16.mlp.experts.7.down_proj", "model.layers.16.mlp.experts.8.down_proj", "model.layers.16.mlp.experts.9.down_proj", "model.layers.16.mlp.experts.10.down_proj", "model.layers.16.mlp.experts.11.down_proj", "model.layers.16.mlp.experts.12.down_proj", "model.layers.16.mlp.experts.13.down_proj", "model.layers.16.mlp.experts.14.down_proj", "model.layers.16.mlp.experts.15.down_proj", "model.layers.16.mlp.experts.16.down_proj", "model.layers.16.mlp.experts.17.down_proj", "model.layers.16.mlp.experts.18.down_proj", "model.layers.16.mlp.experts.19.down_proj", "model.layers.16.mlp.experts.20.down_proj", "model.layers.16.mlp.experts.21.down_proj", "model.layers.16.mlp.experts.22.down_proj", "model.layers.16.mlp.experts.23.down_proj", "model.layers.16.mlp.experts.24.down_proj", "model.layers.16.mlp.experts.25.down_proj", "model.layers.16.mlp.experts.26.down_proj", "model.layers.16.mlp.experts.27.down_proj", "model.layers.16.mlp.experts.28.down_proj", "model.layers.16.mlp.experts.29.down_proj", "model.layers.16.mlp.experts.30.down_proj", "model.layers.16.mlp.experts.31.down_proj", "model.layers.16.mlp.experts.32.down_proj", "model.layers.16.mlp.experts.33.down_proj", "model.layers.16.mlp.experts.34.down_proj", "model.layers.16.mlp.experts.35.down_proj", "model.layers.16.mlp.experts.36.down_proj", "model.layers.16.mlp.experts.37.down_proj", "model.layers.16.mlp.experts.38.down_proj", "model.layers.16.mlp.experts.39.down_proj", "model.layers.16.mlp.experts.40.down_proj", "model.layers.16.mlp.experts.41.down_proj", "model.layers.16.mlp.experts.42.down_proj", "model.layers.16.mlp.experts.43.down_proj", "model.layers.16.mlp.experts.44.down_proj", "model.layers.16.mlp.experts.45.down_proj", "model.layers.16.mlp.experts.46.down_proj", "model.layers.16.mlp.experts.47.down_proj", "model.layers.16.mlp.experts.48.down_proj", "model.layers.16.mlp.experts.49.down_proj", "model.layers.16.mlp.experts.50.down_proj", "model.layers.16.mlp.experts.51.down_proj", "model.layers.16.mlp.experts.52.down_proj", "model.layers.16.mlp.experts.53.down_proj", "model.layers.16.mlp.experts.54.down_proj", "model.layers.16.mlp.experts.55.down_proj", "model.layers.16.mlp.experts.56.down_proj", "model.layers.16.mlp.experts.57.down_proj", "model.layers.16.mlp.experts.58.down_proj", "model.layers.16.mlp.experts.59.down_proj", "model.layers.16.mlp.experts.60.down_proj", "model.layers.16.mlp.experts.61.down_proj", "model.layers.16.mlp.experts.62.down_proj", "model.layers.16.mlp.experts.63.down_proj", "model.layers.16.mlp.experts.64.down_proj", "model.layers.16.mlp.experts.65.down_proj", "model.layers.16.mlp.experts.66.down_proj", "model.layers.16.mlp.experts.67.down_proj", "model.layers.16.mlp.experts.68.down_proj", "model.layers.16.mlp.experts.69.down_proj", "model.layers.16.mlp.experts.70.down_proj", "model.layers.16.mlp.experts.71.down_proj", "model.layers.16.mlp.experts.72.down_proj", "model.layers.16.mlp.experts.73.down_proj", "model.layers.16.mlp.experts.74.down_proj", "model.layers.16.mlp.experts.75.down_proj", "model.layers.16.mlp.experts.76.down_proj", "model.layers.16.mlp.experts.77.down_proj", "model.layers.16.mlp.experts.78.down_proj", "model.layers.16.mlp.experts.79.down_proj", "model.layers.16.mlp.experts.80.down_proj", "model.layers.16.mlp.experts.81.down_proj", "model.layers.16.mlp.experts.82.down_proj", "model.layers.16.mlp.experts.83.down_proj", "model.layers.16.mlp.experts.84.down_proj", "model.layers.16.mlp.experts.85.down_proj", "model.layers.16.mlp.experts.86.down_proj", "model.layers.16.mlp.experts.87.down_proj", "model.layers.16.mlp.experts.88.down_proj", "model.layers.16.mlp.experts.89.down_proj", "model.layers.16.mlp.experts.90.down_proj", "model.layers.16.mlp.experts.91.down_proj", "model.layers.16.mlp.experts.92.down_proj", "model.layers.16.mlp.experts.93.down_proj", "model.layers.16.mlp.experts.94.down_proj", "model.layers.16.mlp.experts.95.down_proj", "model.layers.16.mlp.experts.96.down_proj", "model.layers.16.mlp.experts.97.down_proj", "model.layers.16.mlp.experts.98.down_proj", "model.layers.16.mlp.experts.99.down_proj", "model.layers.16.mlp.experts.100.down_proj", "model.layers.16.mlp.experts.101.down_proj", "model.layers.16.mlp.experts.102.down_proj", "model.layers.16.mlp.experts.103.down_proj", "model.layers.16.mlp.experts.104.down_proj", "model.layers.16.mlp.experts.105.down_proj", "model.layers.16.mlp.experts.106.down_proj", "model.layers.16.mlp.experts.107.down_proj", "model.layers.16.mlp.experts.108.down_proj", "model.layers.16.mlp.experts.109.down_proj", "model.layers.16.mlp.experts.110.down_proj", "model.layers.16.mlp.experts.111.down_proj", "model.layers.16.mlp.experts.112.down_proj", "model.layers.16.mlp.experts.113.down_proj", "model.layers.16.mlp.experts.114.down_proj", "model.layers.16.mlp.experts.115.down_proj", "model.layers.16.mlp.experts.116.down_proj", "model.layers.16.mlp.experts.117.down_proj", "model.layers.16.mlp.experts.118.down_proj", "model.layers.16.mlp.experts.119.down_proj", "model.layers.16.mlp.experts.120.down_proj", "model.layers.16.mlp.experts.121.down_proj", "model.layers.16.mlp.experts.122.down_proj", "model.layers.16.mlp.experts.123.down_proj", "model.layers.16.mlp.experts.124.down_proj", "model.layers.16.mlp.experts.125.down_proj", "model.layers.16.mlp.experts.126.down_proj", "model.layers.16.mlp.experts.127.down_proj", "model.layers.16.mlp.experts.128.down_proj", "model.layers.16.mlp.experts.129.down_proj", "model.layers.16.mlp.experts.130.down_proj", "model.layers.16.mlp.experts.131.down_proj", "model.layers.16.mlp.experts.132.down_proj", "model.layers.16.mlp.experts.133.down_proj", "model.layers.16.mlp.experts.134.down_proj", "model.layers.16.mlp.experts.135.down_proj", "model.layers.16.mlp.experts.136.down_proj", "model.layers.16.mlp.experts.137.down_proj", "model.layers.16.mlp.experts.138.down_proj", "model.layers.16.mlp.experts.139.down_proj", "model.layers.16.mlp.experts.140.down_proj", "model.layers.16.mlp.experts.141.down_proj", "model.layers.16.mlp.experts.142.down_proj", "model.layers.16.mlp.experts.143.down_proj", "model.layers.16.mlp.experts.144.down_proj", "model.layers.16.mlp.experts.145.down_proj", "model.layers.16.mlp.experts.146.down_proj", "model.layers.16.mlp.experts.147.down_proj", "model.layers.16.mlp.experts.148.down_proj", "model.layers.16.mlp.experts.149.down_proj", "model.layers.16.mlp.experts.150.down_proj", "model.layers.16.mlp.experts.151.down_proj", "model.layers.16.mlp.experts.152.down_proj", "model.layers.16.mlp.experts.153.down_proj", "model.layers.16.mlp.experts.154.down_proj", "model.layers.16.mlp.experts.155.down_proj", "model.layers.16.mlp.experts.156.down_proj", "model.layers.16.mlp.experts.157.down_proj", "model.layers.16.mlp.experts.158.down_proj", "model.layers.16.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0021625988185405842, "dbits": 1258291200 } ] }, { "idx": 99, "layers": [ "model.layers.17.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0009582325816154258, "dbits": 62914560 } ] }, { "idx": 100, "layers": [ "model.layers.17.self_attn.k_proj", "model.layers.17.self_attn.v_proj" ], "candidates": [ { "dkld": -0.005365988612175021, "dbits": 10485760 } ] }, { "idx": 101, "layers": [ "model.layers.17.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0008056990802288166, "dbits": 62914560 } ] }, { "idx": 102, "layers": [ "model.layers.17.mlp.shared_experts.gate_proj", "model.layers.17.mlp.shared_experts.up_proj", "model.layers.17.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0015029415488242992, "dbits": 23592960 } ] }, { "idx": 103, "layers": [ "model.layers.17.mlp.experts.0.gate_proj", "model.layers.17.mlp.experts.1.gate_proj", "model.layers.17.mlp.experts.2.gate_proj", "model.layers.17.mlp.experts.3.gate_proj", "model.layers.17.mlp.experts.4.gate_proj", "model.layers.17.mlp.experts.5.gate_proj", "model.layers.17.mlp.experts.6.gate_proj", "model.layers.17.mlp.experts.7.gate_proj", "model.layers.17.mlp.experts.8.gate_proj", "model.layers.17.mlp.experts.9.gate_proj", "model.layers.17.mlp.experts.10.gate_proj", "model.layers.17.mlp.experts.11.gate_proj", "model.layers.17.mlp.experts.12.gate_proj", "model.layers.17.mlp.experts.13.gate_proj", "model.layers.17.mlp.experts.14.gate_proj", "model.layers.17.mlp.experts.15.gate_proj", "model.layers.17.mlp.experts.16.gate_proj", "model.layers.17.mlp.experts.17.gate_proj", "model.layers.17.mlp.experts.18.gate_proj", "model.layers.17.mlp.experts.19.gate_proj", "model.layers.17.mlp.experts.20.gate_proj", "model.layers.17.mlp.experts.21.gate_proj", "model.layers.17.mlp.experts.22.gate_proj", "model.layers.17.mlp.experts.23.gate_proj", "model.layers.17.mlp.experts.24.gate_proj", "model.layers.17.mlp.experts.25.gate_proj", "model.layers.17.mlp.experts.26.gate_proj", "model.layers.17.mlp.experts.27.gate_proj", "model.layers.17.mlp.experts.28.gate_proj", "model.layers.17.mlp.experts.29.gate_proj", "model.layers.17.mlp.experts.30.gate_proj", "model.layers.17.mlp.experts.31.gate_proj", "model.layers.17.mlp.experts.32.gate_proj", "model.layers.17.mlp.experts.33.gate_proj", "model.layers.17.mlp.experts.34.gate_proj", "model.layers.17.mlp.experts.35.gate_proj", "model.layers.17.mlp.experts.36.gate_proj", "model.layers.17.mlp.experts.37.gate_proj", "model.layers.17.mlp.experts.38.gate_proj", "model.layers.17.mlp.experts.39.gate_proj", "model.layers.17.mlp.experts.40.gate_proj", "model.layers.17.mlp.experts.41.gate_proj", "model.layers.17.mlp.experts.42.gate_proj", "model.layers.17.mlp.experts.43.gate_proj", "model.layers.17.mlp.experts.44.gate_proj", "model.layers.17.mlp.experts.45.gate_proj", "model.layers.17.mlp.experts.46.gate_proj", "model.layers.17.mlp.experts.47.gate_proj", "model.layers.17.mlp.experts.48.gate_proj", "model.layers.17.mlp.experts.49.gate_proj", "model.layers.17.mlp.experts.50.gate_proj", "model.layers.17.mlp.experts.51.gate_proj", "model.layers.17.mlp.experts.52.gate_proj", "model.layers.17.mlp.experts.53.gate_proj", "model.layers.17.mlp.experts.54.gate_proj", "model.layers.17.mlp.experts.55.gate_proj", "model.layers.17.mlp.experts.56.gate_proj", "model.layers.17.mlp.experts.57.gate_proj", "model.layers.17.mlp.experts.58.gate_proj", "model.layers.17.mlp.experts.59.gate_proj", "model.layers.17.mlp.experts.60.gate_proj", "model.layers.17.mlp.experts.61.gate_proj", "model.layers.17.mlp.experts.62.gate_proj", "model.layers.17.mlp.experts.63.gate_proj", "model.layers.17.mlp.experts.64.gate_proj", "model.layers.17.mlp.experts.65.gate_proj", "model.layers.17.mlp.experts.66.gate_proj", "model.layers.17.mlp.experts.67.gate_proj", "model.layers.17.mlp.experts.68.gate_proj", "model.layers.17.mlp.experts.69.gate_proj", "model.layers.17.mlp.experts.70.gate_proj", "model.layers.17.mlp.experts.71.gate_proj", "model.layers.17.mlp.experts.72.gate_proj", "model.layers.17.mlp.experts.73.gate_proj", "model.layers.17.mlp.experts.74.gate_proj", "model.layers.17.mlp.experts.75.gate_proj", "model.layers.17.mlp.experts.76.gate_proj", "model.layers.17.mlp.experts.77.gate_proj", "model.layers.17.mlp.experts.78.gate_proj", "model.layers.17.mlp.experts.79.gate_proj", "model.layers.17.mlp.experts.80.gate_proj", "model.layers.17.mlp.experts.81.gate_proj", "model.layers.17.mlp.experts.82.gate_proj", "model.layers.17.mlp.experts.83.gate_proj", "model.layers.17.mlp.experts.84.gate_proj", "model.layers.17.mlp.experts.85.gate_proj", "model.layers.17.mlp.experts.86.gate_proj", "model.layers.17.mlp.experts.87.gate_proj", "model.layers.17.mlp.experts.88.gate_proj", "model.layers.17.mlp.experts.89.gate_proj", "model.layers.17.mlp.experts.90.gate_proj", "model.layers.17.mlp.experts.91.gate_proj", "model.layers.17.mlp.experts.92.gate_proj", "model.layers.17.mlp.experts.93.gate_proj", "model.layers.17.mlp.experts.94.gate_proj", "model.layers.17.mlp.experts.95.gate_proj", "model.layers.17.mlp.experts.96.gate_proj", "model.layers.17.mlp.experts.97.gate_proj", "model.layers.17.mlp.experts.98.gate_proj", "model.layers.17.mlp.experts.99.gate_proj", "model.layers.17.mlp.experts.100.gate_proj", "model.layers.17.mlp.experts.101.gate_proj", "model.layers.17.mlp.experts.102.gate_proj", "model.layers.17.mlp.experts.103.gate_proj", "model.layers.17.mlp.experts.104.gate_proj", "model.layers.17.mlp.experts.105.gate_proj", "model.layers.17.mlp.experts.106.gate_proj", "model.layers.17.mlp.experts.107.gate_proj", "model.layers.17.mlp.experts.108.gate_proj", "model.layers.17.mlp.experts.109.gate_proj", "model.layers.17.mlp.experts.110.gate_proj", "model.layers.17.mlp.experts.111.gate_proj", "model.layers.17.mlp.experts.112.gate_proj", "model.layers.17.mlp.experts.113.gate_proj", "model.layers.17.mlp.experts.114.gate_proj", "model.layers.17.mlp.experts.115.gate_proj", "model.layers.17.mlp.experts.116.gate_proj", "model.layers.17.mlp.experts.117.gate_proj", "model.layers.17.mlp.experts.118.gate_proj", "model.layers.17.mlp.experts.119.gate_proj", "model.layers.17.mlp.experts.120.gate_proj", "model.layers.17.mlp.experts.121.gate_proj", "model.layers.17.mlp.experts.122.gate_proj", "model.layers.17.mlp.experts.123.gate_proj", "model.layers.17.mlp.experts.124.gate_proj", "model.layers.17.mlp.experts.125.gate_proj", "model.layers.17.mlp.experts.126.gate_proj", "model.layers.17.mlp.experts.127.gate_proj", "model.layers.17.mlp.experts.128.gate_proj", "model.layers.17.mlp.experts.129.gate_proj", "model.layers.17.mlp.experts.130.gate_proj", "model.layers.17.mlp.experts.131.gate_proj", "model.layers.17.mlp.experts.132.gate_proj", "model.layers.17.mlp.experts.133.gate_proj", "model.layers.17.mlp.experts.134.gate_proj", "model.layers.17.mlp.experts.135.gate_proj", "model.layers.17.mlp.experts.136.gate_proj", "model.layers.17.mlp.experts.137.gate_proj", "model.layers.17.mlp.experts.138.gate_proj", "model.layers.17.mlp.experts.139.gate_proj", "model.layers.17.mlp.experts.140.gate_proj", "model.layers.17.mlp.experts.141.gate_proj", "model.layers.17.mlp.experts.142.gate_proj", "model.layers.17.mlp.experts.143.gate_proj", "model.layers.17.mlp.experts.144.gate_proj", "model.layers.17.mlp.experts.145.gate_proj", "model.layers.17.mlp.experts.146.gate_proj", "model.layers.17.mlp.experts.147.gate_proj", "model.layers.17.mlp.experts.148.gate_proj", "model.layers.17.mlp.experts.149.gate_proj", "model.layers.17.mlp.experts.150.gate_proj", "model.layers.17.mlp.experts.151.gate_proj", "model.layers.17.mlp.experts.152.gate_proj", "model.layers.17.mlp.experts.153.gate_proj", "model.layers.17.mlp.experts.154.gate_proj", "model.layers.17.mlp.experts.155.gate_proj", "model.layers.17.mlp.experts.156.gate_proj", "model.layers.17.mlp.experts.157.gate_proj", "model.layers.17.mlp.experts.158.gate_proj", "model.layers.17.mlp.experts.159.gate_proj", "model.layers.17.mlp.experts.0.up_proj", "model.layers.17.mlp.experts.1.up_proj", "model.layers.17.mlp.experts.2.up_proj", "model.layers.17.mlp.experts.3.up_proj", "model.layers.17.mlp.experts.4.up_proj", "model.layers.17.mlp.experts.5.up_proj", "model.layers.17.mlp.experts.6.up_proj", "model.layers.17.mlp.experts.7.up_proj", "model.layers.17.mlp.experts.8.up_proj", "model.layers.17.mlp.experts.9.up_proj", "model.layers.17.mlp.experts.10.up_proj", "model.layers.17.mlp.experts.11.up_proj", "model.layers.17.mlp.experts.12.up_proj", "model.layers.17.mlp.experts.13.up_proj", "model.layers.17.mlp.experts.14.up_proj", "model.layers.17.mlp.experts.15.up_proj", "model.layers.17.mlp.experts.16.up_proj", "model.layers.17.mlp.experts.17.up_proj", "model.layers.17.mlp.experts.18.up_proj", "model.layers.17.mlp.experts.19.up_proj", "model.layers.17.mlp.experts.20.up_proj", "model.layers.17.mlp.experts.21.up_proj", "model.layers.17.mlp.experts.22.up_proj", "model.layers.17.mlp.experts.23.up_proj", "model.layers.17.mlp.experts.24.up_proj", "model.layers.17.mlp.experts.25.up_proj", "model.layers.17.mlp.experts.26.up_proj", "model.layers.17.mlp.experts.27.up_proj", "model.layers.17.mlp.experts.28.up_proj", "model.layers.17.mlp.experts.29.up_proj", "model.layers.17.mlp.experts.30.up_proj", "model.layers.17.mlp.experts.31.up_proj", "model.layers.17.mlp.experts.32.up_proj", "model.layers.17.mlp.experts.33.up_proj", "model.layers.17.mlp.experts.34.up_proj", "model.layers.17.mlp.experts.35.up_proj", "model.layers.17.mlp.experts.36.up_proj", "model.layers.17.mlp.experts.37.up_proj", "model.layers.17.mlp.experts.38.up_proj", "model.layers.17.mlp.experts.39.up_proj", "model.layers.17.mlp.experts.40.up_proj", "model.layers.17.mlp.experts.41.up_proj", "model.layers.17.mlp.experts.42.up_proj", "model.layers.17.mlp.experts.43.up_proj", "model.layers.17.mlp.experts.44.up_proj", "model.layers.17.mlp.experts.45.up_proj", "model.layers.17.mlp.experts.46.up_proj", "model.layers.17.mlp.experts.47.up_proj", "model.layers.17.mlp.experts.48.up_proj", "model.layers.17.mlp.experts.49.up_proj", "model.layers.17.mlp.experts.50.up_proj", "model.layers.17.mlp.experts.51.up_proj", "model.layers.17.mlp.experts.52.up_proj", "model.layers.17.mlp.experts.53.up_proj", "model.layers.17.mlp.experts.54.up_proj", "model.layers.17.mlp.experts.55.up_proj", "model.layers.17.mlp.experts.56.up_proj", "model.layers.17.mlp.experts.57.up_proj", "model.layers.17.mlp.experts.58.up_proj", "model.layers.17.mlp.experts.59.up_proj", "model.layers.17.mlp.experts.60.up_proj", "model.layers.17.mlp.experts.61.up_proj", "model.layers.17.mlp.experts.62.up_proj", "model.layers.17.mlp.experts.63.up_proj", "model.layers.17.mlp.experts.64.up_proj", "model.layers.17.mlp.experts.65.up_proj", "model.layers.17.mlp.experts.66.up_proj", "model.layers.17.mlp.experts.67.up_proj", "model.layers.17.mlp.experts.68.up_proj", "model.layers.17.mlp.experts.69.up_proj", "model.layers.17.mlp.experts.70.up_proj", "model.layers.17.mlp.experts.71.up_proj", "model.layers.17.mlp.experts.72.up_proj", "model.layers.17.mlp.experts.73.up_proj", "model.layers.17.mlp.experts.74.up_proj", "model.layers.17.mlp.experts.75.up_proj", "model.layers.17.mlp.experts.76.up_proj", "model.layers.17.mlp.experts.77.up_proj", "model.layers.17.mlp.experts.78.up_proj", "model.layers.17.mlp.experts.79.up_proj", "model.layers.17.mlp.experts.80.up_proj", "model.layers.17.mlp.experts.81.up_proj", "model.layers.17.mlp.experts.82.up_proj", "model.layers.17.mlp.experts.83.up_proj", "model.layers.17.mlp.experts.84.up_proj", "model.layers.17.mlp.experts.85.up_proj", "model.layers.17.mlp.experts.86.up_proj", "model.layers.17.mlp.experts.87.up_proj", "model.layers.17.mlp.experts.88.up_proj", "model.layers.17.mlp.experts.89.up_proj", "model.layers.17.mlp.experts.90.up_proj", "model.layers.17.mlp.experts.91.up_proj", "model.layers.17.mlp.experts.92.up_proj", "model.layers.17.mlp.experts.93.up_proj", "model.layers.17.mlp.experts.94.up_proj", "model.layers.17.mlp.experts.95.up_proj", "model.layers.17.mlp.experts.96.up_proj", "model.layers.17.mlp.experts.97.up_proj", "model.layers.17.mlp.experts.98.up_proj", "model.layers.17.mlp.experts.99.up_proj", "model.layers.17.mlp.experts.100.up_proj", "model.layers.17.mlp.experts.101.up_proj", "model.layers.17.mlp.experts.102.up_proj", "model.layers.17.mlp.experts.103.up_proj", "model.layers.17.mlp.experts.104.up_proj", "model.layers.17.mlp.experts.105.up_proj", "model.layers.17.mlp.experts.106.up_proj", "model.layers.17.mlp.experts.107.up_proj", "model.layers.17.mlp.experts.108.up_proj", "model.layers.17.mlp.experts.109.up_proj", "model.layers.17.mlp.experts.110.up_proj", "model.layers.17.mlp.experts.111.up_proj", "model.layers.17.mlp.experts.112.up_proj", "model.layers.17.mlp.experts.113.up_proj", "model.layers.17.mlp.experts.114.up_proj", "model.layers.17.mlp.experts.115.up_proj", "model.layers.17.mlp.experts.116.up_proj", "model.layers.17.mlp.experts.117.up_proj", "model.layers.17.mlp.experts.118.up_proj", "model.layers.17.mlp.experts.119.up_proj", "model.layers.17.mlp.experts.120.up_proj", "model.layers.17.mlp.experts.121.up_proj", "model.layers.17.mlp.experts.122.up_proj", "model.layers.17.mlp.experts.123.up_proj", "model.layers.17.mlp.experts.124.up_proj", "model.layers.17.mlp.experts.125.up_proj", "model.layers.17.mlp.experts.126.up_proj", "model.layers.17.mlp.experts.127.up_proj", "model.layers.17.mlp.experts.128.up_proj", "model.layers.17.mlp.experts.129.up_proj", "model.layers.17.mlp.experts.130.up_proj", "model.layers.17.mlp.experts.131.up_proj", "model.layers.17.mlp.experts.132.up_proj", "model.layers.17.mlp.experts.133.up_proj", "model.layers.17.mlp.experts.134.up_proj", "model.layers.17.mlp.experts.135.up_proj", "model.layers.17.mlp.experts.136.up_proj", "model.layers.17.mlp.experts.137.up_proj", "model.layers.17.mlp.experts.138.up_proj", "model.layers.17.mlp.experts.139.up_proj", "model.layers.17.mlp.experts.140.up_proj", "model.layers.17.mlp.experts.141.up_proj", "model.layers.17.mlp.experts.142.up_proj", "model.layers.17.mlp.experts.143.up_proj", "model.layers.17.mlp.experts.144.up_proj", "model.layers.17.mlp.experts.145.up_proj", "model.layers.17.mlp.experts.146.up_proj", "model.layers.17.mlp.experts.147.up_proj", "model.layers.17.mlp.experts.148.up_proj", "model.layers.17.mlp.experts.149.up_proj", "model.layers.17.mlp.experts.150.up_proj", "model.layers.17.mlp.experts.151.up_proj", "model.layers.17.mlp.experts.152.up_proj", "model.layers.17.mlp.experts.153.up_proj", "model.layers.17.mlp.experts.154.up_proj", "model.layers.17.mlp.experts.155.up_proj", "model.layers.17.mlp.experts.156.up_proj", "model.layers.17.mlp.experts.157.up_proj", "model.layers.17.mlp.experts.158.up_proj", "model.layers.17.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0029378540813922993, "dbits": 2516582400 } ] }, { "idx": 104, "layers": [ "model.layers.17.mlp.experts.0.down_proj", "model.layers.17.mlp.experts.1.down_proj", "model.layers.17.mlp.experts.2.down_proj", "model.layers.17.mlp.experts.3.down_proj", "model.layers.17.mlp.experts.4.down_proj", "model.layers.17.mlp.experts.5.down_proj", "model.layers.17.mlp.experts.6.down_proj", "model.layers.17.mlp.experts.7.down_proj", "model.layers.17.mlp.experts.8.down_proj", "model.layers.17.mlp.experts.9.down_proj", "model.layers.17.mlp.experts.10.down_proj", "model.layers.17.mlp.experts.11.down_proj", "model.layers.17.mlp.experts.12.down_proj", "model.layers.17.mlp.experts.13.down_proj", "model.layers.17.mlp.experts.14.down_proj", "model.layers.17.mlp.experts.15.down_proj", "model.layers.17.mlp.experts.16.down_proj", "model.layers.17.mlp.experts.17.down_proj", "model.layers.17.mlp.experts.18.down_proj", "model.layers.17.mlp.experts.19.down_proj", "model.layers.17.mlp.experts.20.down_proj", "model.layers.17.mlp.experts.21.down_proj", "model.layers.17.mlp.experts.22.down_proj", "model.layers.17.mlp.experts.23.down_proj", "model.layers.17.mlp.experts.24.down_proj", "model.layers.17.mlp.experts.25.down_proj", "model.layers.17.mlp.experts.26.down_proj", "model.layers.17.mlp.experts.27.down_proj", "model.layers.17.mlp.experts.28.down_proj", "model.layers.17.mlp.experts.29.down_proj", "model.layers.17.mlp.experts.30.down_proj", "model.layers.17.mlp.experts.31.down_proj", "model.layers.17.mlp.experts.32.down_proj", "model.layers.17.mlp.experts.33.down_proj", "model.layers.17.mlp.experts.34.down_proj", "model.layers.17.mlp.experts.35.down_proj", "model.layers.17.mlp.experts.36.down_proj", "model.layers.17.mlp.experts.37.down_proj", "model.layers.17.mlp.experts.38.down_proj", "model.layers.17.mlp.experts.39.down_proj", "model.layers.17.mlp.experts.40.down_proj", "model.layers.17.mlp.experts.41.down_proj", "model.layers.17.mlp.experts.42.down_proj", "model.layers.17.mlp.experts.43.down_proj", "model.layers.17.mlp.experts.44.down_proj", "model.layers.17.mlp.experts.45.down_proj", "model.layers.17.mlp.experts.46.down_proj", "model.layers.17.mlp.experts.47.down_proj", "model.layers.17.mlp.experts.48.down_proj", "model.layers.17.mlp.experts.49.down_proj", "model.layers.17.mlp.experts.50.down_proj", "model.layers.17.mlp.experts.51.down_proj", "model.layers.17.mlp.experts.52.down_proj", "model.layers.17.mlp.experts.53.down_proj", "model.layers.17.mlp.experts.54.down_proj", "model.layers.17.mlp.experts.55.down_proj", "model.layers.17.mlp.experts.56.down_proj", "model.layers.17.mlp.experts.57.down_proj", "model.layers.17.mlp.experts.58.down_proj", "model.layers.17.mlp.experts.59.down_proj", "model.layers.17.mlp.experts.60.down_proj", "model.layers.17.mlp.experts.61.down_proj", "model.layers.17.mlp.experts.62.down_proj", "model.layers.17.mlp.experts.63.down_proj", "model.layers.17.mlp.experts.64.down_proj", "model.layers.17.mlp.experts.65.down_proj", "model.layers.17.mlp.experts.66.down_proj", "model.layers.17.mlp.experts.67.down_proj", "model.layers.17.mlp.experts.68.down_proj", "model.layers.17.mlp.experts.69.down_proj", "model.layers.17.mlp.experts.70.down_proj", "model.layers.17.mlp.experts.71.down_proj", "model.layers.17.mlp.experts.72.down_proj", "model.layers.17.mlp.experts.73.down_proj", "model.layers.17.mlp.experts.74.down_proj", "model.layers.17.mlp.experts.75.down_proj", "model.layers.17.mlp.experts.76.down_proj", "model.layers.17.mlp.experts.77.down_proj", "model.layers.17.mlp.experts.78.down_proj", "model.layers.17.mlp.experts.79.down_proj", "model.layers.17.mlp.experts.80.down_proj", "model.layers.17.mlp.experts.81.down_proj", "model.layers.17.mlp.experts.82.down_proj", "model.layers.17.mlp.experts.83.down_proj", "model.layers.17.mlp.experts.84.down_proj", "model.layers.17.mlp.experts.85.down_proj", "model.layers.17.mlp.experts.86.down_proj", "model.layers.17.mlp.experts.87.down_proj", "model.layers.17.mlp.experts.88.down_proj", "model.layers.17.mlp.experts.89.down_proj", "model.layers.17.mlp.experts.90.down_proj", "model.layers.17.mlp.experts.91.down_proj", "model.layers.17.mlp.experts.92.down_proj", "model.layers.17.mlp.experts.93.down_proj", "model.layers.17.mlp.experts.94.down_proj", "model.layers.17.mlp.experts.95.down_proj", "model.layers.17.mlp.experts.96.down_proj", "model.layers.17.mlp.experts.97.down_proj", "model.layers.17.mlp.experts.98.down_proj", "model.layers.17.mlp.experts.99.down_proj", "model.layers.17.mlp.experts.100.down_proj", "model.layers.17.mlp.experts.101.down_proj", "model.layers.17.mlp.experts.102.down_proj", "model.layers.17.mlp.experts.103.down_proj", "model.layers.17.mlp.experts.104.down_proj", "model.layers.17.mlp.experts.105.down_proj", "model.layers.17.mlp.experts.106.down_proj", "model.layers.17.mlp.experts.107.down_proj", "model.layers.17.mlp.experts.108.down_proj", "model.layers.17.mlp.experts.109.down_proj", "model.layers.17.mlp.experts.110.down_proj", "model.layers.17.mlp.experts.111.down_proj", "model.layers.17.mlp.experts.112.down_proj", "model.layers.17.mlp.experts.113.down_proj", "model.layers.17.mlp.experts.114.down_proj", "model.layers.17.mlp.experts.115.down_proj", "model.layers.17.mlp.experts.116.down_proj", "model.layers.17.mlp.experts.117.down_proj", "model.layers.17.mlp.experts.118.down_proj", "model.layers.17.mlp.experts.119.down_proj", "model.layers.17.mlp.experts.120.down_proj", "model.layers.17.mlp.experts.121.down_proj", "model.layers.17.mlp.experts.122.down_proj", "model.layers.17.mlp.experts.123.down_proj", "model.layers.17.mlp.experts.124.down_proj", "model.layers.17.mlp.experts.125.down_proj", "model.layers.17.mlp.experts.126.down_proj", "model.layers.17.mlp.experts.127.down_proj", "model.layers.17.mlp.experts.128.down_proj", "model.layers.17.mlp.experts.129.down_proj", "model.layers.17.mlp.experts.130.down_proj", "model.layers.17.mlp.experts.131.down_proj", "model.layers.17.mlp.experts.132.down_proj", "model.layers.17.mlp.experts.133.down_proj", "model.layers.17.mlp.experts.134.down_proj", "model.layers.17.mlp.experts.135.down_proj", "model.layers.17.mlp.experts.136.down_proj", "model.layers.17.mlp.experts.137.down_proj", "model.layers.17.mlp.experts.138.down_proj", "model.layers.17.mlp.experts.139.down_proj", "model.layers.17.mlp.experts.140.down_proj", "model.layers.17.mlp.experts.141.down_proj", "model.layers.17.mlp.experts.142.down_proj", "model.layers.17.mlp.experts.143.down_proj", "model.layers.17.mlp.experts.144.down_proj", "model.layers.17.mlp.experts.145.down_proj", "model.layers.17.mlp.experts.146.down_proj", "model.layers.17.mlp.experts.147.down_proj", "model.layers.17.mlp.experts.148.down_proj", "model.layers.17.mlp.experts.149.down_proj", "model.layers.17.mlp.experts.150.down_proj", "model.layers.17.mlp.experts.151.down_proj", "model.layers.17.mlp.experts.152.down_proj", "model.layers.17.mlp.experts.153.down_proj", "model.layers.17.mlp.experts.154.down_proj", "model.layers.17.mlp.experts.155.down_proj", "model.layers.17.mlp.experts.156.down_proj", "model.layers.17.mlp.experts.157.down_proj", "model.layers.17.mlp.experts.158.down_proj", "model.layers.17.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002970235794782672, "dbits": 1258291200 } ] }, { "idx": 105, "layers": [ "model.layers.18.self_attn.q_proj" ], "candidates": [ { "dkld": -0.000942894071340572, "dbits": 62914560 } ] }, { "idx": 106, "layers": [ "model.layers.18.self_attn.k_proj", "model.layers.18.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00257141813635825, "dbits": 10485760 } ] }, { "idx": 107, "layers": [ "model.layers.18.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0013110980391502491, "dbits": 62914560 } ] }, { "idx": 108, "layers": [ "model.layers.18.mlp.shared_experts.gate_proj", "model.layers.18.mlp.shared_experts.up_proj", "model.layers.18.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0005784839391708707, "dbits": 23592960 } ] }, { "idx": 109, "layers": [ "model.layers.18.mlp.experts.0.gate_proj", "model.layers.18.mlp.experts.1.gate_proj", "model.layers.18.mlp.experts.2.gate_proj", "model.layers.18.mlp.experts.3.gate_proj", "model.layers.18.mlp.experts.4.gate_proj", "model.layers.18.mlp.experts.5.gate_proj", "model.layers.18.mlp.experts.6.gate_proj", "model.layers.18.mlp.experts.7.gate_proj", "model.layers.18.mlp.experts.8.gate_proj", "model.layers.18.mlp.experts.9.gate_proj", "model.layers.18.mlp.experts.10.gate_proj", "model.layers.18.mlp.experts.11.gate_proj", "model.layers.18.mlp.experts.12.gate_proj", "model.layers.18.mlp.experts.13.gate_proj", "model.layers.18.mlp.experts.14.gate_proj", "model.layers.18.mlp.experts.15.gate_proj", "model.layers.18.mlp.experts.16.gate_proj", "model.layers.18.mlp.experts.17.gate_proj", "model.layers.18.mlp.experts.18.gate_proj", "model.layers.18.mlp.experts.19.gate_proj", "model.layers.18.mlp.experts.20.gate_proj", "model.layers.18.mlp.experts.21.gate_proj", "model.layers.18.mlp.experts.22.gate_proj", "model.layers.18.mlp.experts.23.gate_proj", "model.layers.18.mlp.experts.24.gate_proj", "model.layers.18.mlp.experts.25.gate_proj", "model.layers.18.mlp.experts.26.gate_proj", "model.layers.18.mlp.experts.27.gate_proj", "model.layers.18.mlp.experts.28.gate_proj", "model.layers.18.mlp.experts.29.gate_proj", "model.layers.18.mlp.experts.30.gate_proj", "model.layers.18.mlp.experts.31.gate_proj", "model.layers.18.mlp.experts.32.gate_proj", "model.layers.18.mlp.experts.33.gate_proj", "model.layers.18.mlp.experts.34.gate_proj", "model.layers.18.mlp.experts.35.gate_proj", "model.layers.18.mlp.experts.36.gate_proj", "model.layers.18.mlp.experts.37.gate_proj", "model.layers.18.mlp.experts.38.gate_proj", "model.layers.18.mlp.experts.39.gate_proj", "model.layers.18.mlp.experts.40.gate_proj", "model.layers.18.mlp.experts.41.gate_proj", "model.layers.18.mlp.experts.42.gate_proj", "model.layers.18.mlp.experts.43.gate_proj", "model.layers.18.mlp.experts.44.gate_proj", "model.layers.18.mlp.experts.45.gate_proj", "model.layers.18.mlp.experts.46.gate_proj", "model.layers.18.mlp.experts.47.gate_proj", "model.layers.18.mlp.experts.48.gate_proj", "model.layers.18.mlp.experts.49.gate_proj", "model.layers.18.mlp.experts.50.gate_proj", "model.layers.18.mlp.experts.51.gate_proj", "model.layers.18.mlp.experts.52.gate_proj", "model.layers.18.mlp.experts.53.gate_proj", "model.layers.18.mlp.experts.54.gate_proj", "model.layers.18.mlp.experts.55.gate_proj", "model.layers.18.mlp.experts.56.gate_proj", "model.layers.18.mlp.experts.57.gate_proj", "model.layers.18.mlp.experts.58.gate_proj", "model.layers.18.mlp.experts.59.gate_proj", "model.layers.18.mlp.experts.60.gate_proj", "model.layers.18.mlp.experts.61.gate_proj", "model.layers.18.mlp.experts.62.gate_proj", "model.layers.18.mlp.experts.63.gate_proj", "model.layers.18.mlp.experts.64.gate_proj", "model.layers.18.mlp.experts.65.gate_proj", "model.layers.18.mlp.experts.66.gate_proj", "model.layers.18.mlp.experts.67.gate_proj", "model.layers.18.mlp.experts.68.gate_proj", "model.layers.18.mlp.experts.69.gate_proj", "model.layers.18.mlp.experts.70.gate_proj", "model.layers.18.mlp.experts.71.gate_proj", "model.layers.18.mlp.experts.72.gate_proj", "model.layers.18.mlp.experts.73.gate_proj", "model.layers.18.mlp.experts.74.gate_proj", "model.layers.18.mlp.experts.75.gate_proj", "model.layers.18.mlp.experts.76.gate_proj", "model.layers.18.mlp.experts.77.gate_proj", "model.layers.18.mlp.experts.78.gate_proj", "model.layers.18.mlp.experts.79.gate_proj", "model.layers.18.mlp.experts.80.gate_proj", "model.layers.18.mlp.experts.81.gate_proj", "model.layers.18.mlp.experts.82.gate_proj", "model.layers.18.mlp.experts.83.gate_proj", "model.layers.18.mlp.experts.84.gate_proj", "model.layers.18.mlp.experts.85.gate_proj", "model.layers.18.mlp.experts.86.gate_proj", "model.layers.18.mlp.experts.87.gate_proj", "model.layers.18.mlp.experts.88.gate_proj", "model.layers.18.mlp.experts.89.gate_proj", "model.layers.18.mlp.experts.90.gate_proj", "model.layers.18.mlp.experts.91.gate_proj", "model.layers.18.mlp.experts.92.gate_proj", "model.layers.18.mlp.experts.93.gate_proj", "model.layers.18.mlp.experts.94.gate_proj", "model.layers.18.mlp.experts.95.gate_proj", "model.layers.18.mlp.experts.96.gate_proj", "model.layers.18.mlp.experts.97.gate_proj", "model.layers.18.mlp.experts.98.gate_proj", "model.layers.18.mlp.experts.99.gate_proj", "model.layers.18.mlp.experts.100.gate_proj", "model.layers.18.mlp.experts.101.gate_proj", "model.layers.18.mlp.experts.102.gate_proj", "model.layers.18.mlp.experts.103.gate_proj", "model.layers.18.mlp.experts.104.gate_proj", "model.layers.18.mlp.experts.105.gate_proj", "model.layers.18.mlp.experts.106.gate_proj", "model.layers.18.mlp.experts.107.gate_proj", "model.layers.18.mlp.experts.108.gate_proj", "model.layers.18.mlp.experts.109.gate_proj", "model.layers.18.mlp.experts.110.gate_proj", "model.layers.18.mlp.experts.111.gate_proj", "model.layers.18.mlp.experts.112.gate_proj", "model.layers.18.mlp.experts.113.gate_proj", "model.layers.18.mlp.experts.114.gate_proj", "model.layers.18.mlp.experts.115.gate_proj", "model.layers.18.mlp.experts.116.gate_proj", "model.layers.18.mlp.experts.117.gate_proj", "model.layers.18.mlp.experts.118.gate_proj", "model.layers.18.mlp.experts.119.gate_proj", "model.layers.18.mlp.experts.120.gate_proj", "model.layers.18.mlp.experts.121.gate_proj", "model.layers.18.mlp.experts.122.gate_proj", "model.layers.18.mlp.experts.123.gate_proj", "model.layers.18.mlp.experts.124.gate_proj", "model.layers.18.mlp.experts.125.gate_proj", "model.layers.18.mlp.experts.126.gate_proj", "model.layers.18.mlp.experts.127.gate_proj", "model.layers.18.mlp.experts.128.gate_proj", "model.layers.18.mlp.experts.129.gate_proj", "model.layers.18.mlp.experts.130.gate_proj", "model.layers.18.mlp.experts.131.gate_proj", "model.layers.18.mlp.experts.132.gate_proj", "model.layers.18.mlp.experts.133.gate_proj", "model.layers.18.mlp.experts.134.gate_proj", "model.layers.18.mlp.experts.135.gate_proj", "model.layers.18.mlp.experts.136.gate_proj", "model.layers.18.mlp.experts.137.gate_proj", "model.layers.18.mlp.experts.138.gate_proj", "model.layers.18.mlp.experts.139.gate_proj", "model.layers.18.mlp.experts.140.gate_proj", "model.layers.18.mlp.experts.141.gate_proj", "model.layers.18.mlp.experts.142.gate_proj", "model.layers.18.mlp.experts.143.gate_proj", "model.layers.18.mlp.experts.144.gate_proj", "model.layers.18.mlp.experts.145.gate_proj", "model.layers.18.mlp.experts.146.gate_proj", "model.layers.18.mlp.experts.147.gate_proj", "model.layers.18.mlp.experts.148.gate_proj", "model.layers.18.mlp.experts.149.gate_proj", "model.layers.18.mlp.experts.150.gate_proj", "model.layers.18.mlp.experts.151.gate_proj", "model.layers.18.mlp.experts.152.gate_proj", "model.layers.18.mlp.experts.153.gate_proj", "model.layers.18.mlp.experts.154.gate_proj", "model.layers.18.mlp.experts.155.gate_proj", "model.layers.18.mlp.experts.156.gate_proj", "model.layers.18.mlp.experts.157.gate_proj", "model.layers.18.mlp.experts.158.gate_proj", "model.layers.18.mlp.experts.159.gate_proj", "model.layers.18.mlp.experts.0.up_proj", "model.layers.18.mlp.experts.1.up_proj", "model.layers.18.mlp.experts.2.up_proj", "model.layers.18.mlp.experts.3.up_proj", "model.layers.18.mlp.experts.4.up_proj", "model.layers.18.mlp.experts.5.up_proj", "model.layers.18.mlp.experts.6.up_proj", "model.layers.18.mlp.experts.7.up_proj", "model.layers.18.mlp.experts.8.up_proj", "model.layers.18.mlp.experts.9.up_proj", "model.layers.18.mlp.experts.10.up_proj", "model.layers.18.mlp.experts.11.up_proj", "model.layers.18.mlp.experts.12.up_proj", "model.layers.18.mlp.experts.13.up_proj", "model.layers.18.mlp.experts.14.up_proj", "model.layers.18.mlp.experts.15.up_proj", "model.layers.18.mlp.experts.16.up_proj", "model.layers.18.mlp.experts.17.up_proj", "model.layers.18.mlp.experts.18.up_proj", "model.layers.18.mlp.experts.19.up_proj", "model.layers.18.mlp.experts.20.up_proj", "model.layers.18.mlp.experts.21.up_proj", "model.layers.18.mlp.experts.22.up_proj", "model.layers.18.mlp.experts.23.up_proj", "model.layers.18.mlp.experts.24.up_proj", "model.layers.18.mlp.experts.25.up_proj", "model.layers.18.mlp.experts.26.up_proj", "model.layers.18.mlp.experts.27.up_proj", "model.layers.18.mlp.experts.28.up_proj", "model.layers.18.mlp.experts.29.up_proj", "model.layers.18.mlp.experts.30.up_proj", "model.layers.18.mlp.experts.31.up_proj", "model.layers.18.mlp.experts.32.up_proj", "model.layers.18.mlp.experts.33.up_proj", "model.layers.18.mlp.experts.34.up_proj", "model.layers.18.mlp.experts.35.up_proj", "model.layers.18.mlp.experts.36.up_proj", "model.layers.18.mlp.experts.37.up_proj", "model.layers.18.mlp.experts.38.up_proj", "model.layers.18.mlp.experts.39.up_proj", "model.layers.18.mlp.experts.40.up_proj", "model.layers.18.mlp.experts.41.up_proj", "model.layers.18.mlp.experts.42.up_proj", "model.layers.18.mlp.experts.43.up_proj", "model.layers.18.mlp.experts.44.up_proj", "model.layers.18.mlp.experts.45.up_proj", "model.layers.18.mlp.experts.46.up_proj", "model.layers.18.mlp.experts.47.up_proj", "model.layers.18.mlp.experts.48.up_proj", "model.layers.18.mlp.experts.49.up_proj", "model.layers.18.mlp.experts.50.up_proj", "model.layers.18.mlp.experts.51.up_proj", "model.layers.18.mlp.experts.52.up_proj", "model.layers.18.mlp.experts.53.up_proj", "model.layers.18.mlp.experts.54.up_proj", "model.layers.18.mlp.experts.55.up_proj", "model.layers.18.mlp.experts.56.up_proj", "model.layers.18.mlp.experts.57.up_proj", "model.layers.18.mlp.experts.58.up_proj", "model.layers.18.mlp.experts.59.up_proj", "model.layers.18.mlp.experts.60.up_proj", "model.layers.18.mlp.experts.61.up_proj", "model.layers.18.mlp.experts.62.up_proj", "model.layers.18.mlp.experts.63.up_proj", "model.layers.18.mlp.experts.64.up_proj", "model.layers.18.mlp.experts.65.up_proj", "model.layers.18.mlp.experts.66.up_proj", "model.layers.18.mlp.experts.67.up_proj", "model.layers.18.mlp.experts.68.up_proj", "model.layers.18.mlp.experts.69.up_proj", "model.layers.18.mlp.experts.70.up_proj", "model.layers.18.mlp.experts.71.up_proj", "model.layers.18.mlp.experts.72.up_proj", "model.layers.18.mlp.experts.73.up_proj", "model.layers.18.mlp.experts.74.up_proj", "model.layers.18.mlp.experts.75.up_proj", "model.layers.18.mlp.experts.76.up_proj", "model.layers.18.mlp.experts.77.up_proj", "model.layers.18.mlp.experts.78.up_proj", "model.layers.18.mlp.experts.79.up_proj", "model.layers.18.mlp.experts.80.up_proj", "model.layers.18.mlp.experts.81.up_proj", "model.layers.18.mlp.experts.82.up_proj", "model.layers.18.mlp.experts.83.up_proj", "model.layers.18.mlp.experts.84.up_proj", "model.layers.18.mlp.experts.85.up_proj", "model.layers.18.mlp.experts.86.up_proj", "model.layers.18.mlp.experts.87.up_proj", "model.layers.18.mlp.experts.88.up_proj", "model.layers.18.mlp.experts.89.up_proj", "model.layers.18.mlp.experts.90.up_proj", "model.layers.18.mlp.experts.91.up_proj", "model.layers.18.mlp.experts.92.up_proj", "model.layers.18.mlp.experts.93.up_proj", "model.layers.18.mlp.experts.94.up_proj", "model.layers.18.mlp.experts.95.up_proj", "model.layers.18.mlp.experts.96.up_proj", "model.layers.18.mlp.experts.97.up_proj", "model.layers.18.mlp.experts.98.up_proj", "model.layers.18.mlp.experts.99.up_proj", "model.layers.18.mlp.experts.100.up_proj", "model.layers.18.mlp.experts.101.up_proj", "model.layers.18.mlp.experts.102.up_proj", "model.layers.18.mlp.experts.103.up_proj", "model.layers.18.mlp.experts.104.up_proj", "model.layers.18.mlp.experts.105.up_proj", "model.layers.18.mlp.experts.106.up_proj", "model.layers.18.mlp.experts.107.up_proj", "model.layers.18.mlp.experts.108.up_proj", "model.layers.18.mlp.experts.109.up_proj", "model.layers.18.mlp.experts.110.up_proj", "model.layers.18.mlp.experts.111.up_proj", "model.layers.18.mlp.experts.112.up_proj", "model.layers.18.mlp.experts.113.up_proj", "model.layers.18.mlp.experts.114.up_proj", "model.layers.18.mlp.experts.115.up_proj", "model.layers.18.mlp.experts.116.up_proj", "model.layers.18.mlp.experts.117.up_proj", "model.layers.18.mlp.experts.118.up_proj", "model.layers.18.mlp.experts.119.up_proj", "model.layers.18.mlp.experts.120.up_proj", "model.layers.18.mlp.experts.121.up_proj", "model.layers.18.mlp.experts.122.up_proj", "model.layers.18.mlp.experts.123.up_proj", "model.layers.18.mlp.experts.124.up_proj", "model.layers.18.mlp.experts.125.up_proj", "model.layers.18.mlp.experts.126.up_proj", "model.layers.18.mlp.experts.127.up_proj", "model.layers.18.mlp.experts.128.up_proj", "model.layers.18.mlp.experts.129.up_proj", "model.layers.18.mlp.experts.130.up_proj", "model.layers.18.mlp.experts.131.up_proj", "model.layers.18.mlp.experts.132.up_proj", "model.layers.18.mlp.experts.133.up_proj", "model.layers.18.mlp.experts.134.up_proj", "model.layers.18.mlp.experts.135.up_proj", "model.layers.18.mlp.experts.136.up_proj", "model.layers.18.mlp.experts.137.up_proj", "model.layers.18.mlp.experts.138.up_proj", "model.layers.18.mlp.experts.139.up_proj", "model.layers.18.mlp.experts.140.up_proj", "model.layers.18.mlp.experts.141.up_proj", "model.layers.18.mlp.experts.142.up_proj", "model.layers.18.mlp.experts.143.up_proj", "model.layers.18.mlp.experts.144.up_proj", "model.layers.18.mlp.experts.145.up_proj", "model.layers.18.mlp.experts.146.up_proj", "model.layers.18.mlp.experts.147.up_proj", "model.layers.18.mlp.experts.148.up_proj", "model.layers.18.mlp.experts.149.up_proj", "model.layers.18.mlp.experts.150.up_proj", "model.layers.18.mlp.experts.151.up_proj", "model.layers.18.mlp.experts.152.up_proj", "model.layers.18.mlp.experts.153.up_proj", "model.layers.18.mlp.experts.154.up_proj", "model.layers.18.mlp.experts.155.up_proj", "model.layers.18.mlp.experts.156.up_proj", "model.layers.18.mlp.experts.157.up_proj", "model.layers.18.mlp.experts.158.up_proj", "model.layers.18.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.003179170936346065, "dbits": 2516582400 } ] }, { "idx": 110, "layers": [ "model.layers.18.mlp.experts.0.down_proj", "model.layers.18.mlp.experts.1.down_proj", "model.layers.18.mlp.experts.2.down_proj", "model.layers.18.mlp.experts.3.down_proj", "model.layers.18.mlp.experts.4.down_proj", "model.layers.18.mlp.experts.5.down_proj", "model.layers.18.mlp.experts.6.down_proj", "model.layers.18.mlp.experts.7.down_proj", "model.layers.18.mlp.experts.8.down_proj", "model.layers.18.mlp.experts.9.down_proj", "model.layers.18.mlp.experts.10.down_proj", "model.layers.18.mlp.experts.11.down_proj", "model.layers.18.mlp.experts.12.down_proj", "model.layers.18.mlp.experts.13.down_proj", "model.layers.18.mlp.experts.14.down_proj", "model.layers.18.mlp.experts.15.down_proj", "model.layers.18.mlp.experts.16.down_proj", "model.layers.18.mlp.experts.17.down_proj", "model.layers.18.mlp.experts.18.down_proj", "model.layers.18.mlp.experts.19.down_proj", "model.layers.18.mlp.experts.20.down_proj", "model.layers.18.mlp.experts.21.down_proj", "model.layers.18.mlp.experts.22.down_proj", "model.layers.18.mlp.experts.23.down_proj", "model.layers.18.mlp.experts.24.down_proj", "model.layers.18.mlp.experts.25.down_proj", "model.layers.18.mlp.experts.26.down_proj", "model.layers.18.mlp.experts.27.down_proj", "model.layers.18.mlp.experts.28.down_proj", "model.layers.18.mlp.experts.29.down_proj", "model.layers.18.mlp.experts.30.down_proj", "model.layers.18.mlp.experts.31.down_proj", "model.layers.18.mlp.experts.32.down_proj", "model.layers.18.mlp.experts.33.down_proj", "model.layers.18.mlp.experts.34.down_proj", "model.layers.18.mlp.experts.35.down_proj", "model.layers.18.mlp.experts.36.down_proj", "model.layers.18.mlp.experts.37.down_proj", "model.layers.18.mlp.experts.38.down_proj", "model.layers.18.mlp.experts.39.down_proj", "model.layers.18.mlp.experts.40.down_proj", "model.layers.18.mlp.experts.41.down_proj", "model.layers.18.mlp.experts.42.down_proj", "model.layers.18.mlp.experts.43.down_proj", "model.layers.18.mlp.experts.44.down_proj", "model.layers.18.mlp.experts.45.down_proj", "model.layers.18.mlp.experts.46.down_proj", "model.layers.18.mlp.experts.47.down_proj", "model.layers.18.mlp.experts.48.down_proj", "model.layers.18.mlp.experts.49.down_proj", "model.layers.18.mlp.experts.50.down_proj", "model.layers.18.mlp.experts.51.down_proj", "model.layers.18.mlp.experts.52.down_proj", "model.layers.18.mlp.experts.53.down_proj", "model.layers.18.mlp.experts.54.down_proj", "model.layers.18.mlp.experts.55.down_proj", "model.layers.18.mlp.experts.56.down_proj", "model.layers.18.mlp.experts.57.down_proj", "model.layers.18.mlp.experts.58.down_proj", "model.layers.18.mlp.experts.59.down_proj", "model.layers.18.mlp.experts.60.down_proj", "model.layers.18.mlp.experts.61.down_proj", "model.layers.18.mlp.experts.62.down_proj", "model.layers.18.mlp.experts.63.down_proj", "model.layers.18.mlp.experts.64.down_proj", "model.layers.18.mlp.experts.65.down_proj", "model.layers.18.mlp.experts.66.down_proj", "model.layers.18.mlp.experts.67.down_proj", "model.layers.18.mlp.experts.68.down_proj", "model.layers.18.mlp.experts.69.down_proj", "model.layers.18.mlp.experts.70.down_proj", "model.layers.18.mlp.experts.71.down_proj", "model.layers.18.mlp.experts.72.down_proj", "model.layers.18.mlp.experts.73.down_proj", "model.layers.18.mlp.experts.74.down_proj", "model.layers.18.mlp.experts.75.down_proj", "model.layers.18.mlp.experts.76.down_proj", "model.layers.18.mlp.experts.77.down_proj", "model.layers.18.mlp.experts.78.down_proj", "model.layers.18.mlp.experts.79.down_proj", "model.layers.18.mlp.experts.80.down_proj", "model.layers.18.mlp.experts.81.down_proj", "model.layers.18.mlp.experts.82.down_proj", "model.layers.18.mlp.experts.83.down_proj", "model.layers.18.mlp.experts.84.down_proj", "model.layers.18.mlp.experts.85.down_proj", "model.layers.18.mlp.experts.86.down_proj", "model.layers.18.mlp.experts.87.down_proj", "model.layers.18.mlp.experts.88.down_proj", "model.layers.18.mlp.experts.89.down_proj", "model.layers.18.mlp.experts.90.down_proj", "model.layers.18.mlp.experts.91.down_proj", "model.layers.18.mlp.experts.92.down_proj", "model.layers.18.mlp.experts.93.down_proj", "model.layers.18.mlp.experts.94.down_proj", "model.layers.18.mlp.experts.95.down_proj", "model.layers.18.mlp.experts.96.down_proj", "model.layers.18.mlp.experts.97.down_proj", "model.layers.18.mlp.experts.98.down_proj", "model.layers.18.mlp.experts.99.down_proj", "model.layers.18.mlp.experts.100.down_proj", "model.layers.18.mlp.experts.101.down_proj", "model.layers.18.mlp.experts.102.down_proj", "model.layers.18.mlp.experts.103.down_proj", "model.layers.18.mlp.experts.104.down_proj", "model.layers.18.mlp.experts.105.down_proj", "model.layers.18.mlp.experts.106.down_proj", "model.layers.18.mlp.experts.107.down_proj", "model.layers.18.mlp.experts.108.down_proj", "model.layers.18.mlp.experts.109.down_proj", "model.layers.18.mlp.experts.110.down_proj", "model.layers.18.mlp.experts.111.down_proj", "model.layers.18.mlp.experts.112.down_proj", "model.layers.18.mlp.experts.113.down_proj", "model.layers.18.mlp.experts.114.down_proj", "model.layers.18.mlp.experts.115.down_proj", "model.layers.18.mlp.experts.116.down_proj", "model.layers.18.mlp.experts.117.down_proj", "model.layers.18.mlp.experts.118.down_proj", "model.layers.18.mlp.experts.119.down_proj", "model.layers.18.mlp.experts.120.down_proj", "model.layers.18.mlp.experts.121.down_proj", "model.layers.18.mlp.experts.122.down_proj", "model.layers.18.mlp.experts.123.down_proj", "model.layers.18.mlp.experts.124.down_proj", "model.layers.18.mlp.experts.125.down_proj", "model.layers.18.mlp.experts.126.down_proj", "model.layers.18.mlp.experts.127.down_proj", "model.layers.18.mlp.experts.128.down_proj", "model.layers.18.mlp.experts.129.down_proj", "model.layers.18.mlp.experts.130.down_proj", "model.layers.18.mlp.experts.131.down_proj", "model.layers.18.mlp.experts.132.down_proj", "model.layers.18.mlp.experts.133.down_proj", "model.layers.18.mlp.experts.134.down_proj", "model.layers.18.mlp.experts.135.down_proj", "model.layers.18.mlp.experts.136.down_proj", "model.layers.18.mlp.experts.137.down_proj", "model.layers.18.mlp.experts.138.down_proj", "model.layers.18.mlp.experts.139.down_proj", "model.layers.18.mlp.experts.140.down_proj", "model.layers.18.mlp.experts.141.down_proj", "model.layers.18.mlp.experts.142.down_proj", "model.layers.18.mlp.experts.143.down_proj", "model.layers.18.mlp.experts.144.down_proj", "model.layers.18.mlp.experts.145.down_proj", "model.layers.18.mlp.experts.146.down_proj", "model.layers.18.mlp.experts.147.down_proj", "model.layers.18.mlp.experts.148.down_proj", "model.layers.18.mlp.experts.149.down_proj", "model.layers.18.mlp.experts.150.down_proj", "model.layers.18.mlp.experts.151.down_proj", "model.layers.18.mlp.experts.152.down_proj", "model.layers.18.mlp.experts.153.down_proj", "model.layers.18.mlp.experts.154.down_proj", "model.layers.18.mlp.experts.155.down_proj", "model.layers.18.mlp.experts.156.down_proj", "model.layers.18.mlp.experts.157.down_proj", "model.layers.18.mlp.experts.158.down_proj", "model.layers.18.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00249636173248291, "dbits": 1258291200 } ] }, { "idx": 111, "layers": [ "model.layers.19.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0003737524151802285, "dbits": 62914560 } ] }, { "idx": 112, "layers": [ "model.layers.19.self_attn.k_proj", "model.layers.19.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0017167530953884347, "dbits": 10485760 } ] }, { "idx": 113, "layers": [ "model.layers.19.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0002881363034248352, "dbits": 62914560 } ] }, { "idx": 114, "layers": [ "model.layers.19.mlp.shared_experts.gate_proj", "model.layers.19.mlp.shared_experts.up_proj", "model.layers.19.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.002400287240743648, "dbits": 23592960 } ] }, { "idx": 115, "layers": [ "model.layers.19.mlp.experts.0.gate_proj", "model.layers.19.mlp.experts.1.gate_proj", "model.layers.19.mlp.experts.2.gate_proj", "model.layers.19.mlp.experts.3.gate_proj", "model.layers.19.mlp.experts.4.gate_proj", "model.layers.19.mlp.experts.5.gate_proj", "model.layers.19.mlp.experts.6.gate_proj", "model.layers.19.mlp.experts.7.gate_proj", "model.layers.19.mlp.experts.8.gate_proj", "model.layers.19.mlp.experts.9.gate_proj", "model.layers.19.mlp.experts.10.gate_proj", "model.layers.19.mlp.experts.11.gate_proj", "model.layers.19.mlp.experts.12.gate_proj", "model.layers.19.mlp.experts.13.gate_proj", "model.layers.19.mlp.experts.14.gate_proj", "model.layers.19.mlp.experts.15.gate_proj", "model.layers.19.mlp.experts.16.gate_proj", "model.layers.19.mlp.experts.17.gate_proj", "model.layers.19.mlp.experts.18.gate_proj", "model.layers.19.mlp.experts.19.gate_proj", "model.layers.19.mlp.experts.20.gate_proj", "model.layers.19.mlp.experts.21.gate_proj", "model.layers.19.mlp.experts.22.gate_proj", "model.layers.19.mlp.experts.23.gate_proj", "model.layers.19.mlp.experts.24.gate_proj", "model.layers.19.mlp.experts.25.gate_proj", "model.layers.19.mlp.experts.26.gate_proj", "model.layers.19.mlp.experts.27.gate_proj", "model.layers.19.mlp.experts.28.gate_proj", "model.layers.19.mlp.experts.29.gate_proj", "model.layers.19.mlp.experts.30.gate_proj", "model.layers.19.mlp.experts.31.gate_proj", "model.layers.19.mlp.experts.32.gate_proj", "model.layers.19.mlp.experts.33.gate_proj", "model.layers.19.mlp.experts.34.gate_proj", "model.layers.19.mlp.experts.35.gate_proj", "model.layers.19.mlp.experts.36.gate_proj", "model.layers.19.mlp.experts.37.gate_proj", "model.layers.19.mlp.experts.38.gate_proj", "model.layers.19.mlp.experts.39.gate_proj", "model.layers.19.mlp.experts.40.gate_proj", "model.layers.19.mlp.experts.41.gate_proj", "model.layers.19.mlp.experts.42.gate_proj", "model.layers.19.mlp.experts.43.gate_proj", "model.layers.19.mlp.experts.44.gate_proj", "model.layers.19.mlp.experts.45.gate_proj", "model.layers.19.mlp.experts.46.gate_proj", "model.layers.19.mlp.experts.47.gate_proj", "model.layers.19.mlp.experts.48.gate_proj", "model.layers.19.mlp.experts.49.gate_proj", "model.layers.19.mlp.experts.50.gate_proj", "model.layers.19.mlp.experts.51.gate_proj", "model.layers.19.mlp.experts.52.gate_proj", "model.layers.19.mlp.experts.53.gate_proj", "model.layers.19.mlp.experts.54.gate_proj", "model.layers.19.mlp.experts.55.gate_proj", "model.layers.19.mlp.experts.56.gate_proj", "model.layers.19.mlp.experts.57.gate_proj", "model.layers.19.mlp.experts.58.gate_proj", "model.layers.19.mlp.experts.59.gate_proj", "model.layers.19.mlp.experts.60.gate_proj", "model.layers.19.mlp.experts.61.gate_proj", "model.layers.19.mlp.experts.62.gate_proj", "model.layers.19.mlp.experts.63.gate_proj", "model.layers.19.mlp.experts.64.gate_proj", "model.layers.19.mlp.experts.65.gate_proj", "model.layers.19.mlp.experts.66.gate_proj", "model.layers.19.mlp.experts.67.gate_proj", "model.layers.19.mlp.experts.68.gate_proj", "model.layers.19.mlp.experts.69.gate_proj", "model.layers.19.mlp.experts.70.gate_proj", "model.layers.19.mlp.experts.71.gate_proj", "model.layers.19.mlp.experts.72.gate_proj", "model.layers.19.mlp.experts.73.gate_proj", "model.layers.19.mlp.experts.74.gate_proj", "model.layers.19.mlp.experts.75.gate_proj", "model.layers.19.mlp.experts.76.gate_proj", "model.layers.19.mlp.experts.77.gate_proj", "model.layers.19.mlp.experts.78.gate_proj", "model.layers.19.mlp.experts.79.gate_proj", "model.layers.19.mlp.experts.80.gate_proj", "model.layers.19.mlp.experts.81.gate_proj", "model.layers.19.mlp.experts.82.gate_proj", "model.layers.19.mlp.experts.83.gate_proj", "model.layers.19.mlp.experts.84.gate_proj", "model.layers.19.mlp.experts.85.gate_proj", "model.layers.19.mlp.experts.86.gate_proj", "model.layers.19.mlp.experts.87.gate_proj", "model.layers.19.mlp.experts.88.gate_proj", "model.layers.19.mlp.experts.89.gate_proj", "model.layers.19.mlp.experts.90.gate_proj", "model.layers.19.mlp.experts.91.gate_proj", "model.layers.19.mlp.experts.92.gate_proj", "model.layers.19.mlp.experts.93.gate_proj", "model.layers.19.mlp.experts.94.gate_proj", "model.layers.19.mlp.experts.95.gate_proj", "model.layers.19.mlp.experts.96.gate_proj", "model.layers.19.mlp.experts.97.gate_proj", "model.layers.19.mlp.experts.98.gate_proj", "model.layers.19.mlp.experts.99.gate_proj", "model.layers.19.mlp.experts.100.gate_proj", "model.layers.19.mlp.experts.101.gate_proj", "model.layers.19.mlp.experts.102.gate_proj", "model.layers.19.mlp.experts.103.gate_proj", "model.layers.19.mlp.experts.104.gate_proj", "model.layers.19.mlp.experts.105.gate_proj", "model.layers.19.mlp.experts.106.gate_proj", "model.layers.19.mlp.experts.107.gate_proj", "model.layers.19.mlp.experts.108.gate_proj", "model.layers.19.mlp.experts.109.gate_proj", "model.layers.19.mlp.experts.110.gate_proj", "model.layers.19.mlp.experts.111.gate_proj", "model.layers.19.mlp.experts.112.gate_proj", "model.layers.19.mlp.experts.113.gate_proj", "model.layers.19.mlp.experts.114.gate_proj", "model.layers.19.mlp.experts.115.gate_proj", "model.layers.19.mlp.experts.116.gate_proj", "model.layers.19.mlp.experts.117.gate_proj", "model.layers.19.mlp.experts.118.gate_proj", "model.layers.19.mlp.experts.119.gate_proj", "model.layers.19.mlp.experts.120.gate_proj", "model.layers.19.mlp.experts.121.gate_proj", "model.layers.19.mlp.experts.122.gate_proj", "model.layers.19.mlp.experts.123.gate_proj", "model.layers.19.mlp.experts.124.gate_proj", "model.layers.19.mlp.experts.125.gate_proj", "model.layers.19.mlp.experts.126.gate_proj", "model.layers.19.mlp.experts.127.gate_proj", "model.layers.19.mlp.experts.128.gate_proj", "model.layers.19.mlp.experts.129.gate_proj", "model.layers.19.mlp.experts.130.gate_proj", "model.layers.19.mlp.experts.131.gate_proj", "model.layers.19.mlp.experts.132.gate_proj", "model.layers.19.mlp.experts.133.gate_proj", "model.layers.19.mlp.experts.134.gate_proj", "model.layers.19.mlp.experts.135.gate_proj", "model.layers.19.mlp.experts.136.gate_proj", "model.layers.19.mlp.experts.137.gate_proj", "model.layers.19.mlp.experts.138.gate_proj", "model.layers.19.mlp.experts.139.gate_proj", "model.layers.19.mlp.experts.140.gate_proj", "model.layers.19.mlp.experts.141.gate_proj", "model.layers.19.mlp.experts.142.gate_proj", "model.layers.19.mlp.experts.143.gate_proj", "model.layers.19.mlp.experts.144.gate_proj", "model.layers.19.mlp.experts.145.gate_proj", "model.layers.19.mlp.experts.146.gate_proj", "model.layers.19.mlp.experts.147.gate_proj", "model.layers.19.mlp.experts.148.gate_proj", "model.layers.19.mlp.experts.149.gate_proj", "model.layers.19.mlp.experts.150.gate_proj", "model.layers.19.mlp.experts.151.gate_proj", "model.layers.19.mlp.experts.152.gate_proj", "model.layers.19.mlp.experts.153.gate_proj", "model.layers.19.mlp.experts.154.gate_proj", "model.layers.19.mlp.experts.155.gate_proj", "model.layers.19.mlp.experts.156.gate_proj", "model.layers.19.mlp.experts.157.gate_proj", "model.layers.19.mlp.experts.158.gate_proj", "model.layers.19.mlp.experts.159.gate_proj", "model.layers.19.mlp.experts.0.up_proj", "model.layers.19.mlp.experts.1.up_proj", "model.layers.19.mlp.experts.2.up_proj", "model.layers.19.mlp.experts.3.up_proj", "model.layers.19.mlp.experts.4.up_proj", "model.layers.19.mlp.experts.5.up_proj", "model.layers.19.mlp.experts.6.up_proj", "model.layers.19.mlp.experts.7.up_proj", "model.layers.19.mlp.experts.8.up_proj", "model.layers.19.mlp.experts.9.up_proj", "model.layers.19.mlp.experts.10.up_proj", "model.layers.19.mlp.experts.11.up_proj", "model.layers.19.mlp.experts.12.up_proj", "model.layers.19.mlp.experts.13.up_proj", "model.layers.19.mlp.experts.14.up_proj", "model.layers.19.mlp.experts.15.up_proj", "model.layers.19.mlp.experts.16.up_proj", "model.layers.19.mlp.experts.17.up_proj", "model.layers.19.mlp.experts.18.up_proj", "model.layers.19.mlp.experts.19.up_proj", "model.layers.19.mlp.experts.20.up_proj", "model.layers.19.mlp.experts.21.up_proj", "model.layers.19.mlp.experts.22.up_proj", "model.layers.19.mlp.experts.23.up_proj", "model.layers.19.mlp.experts.24.up_proj", "model.layers.19.mlp.experts.25.up_proj", "model.layers.19.mlp.experts.26.up_proj", "model.layers.19.mlp.experts.27.up_proj", "model.layers.19.mlp.experts.28.up_proj", "model.layers.19.mlp.experts.29.up_proj", "model.layers.19.mlp.experts.30.up_proj", "model.layers.19.mlp.experts.31.up_proj", "model.layers.19.mlp.experts.32.up_proj", "model.layers.19.mlp.experts.33.up_proj", "model.layers.19.mlp.experts.34.up_proj", "model.layers.19.mlp.experts.35.up_proj", "model.layers.19.mlp.experts.36.up_proj", "model.layers.19.mlp.experts.37.up_proj", "model.layers.19.mlp.experts.38.up_proj", "model.layers.19.mlp.experts.39.up_proj", "model.layers.19.mlp.experts.40.up_proj", "model.layers.19.mlp.experts.41.up_proj", "model.layers.19.mlp.experts.42.up_proj", "model.layers.19.mlp.experts.43.up_proj", "model.layers.19.mlp.experts.44.up_proj", "model.layers.19.mlp.experts.45.up_proj", "model.layers.19.mlp.experts.46.up_proj", "model.layers.19.mlp.experts.47.up_proj", "model.layers.19.mlp.experts.48.up_proj", "model.layers.19.mlp.experts.49.up_proj", "model.layers.19.mlp.experts.50.up_proj", "model.layers.19.mlp.experts.51.up_proj", "model.layers.19.mlp.experts.52.up_proj", "model.layers.19.mlp.experts.53.up_proj", "model.layers.19.mlp.experts.54.up_proj", "model.layers.19.mlp.experts.55.up_proj", "model.layers.19.mlp.experts.56.up_proj", "model.layers.19.mlp.experts.57.up_proj", "model.layers.19.mlp.experts.58.up_proj", "model.layers.19.mlp.experts.59.up_proj", "model.layers.19.mlp.experts.60.up_proj", "model.layers.19.mlp.experts.61.up_proj", "model.layers.19.mlp.experts.62.up_proj", "model.layers.19.mlp.experts.63.up_proj", "model.layers.19.mlp.experts.64.up_proj", "model.layers.19.mlp.experts.65.up_proj", "model.layers.19.mlp.experts.66.up_proj", "model.layers.19.mlp.experts.67.up_proj", "model.layers.19.mlp.experts.68.up_proj", "model.layers.19.mlp.experts.69.up_proj", "model.layers.19.mlp.experts.70.up_proj", "model.layers.19.mlp.experts.71.up_proj", "model.layers.19.mlp.experts.72.up_proj", "model.layers.19.mlp.experts.73.up_proj", "model.layers.19.mlp.experts.74.up_proj", "model.layers.19.mlp.experts.75.up_proj", "model.layers.19.mlp.experts.76.up_proj", "model.layers.19.mlp.experts.77.up_proj", "model.layers.19.mlp.experts.78.up_proj", "model.layers.19.mlp.experts.79.up_proj", "model.layers.19.mlp.experts.80.up_proj", "model.layers.19.mlp.experts.81.up_proj", "model.layers.19.mlp.experts.82.up_proj", "model.layers.19.mlp.experts.83.up_proj", "model.layers.19.mlp.experts.84.up_proj", "model.layers.19.mlp.experts.85.up_proj", "model.layers.19.mlp.experts.86.up_proj", "model.layers.19.mlp.experts.87.up_proj", "model.layers.19.mlp.experts.88.up_proj", "model.layers.19.mlp.experts.89.up_proj", "model.layers.19.mlp.experts.90.up_proj", "model.layers.19.mlp.experts.91.up_proj", "model.layers.19.mlp.experts.92.up_proj", "model.layers.19.mlp.experts.93.up_proj", "model.layers.19.mlp.experts.94.up_proj", "model.layers.19.mlp.experts.95.up_proj", "model.layers.19.mlp.experts.96.up_proj", "model.layers.19.mlp.experts.97.up_proj", "model.layers.19.mlp.experts.98.up_proj", "model.layers.19.mlp.experts.99.up_proj", "model.layers.19.mlp.experts.100.up_proj", "model.layers.19.mlp.experts.101.up_proj", "model.layers.19.mlp.experts.102.up_proj", "model.layers.19.mlp.experts.103.up_proj", "model.layers.19.mlp.experts.104.up_proj", "model.layers.19.mlp.experts.105.up_proj", "model.layers.19.mlp.experts.106.up_proj", "model.layers.19.mlp.experts.107.up_proj", "model.layers.19.mlp.experts.108.up_proj", "model.layers.19.mlp.experts.109.up_proj", "model.layers.19.mlp.experts.110.up_proj", "model.layers.19.mlp.experts.111.up_proj", "model.layers.19.mlp.experts.112.up_proj", "model.layers.19.mlp.experts.113.up_proj", "model.layers.19.mlp.experts.114.up_proj", "model.layers.19.mlp.experts.115.up_proj", "model.layers.19.mlp.experts.116.up_proj", "model.layers.19.mlp.experts.117.up_proj", "model.layers.19.mlp.experts.118.up_proj", "model.layers.19.mlp.experts.119.up_proj", "model.layers.19.mlp.experts.120.up_proj", "model.layers.19.mlp.experts.121.up_proj", "model.layers.19.mlp.experts.122.up_proj", "model.layers.19.mlp.experts.123.up_proj", "model.layers.19.mlp.experts.124.up_proj", "model.layers.19.mlp.experts.125.up_proj", "model.layers.19.mlp.experts.126.up_proj", "model.layers.19.mlp.experts.127.up_proj", "model.layers.19.mlp.experts.128.up_proj", "model.layers.19.mlp.experts.129.up_proj", "model.layers.19.mlp.experts.130.up_proj", "model.layers.19.mlp.experts.131.up_proj", "model.layers.19.mlp.experts.132.up_proj", "model.layers.19.mlp.experts.133.up_proj", "model.layers.19.mlp.experts.134.up_proj", "model.layers.19.mlp.experts.135.up_proj", "model.layers.19.mlp.experts.136.up_proj", "model.layers.19.mlp.experts.137.up_proj", "model.layers.19.mlp.experts.138.up_proj", "model.layers.19.mlp.experts.139.up_proj", "model.layers.19.mlp.experts.140.up_proj", "model.layers.19.mlp.experts.141.up_proj", "model.layers.19.mlp.experts.142.up_proj", "model.layers.19.mlp.experts.143.up_proj", "model.layers.19.mlp.experts.144.up_proj", "model.layers.19.mlp.experts.145.up_proj", "model.layers.19.mlp.experts.146.up_proj", "model.layers.19.mlp.experts.147.up_proj", "model.layers.19.mlp.experts.148.up_proj", "model.layers.19.mlp.experts.149.up_proj", "model.layers.19.mlp.experts.150.up_proj", "model.layers.19.mlp.experts.151.up_proj", "model.layers.19.mlp.experts.152.up_proj", "model.layers.19.mlp.experts.153.up_proj", "model.layers.19.mlp.experts.154.up_proj", "model.layers.19.mlp.experts.155.up_proj", "model.layers.19.mlp.experts.156.up_proj", "model.layers.19.mlp.experts.157.up_proj", "model.layers.19.mlp.experts.158.up_proj", "model.layers.19.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.003777887672185898, "dbits": 2516582400 } ] }, { "idx": 116, "layers": [ "model.layers.19.mlp.experts.0.down_proj", "model.layers.19.mlp.experts.1.down_proj", "model.layers.19.mlp.experts.2.down_proj", "model.layers.19.mlp.experts.3.down_proj", "model.layers.19.mlp.experts.4.down_proj", "model.layers.19.mlp.experts.5.down_proj", "model.layers.19.mlp.experts.6.down_proj", "model.layers.19.mlp.experts.7.down_proj", "model.layers.19.mlp.experts.8.down_proj", "model.layers.19.mlp.experts.9.down_proj", "model.layers.19.mlp.experts.10.down_proj", "model.layers.19.mlp.experts.11.down_proj", "model.layers.19.mlp.experts.12.down_proj", "model.layers.19.mlp.experts.13.down_proj", "model.layers.19.mlp.experts.14.down_proj", "model.layers.19.mlp.experts.15.down_proj", "model.layers.19.mlp.experts.16.down_proj", "model.layers.19.mlp.experts.17.down_proj", "model.layers.19.mlp.experts.18.down_proj", "model.layers.19.mlp.experts.19.down_proj", "model.layers.19.mlp.experts.20.down_proj", "model.layers.19.mlp.experts.21.down_proj", "model.layers.19.mlp.experts.22.down_proj", "model.layers.19.mlp.experts.23.down_proj", "model.layers.19.mlp.experts.24.down_proj", "model.layers.19.mlp.experts.25.down_proj", "model.layers.19.mlp.experts.26.down_proj", "model.layers.19.mlp.experts.27.down_proj", "model.layers.19.mlp.experts.28.down_proj", "model.layers.19.mlp.experts.29.down_proj", "model.layers.19.mlp.experts.30.down_proj", "model.layers.19.mlp.experts.31.down_proj", "model.layers.19.mlp.experts.32.down_proj", "model.layers.19.mlp.experts.33.down_proj", "model.layers.19.mlp.experts.34.down_proj", "model.layers.19.mlp.experts.35.down_proj", "model.layers.19.mlp.experts.36.down_proj", "model.layers.19.mlp.experts.37.down_proj", "model.layers.19.mlp.experts.38.down_proj", "model.layers.19.mlp.experts.39.down_proj", "model.layers.19.mlp.experts.40.down_proj", "model.layers.19.mlp.experts.41.down_proj", "model.layers.19.mlp.experts.42.down_proj", "model.layers.19.mlp.experts.43.down_proj", "model.layers.19.mlp.experts.44.down_proj", "model.layers.19.mlp.experts.45.down_proj", "model.layers.19.mlp.experts.46.down_proj", "model.layers.19.mlp.experts.47.down_proj", "model.layers.19.mlp.experts.48.down_proj", "model.layers.19.mlp.experts.49.down_proj", "model.layers.19.mlp.experts.50.down_proj", "model.layers.19.mlp.experts.51.down_proj", "model.layers.19.mlp.experts.52.down_proj", "model.layers.19.mlp.experts.53.down_proj", "model.layers.19.mlp.experts.54.down_proj", "model.layers.19.mlp.experts.55.down_proj", "model.layers.19.mlp.experts.56.down_proj", "model.layers.19.mlp.experts.57.down_proj", "model.layers.19.mlp.experts.58.down_proj", "model.layers.19.mlp.experts.59.down_proj", "model.layers.19.mlp.experts.60.down_proj", "model.layers.19.mlp.experts.61.down_proj", "model.layers.19.mlp.experts.62.down_proj", "model.layers.19.mlp.experts.63.down_proj", "model.layers.19.mlp.experts.64.down_proj", "model.layers.19.mlp.experts.65.down_proj", "model.layers.19.mlp.experts.66.down_proj", "model.layers.19.mlp.experts.67.down_proj", "model.layers.19.mlp.experts.68.down_proj", "model.layers.19.mlp.experts.69.down_proj", "model.layers.19.mlp.experts.70.down_proj", "model.layers.19.mlp.experts.71.down_proj", "model.layers.19.mlp.experts.72.down_proj", "model.layers.19.mlp.experts.73.down_proj", "model.layers.19.mlp.experts.74.down_proj", "model.layers.19.mlp.experts.75.down_proj", "model.layers.19.mlp.experts.76.down_proj", "model.layers.19.mlp.experts.77.down_proj", "model.layers.19.mlp.experts.78.down_proj", "model.layers.19.mlp.experts.79.down_proj", "model.layers.19.mlp.experts.80.down_proj", "model.layers.19.mlp.experts.81.down_proj", "model.layers.19.mlp.experts.82.down_proj", "model.layers.19.mlp.experts.83.down_proj", "model.layers.19.mlp.experts.84.down_proj", "model.layers.19.mlp.experts.85.down_proj", "model.layers.19.mlp.experts.86.down_proj", "model.layers.19.mlp.experts.87.down_proj", "model.layers.19.mlp.experts.88.down_proj", "model.layers.19.mlp.experts.89.down_proj", "model.layers.19.mlp.experts.90.down_proj", "model.layers.19.mlp.experts.91.down_proj", "model.layers.19.mlp.experts.92.down_proj", "model.layers.19.mlp.experts.93.down_proj", "model.layers.19.mlp.experts.94.down_proj", "model.layers.19.mlp.experts.95.down_proj", "model.layers.19.mlp.experts.96.down_proj", "model.layers.19.mlp.experts.97.down_proj", "model.layers.19.mlp.experts.98.down_proj", "model.layers.19.mlp.experts.99.down_proj", "model.layers.19.mlp.experts.100.down_proj", "model.layers.19.mlp.experts.101.down_proj", "model.layers.19.mlp.experts.102.down_proj", "model.layers.19.mlp.experts.103.down_proj", "model.layers.19.mlp.experts.104.down_proj", "model.layers.19.mlp.experts.105.down_proj", "model.layers.19.mlp.experts.106.down_proj", "model.layers.19.mlp.experts.107.down_proj", "model.layers.19.mlp.experts.108.down_proj", "model.layers.19.mlp.experts.109.down_proj", "model.layers.19.mlp.experts.110.down_proj", "model.layers.19.mlp.experts.111.down_proj", "model.layers.19.mlp.experts.112.down_proj", "model.layers.19.mlp.experts.113.down_proj", "model.layers.19.mlp.experts.114.down_proj", "model.layers.19.mlp.experts.115.down_proj", "model.layers.19.mlp.experts.116.down_proj", "model.layers.19.mlp.experts.117.down_proj", "model.layers.19.mlp.experts.118.down_proj", "model.layers.19.mlp.experts.119.down_proj", "model.layers.19.mlp.experts.120.down_proj", "model.layers.19.mlp.experts.121.down_proj", "model.layers.19.mlp.experts.122.down_proj", "model.layers.19.mlp.experts.123.down_proj", "model.layers.19.mlp.experts.124.down_proj", "model.layers.19.mlp.experts.125.down_proj", "model.layers.19.mlp.experts.126.down_proj", "model.layers.19.mlp.experts.127.down_proj", "model.layers.19.mlp.experts.128.down_proj", "model.layers.19.mlp.experts.129.down_proj", "model.layers.19.mlp.experts.130.down_proj", "model.layers.19.mlp.experts.131.down_proj", "model.layers.19.mlp.experts.132.down_proj", "model.layers.19.mlp.experts.133.down_proj", "model.layers.19.mlp.experts.134.down_proj", "model.layers.19.mlp.experts.135.down_proj", "model.layers.19.mlp.experts.136.down_proj", "model.layers.19.mlp.experts.137.down_proj", "model.layers.19.mlp.experts.138.down_proj", "model.layers.19.mlp.experts.139.down_proj", "model.layers.19.mlp.experts.140.down_proj", "model.layers.19.mlp.experts.141.down_proj", "model.layers.19.mlp.experts.142.down_proj", "model.layers.19.mlp.experts.143.down_proj", "model.layers.19.mlp.experts.144.down_proj", "model.layers.19.mlp.experts.145.down_proj", "model.layers.19.mlp.experts.146.down_proj", "model.layers.19.mlp.experts.147.down_proj", "model.layers.19.mlp.experts.148.down_proj", "model.layers.19.mlp.experts.149.down_proj", "model.layers.19.mlp.experts.150.down_proj", "model.layers.19.mlp.experts.151.down_proj", "model.layers.19.mlp.experts.152.down_proj", "model.layers.19.mlp.experts.153.down_proj", "model.layers.19.mlp.experts.154.down_proj", "model.layers.19.mlp.experts.155.down_proj", "model.layers.19.mlp.experts.156.down_proj", "model.layers.19.mlp.experts.157.down_proj", "model.layers.19.mlp.experts.158.down_proj", "model.layers.19.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0025224439799785836, "dbits": 1258291200 } ] }, { "idx": 117, "layers": [ "model.layers.20.self_attn.q_proj" ], "candidates": [ { "dkld": 0.002295272052288022, "dbits": 62914560 } ] }, { "idx": 118, "layers": [ "model.layers.20.self_attn.k_proj", "model.layers.20.self_attn.v_proj" ], "candidates": [ { "dkld": 0.002938357740640607, "dbits": 10485760 } ] }, { "idx": 119, "layers": [ "model.layers.20.self_attn.o_proj" ], "candidates": [ { "dkld": -0.007648160308599494, "dbits": 62914560 } ] }, { "idx": 120, "layers": [ "model.layers.20.mlp.shared_experts.gate_proj", "model.layers.20.mlp.shared_experts.up_proj", "model.layers.20.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.007621482759714138, "dbits": 23592960 } ] }, { "idx": 121, "layers": [ "model.layers.20.mlp.experts.0.gate_proj", "model.layers.20.mlp.experts.1.gate_proj", "model.layers.20.mlp.experts.2.gate_proj", "model.layers.20.mlp.experts.3.gate_proj", "model.layers.20.mlp.experts.4.gate_proj", "model.layers.20.mlp.experts.5.gate_proj", "model.layers.20.mlp.experts.6.gate_proj", "model.layers.20.mlp.experts.7.gate_proj", "model.layers.20.mlp.experts.8.gate_proj", "model.layers.20.mlp.experts.9.gate_proj", "model.layers.20.mlp.experts.10.gate_proj", "model.layers.20.mlp.experts.11.gate_proj", "model.layers.20.mlp.experts.12.gate_proj", "model.layers.20.mlp.experts.13.gate_proj", "model.layers.20.mlp.experts.14.gate_proj", "model.layers.20.mlp.experts.15.gate_proj", "model.layers.20.mlp.experts.16.gate_proj", "model.layers.20.mlp.experts.17.gate_proj", "model.layers.20.mlp.experts.18.gate_proj", "model.layers.20.mlp.experts.19.gate_proj", "model.layers.20.mlp.experts.20.gate_proj", "model.layers.20.mlp.experts.21.gate_proj", "model.layers.20.mlp.experts.22.gate_proj", "model.layers.20.mlp.experts.23.gate_proj", "model.layers.20.mlp.experts.24.gate_proj", "model.layers.20.mlp.experts.25.gate_proj", "model.layers.20.mlp.experts.26.gate_proj", "model.layers.20.mlp.experts.27.gate_proj", "model.layers.20.mlp.experts.28.gate_proj", "model.layers.20.mlp.experts.29.gate_proj", "model.layers.20.mlp.experts.30.gate_proj", "model.layers.20.mlp.experts.31.gate_proj", "model.layers.20.mlp.experts.32.gate_proj", "model.layers.20.mlp.experts.33.gate_proj", "model.layers.20.mlp.experts.34.gate_proj", "model.layers.20.mlp.experts.35.gate_proj", "model.layers.20.mlp.experts.36.gate_proj", "model.layers.20.mlp.experts.37.gate_proj", "model.layers.20.mlp.experts.38.gate_proj", "model.layers.20.mlp.experts.39.gate_proj", "model.layers.20.mlp.experts.40.gate_proj", "model.layers.20.mlp.experts.41.gate_proj", "model.layers.20.mlp.experts.42.gate_proj", "model.layers.20.mlp.experts.43.gate_proj", "model.layers.20.mlp.experts.44.gate_proj", "model.layers.20.mlp.experts.45.gate_proj", "model.layers.20.mlp.experts.46.gate_proj", "model.layers.20.mlp.experts.47.gate_proj", "model.layers.20.mlp.experts.48.gate_proj", "model.layers.20.mlp.experts.49.gate_proj", "model.layers.20.mlp.experts.50.gate_proj", "model.layers.20.mlp.experts.51.gate_proj", "model.layers.20.mlp.experts.52.gate_proj", "model.layers.20.mlp.experts.53.gate_proj", "model.layers.20.mlp.experts.54.gate_proj", "model.layers.20.mlp.experts.55.gate_proj", "model.layers.20.mlp.experts.56.gate_proj", "model.layers.20.mlp.experts.57.gate_proj", "model.layers.20.mlp.experts.58.gate_proj", "model.layers.20.mlp.experts.59.gate_proj", "model.layers.20.mlp.experts.60.gate_proj", "model.layers.20.mlp.experts.61.gate_proj", "model.layers.20.mlp.experts.62.gate_proj", "model.layers.20.mlp.experts.63.gate_proj", "model.layers.20.mlp.experts.64.gate_proj", "model.layers.20.mlp.experts.65.gate_proj", "model.layers.20.mlp.experts.66.gate_proj", "model.layers.20.mlp.experts.67.gate_proj", "model.layers.20.mlp.experts.68.gate_proj", "model.layers.20.mlp.experts.69.gate_proj", "model.layers.20.mlp.experts.70.gate_proj", "model.layers.20.mlp.experts.71.gate_proj", "model.layers.20.mlp.experts.72.gate_proj", "model.layers.20.mlp.experts.73.gate_proj", "model.layers.20.mlp.experts.74.gate_proj", "model.layers.20.mlp.experts.75.gate_proj", "model.layers.20.mlp.experts.76.gate_proj", "model.layers.20.mlp.experts.77.gate_proj", "model.layers.20.mlp.experts.78.gate_proj", "model.layers.20.mlp.experts.79.gate_proj", "model.layers.20.mlp.experts.80.gate_proj", "model.layers.20.mlp.experts.81.gate_proj", "model.layers.20.mlp.experts.82.gate_proj", "model.layers.20.mlp.experts.83.gate_proj", "model.layers.20.mlp.experts.84.gate_proj", "model.layers.20.mlp.experts.85.gate_proj", "model.layers.20.mlp.experts.86.gate_proj", "model.layers.20.mlp.experts.87.gate_proj", "model.layers.20.mlp.experts.88.gate_proj", "model.layers.20.mlp.experts.89.gate_proj", "model.layers.20.mlp.experts.90.gate_proj", "model.layers.20.mlp.experts.91.gate_proj", "model.layers.20.mlp.experts.92.gate_proj", "model.layers.20.mlp.experts.93.gate_proj", "model.layers.20.mlp.experts.94.gate_proj", "model.layers.20.mlp.experts.95.gate_proj", "model.layers.20.mlp.experts.96.gate_proj", "model.layers.20.mlp.experts.97.gate_proj", "model.layers.20.mlp.experts.98.gate_proj", "model.layers.20.mlp.experts.99.gate_proj", "model.layers.20.mlp.experts.100.gate_proj", "model.layers.20.mlp.experts.101.gate_proj", "model.layers.20.mlp.experts.102.gate_proj", "model.layers.20.mlp.experts.103.gate_proj", "model.layers.20.mlp.experts.104.gate_proj", "model.layers.20.mlp.experts.105.gate_proj", "model.layers.20.mlp.experts.106.gate_proj", "model.layers.20.mlp.experts.107.gate_proj", "model.layers.20.mlp.experts.108.gate_proj", "model.layers.20.mlp.experts.109.gate_proj", "model.layers.20.mlp.experts.110.gate_proj", "model.layers.20.mlp.experts.111.gate_proj", "model.layers.20.mlp.experts.112.gate_proj", "model.layers.20.mlp.experts.113.gate_proj", "model.layers.20.mlp.experts.114.gate_proj", "model.layers.20.mlp.experts.115.gate_proj", "model.layers.20.mlp.experts.116.gate_proj", "model.layers.20.mlp.experts.117.gate_proj", "model.layers.20.mlp.experts.118.gate_proj", "model.layers.20.mlp.experts.119.gate_proj", "model.layers.20.mlp.experts.120.gate_proj", "model.layers.20.mlp.experts.121.gate_proj", "model.layers.20.mlp.experts.122.gate_proj", "model.layers.20.mlp.experts.123.gate_proj", "model.layers.20.mlp.experts.124.gate_proj", "model.layers.20.mlp.experts.125.gate_proj", "model.layers.20.mlp.experts.126.gate_proj", "model.layers.20.mlp.experts.127.gate_proj", "model.layers.20.mlp.experts.128.gate_proj", "model.layers.20.mlp.experts.129.gate_proj", "model.layers.20.mlp.experts.130.gate_proj", "model.layers.20.mlp.experts.131.gate_proj", "model.layers.20.mlp.experts.132.gate_proj", "model.layers.20.mlp.experts.133.gate_proj", "model.layers.20.mlp.experts.134.gate_proj", "model.layers.20.mlp.experts.135.gate_proj", "model.layers.20.mlp.experts.136.gate_proj", "model.layers.20.mlp.experts.137.gate_proj", "model.layers.20.mlp.experts.138.gate_proj", "model.layers.20.mlp.experts.139.gate_proj", "model.layers.20.mlp.experts.140.gate_proj", "model.layers.20.mlp.experts.141.gate_proj", "model.layers.20.mlp.experts.142.gate_proj", "model.layers.20.mlp.experts.143.gate_proj", "model.layers.20.mlp.experts.144.gate_proj", "model.layers.20.mlp.experts.145.gate_proj", "model.layers.20.mlp.experts.146.gate_proj", "model.layers.20.mlp.experts.147.gate_proj", "model.layers.20.mlp.experts.148.gate_proj", "model.layers.20.mlp.experts.149.gate_proj", "model.layers.20.mlp.experts.150.gate_proj", "model.layers.20.mlp.experts.151.gate_proj", "model.layers.20.mlp.experts.152.gate_proj", "model.layers.20.mlp.experts.153.gate_proj", "model.layers.20.mlp.experts.154.gate_proj", "model.layers.20.mlp.experts.155.gate_proj", "model.layers.20.mlp.experts.156.gate_proj", "model.layers.20.mlp.experts.157.gate_proj", "model.layers.20.mlp.experts.158.gate_proj", "model.layers.20.mlp.experts.159.gate_proj", "model.layers.20.mlp.experts.0.up_proj", "model.layers.20.mlp.experts.1.up_proj", "model.layers.20.mlp.experts.2.up_proj", "model.layers.20.mlp.experts.3.up_proj", "model.layers.20.mlp.experts.4.up_proj", "model.layers.20.mlp.experts.5.up_proj", "model.layers.20.mlp.experts.6.up_proj", "model.layers.20.mlp.experts.7.up_proj", "model.layers.20.mlp.experts.8.up_proj", "model.layers.20.mlp.experts.9.up_proj", "model.layers.20.mlp.experts.10.up_proj", "model.layers.20.mlp.experts.11.up_proj", "model.layers.20.mlp.experts.12.up_proj", "model.layers.20.mlp.experts.13.up_proj", "model.layers.20.mlp.experts.14.up_proj", "model.layers.20.mlp.experts.15.up_proj", "model.layers.20.mlp.experts.16.up_proj", "model.layers.20.mlp.experts.17.up_proj", "model.layers.20.mlp.experts.18.up_proj", "model.layers.20.mlp.experts.19.up_proj", "model.layers.20.mlp.experts.20.up_proj", "model.layers.20.mlp.experts.21.up_proj", "model.layers.20.mlp.experts.22.up_proj", "model.layers.20.mlp.experts.23.up_proj", "model.layers.20.mlp.experts.24.up_proj", "model.layers.20.mlp.experts.25.up_proj", "model.layers.20.mlp.experts.26.up_proj", "model.layers.20.mlp.experts.27.up_proj", "model.layers.20.mlp.experts.28.up_proj", "model.layers.20.mlp.experts.29.up_proj", "model.layers.20.mlp.experts.30.up_proj", "model.layers.20.mlp.experts.31.up_proj", "model.layers.20.mlp.experts.32.up_proj", "model.layers.20.mlp.experts.33.up_proj", "model.layers.20.mlp.experts.34.up_proj", "model.layers.20.mlp.experts.35.up_proj", "model.layers.20.mlp.experts.36.up_proj", "model.layers.20.mlp.experts.37.up_proj", "model.layers.20.mlp.experts.38.up_proj", "model.layers.20.mlp.experts.39.up_proj", "model.layers.20.mlp.experts.40.up_proj", "model.layers.20.mlp.experts.41.up_proj", "model.layers.20.mlp.experts.42.up_proj", "model.layers.20.mlp.experts.43.up_proj", "model.layers.20.mlp.experts.44.up_proj", "model.layers.20.mlp.experts.45.up_proj", "model.layers.20.mlp.experts.46.up_proj", "model.layers.20.mlp.experts.47.up_proj", "model.layers.20.mlp.experts.48.up_proj", "model.layers.20.mlp.experts.49.up_proj", "model.layers.20.mlp.experts.50.up_proj", "model.layers.20.mlp.experts.51.up_proj", "model.layers.20.mlp.experts.52.up_proj", "model.layers.20.mlp.experts.53.up_proj", "model.layers.20.mlp.experts.54.up_proj", "model.layers.20.mlp.experts.55.up_proj", "model.layers.20.mlp.experts.56.up_proj", "model.layers.20.mlp.experts.57.up_proj", "model.layers.20.mlp.experts.58.up_proj", "model.layers.20.mlp.experts.59.up_proj", "model.layers.20.mlp.experts.60.up_proj", "model.layers.20.mlp.experts.61.up_proj", "model.layers.20.mlp.experts.62.up_proj", "model.layers.20.mlp.experts.63.up_proj", "model.layers.20.mlp.experts.64.up_proj", "model.layers.20.mlp.experts.65.up_proj", "model.layers.20.mlp.experts.66.up_proj", "model.layers.20.mlp.experts.67.up_proj", "model.layers.20.mlp.experts.68.up_proj", "model.layers.20.mlp.experts.69.up_proj", "model.layers.20.mlp.experts.70.up_proj", "model.layers.20.mlp.experts.71.up_proj", "model.layers.20.mlp.experts.72.up_proj", "model.layers.20.mlp.experts.73.up_proj", "model.layers.20.mlp.experts.74.up_proj", "model.layers.20.mlp.experts.75.up_proj", "model.layers.20.mlp.experts.76.up_proj", "model.layers.20.mlp.experts.77.up_proj", "model.layers.20.mlp.experts.78.up_proj", "model.layers.20.mlp.experts.79.up_proj", "model.layers.20.mlp.experts.80.up_proj", "model.layers.20.mlp.experts.81.up_proj", "model.layers.20.mlp.experts.82.up_proj", "model.layers.20.mlp.experts.83.up_proj", "model.layers.20.mlp.experts.84.up_proj", "model.layers.20.mlp.experts.85.up_proj", "model.layers.20.mlp.experts.86.up_proj", "model.layers.20.mlp.experts.87.up_proj", "model.layers.20.mlp.experts.88.up_proj", "model.layers.20.mlp.experts.89.up_proj", "model.layers.20.mlp.experts.90.up_proj", "model.layers.20.mlp.experts.91.up_proj", "model.layers.20.mlp.experts.92.up_proj", "model.layers.20.mlp.experts.93.up_proj", "model.layers.20.mlp.experts.94.up_proj", "model.layers.20.mlp.experts.95.up_proj", "model.layers.20.mlp.experts.96.up_proj", "model.layers.20.mlp.experts.97.up_proj", "model.layers.20.mlp.experts.98.up_proj", "model.layers.20.mlp.experts.99.up_proj", "model.layers.20.mlp.experts.100.up_proj", "model.layers.20.mlp.experts.101.up_proj", "model.layers.20.mlp.experts.102.up_proj", "model.layers.20.mlp.experts.103.up_proj", "model.layers.20.mlp.experts.104.up_proj", "model.layers.20.mlp.experts.105.up_proj", "model.layers.20.mlp.experts.106.up_proj", "model.layers.20.mlp.experts.107.up_proj", "model.layers.20.mlp.experts.108.up_proj", "model.layers.20.mlp.experts.109.up_proj", "model.layers.20.mlp.experts.110.up_proj", "model.layers.20.mlp.experts.111.up_proj", "model.layers.20.mlp.experts.112.up_proj", "model.layers.20.mlp.experts.113.up_proj", "model.layers.20.mlp.experts.114.up_proj", "model.layers.20.mlp.experts.115.up_proj", "model.layers.20.mlp.experts.116.up_proj", "model.layers.20.mlp.experts.117.up_proj", "model.layers.20.mlp.experts.118.up_proj", "model.layers.20.mlp.experts.119.up_proj", "model.layers.20.mlp.experts.120.up_proj", "model.layers.20.mlp.experts.121.up_proj", "model.layers.20.mlp.experts.122.up_proj", "model.layers.20.mlp.experts.123.up_proj", "model.layers.20.mlp.experts.124.up_proj", "model.layers.20.mlp.experts.125.up_proj", "model.layers.20.mlp.experts.126.up_proj", "model.layers.20.mlp.experts.127.up_proj", "model.layers.20.mlp.experts.128.up_proj", "model.layers.20.mlp.experts.129.up_proj", "model.layers.20.mlp.experts.130.up_proj", "model.layers.20.mlp.experts.131.up_proj", "model.layers.20.mlp.experts.132.up_proj", "model.layers.20.mlp.experts.133.up_proj", "model.layers.20.mlp.experts.134.up_proj", "model.layers.20.mlp.experts.135.up_proj", "model.layers.20.mlp.experts.136.up_proj", "model.layers.20.mlp.experts.137.up_proj", "model.layers.20.mlp.experts.138.up_proj", "model.layers.20.mlp.experts.139.up_proj", "model.layers.20.mlp.experts.140.up_proj", "model.layers.20.mlp.experts.141.up_proj", "model.layers.20.mlp.experts.142.up_proj", "model.layers.20.mlp.experts.143.up_proj", "model.layers.20.mlp.experts.144.up_proj", "model.layers.20.mlp.experts.145.up_proj", "model.layers.20.mlp.experts.146.up_proj", "model.layers.20.mlp.experts.147.up_proj", "model.layers.20.mlp.experts.148.up_proj", "model.layers.20.mlp.experts.149.up_proj", "model.layers.20.mlp.experts.150.up_proj", "model.layers.20.mlp.experts.151.up_proj", "model.layers.20.mlp.experts.152.up_proj", "model.layers.20.mlp.experts.153.up_proj", "model.layers.20.mlp.experts.154.up_proj", "model.layers.20.mlp.experts.155.up_proj", "model.layers.20.mlp.experts.156.up_proj", "model.layers.20.mlp.experts.157.up_proj", "model.layers.20.mlp.experts.158.up_proj", "model.layers.20.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0044104702770710325, "dbits": 2516582400 } ] }, { "idx": 122, "layers": [ "model.layers.20.mlp.experts.0.down_proj", "model.layers.20.mlp.experts.1.down_proj", "model.layers.20.mlp.experts.2.down_proj", "model.layers.20.mlp.experts.3.down_proj", "model.layers.20.mlp.experts.4.down_proj", "model.layers.20.mlp.experts.5.down_proj", "model.layers.20.mlp.experts.6.down_proj", "model.layers.20.mlp.experts.7.down_proj", "model.layers.20.mlp.experts.8.down_proj", "model.layers.20.mlp.experts.9.down_proj", "model.layers.20.mlp.experts.10.down_proj", "model.layers.20.mlp.experts.11.down_proj", "model.layers.20.mlp.experts.12.down_proj", "model.layers.20.mlp.experts.13.down_proj", "model.layers.20.mlp.experts.14.down_proj", "model.layers.20.mlp.experts.15.down_proj", "model.layers.20.mlp.experts.16.down_proj", "model.layers.20.mlp.experts.17.down_proj", "model.layers.20.mlp.experts.18.down_proj", "model.layers.20.mlp.experts.19.down_proj", "model.layers.20.mlp.experts.20.down_proj", "model.layers.20.mlp.experts.21.down_proj", "model.layers.20.mlp.experts.22.down_proj", "model.layers.20.mlp.experts.23.down_proj", "model.layers.20.mlp.experts.24.down_proj", "model.layers.20.mlp.experts.25.down_proj", "model.layers.20.mlp.experts.26.down_proj", "model.layers.20.mlp.experts.27.down_proj", "model.layers.20.mlp.experts.28.down_proj", "model.layers.20.mlp.experts.29.down_proj", "model.layers.20.mlp.experts.30.down_proj", "model.layers.20.mlp.experts.31.down_proj", "model.layers.20.mlp.experts.32.down_proj", "model.layers.20.mlp.experts.33.down_proj", "model.layers.20.mlp.experts.34.down_proj", "model.layers.20.mlp.experts.35.down_proj", "model.layers.20.mlp.experts.36.down_proj", "model.layers.20.mlp.experts.37.down_proj", "model.layers.20.mlp.experts.38.down_proj", "model.layers.20.mlp.experts.39.down_proj", "model.layers.20.mlp.experts.40.down_proj", "model.layers.20.mlp.experts.41.down_proj", "model.layers.20.mlp.experts.42.down_proj", "model.layers.20.mlp.experts.43.down_proj", "model.layers.20.mlp.experts.44.down_proj", "model.layers.20.mlp.experts.45.down_proj", "model.layers.20.mlp.experts.46.down_proj", "model.layers.20.mlp.experts.47.down_proj", "model.layers.20.mlp.experts.48.down_proj", "model.layers.20.mlp.experts.49.down_proj", "model.layers.20.mlp.experts.50.down_proj", "model.layers.20.mlp.experts.51.down_proj", "model.layers.20.mlp.experts.52.down_proj", "model.layers.20.mlp.experts.53.down_proj", "model.layers.20.mlp.experts.54.down_proj", "model.layers.20.mlp.experts.55.down_proj", "model.layers.20.mlp.experts.56.down_proj", "model.layers.20.mlp.experts.57.down_proj", "model.layers.20.mlp.experts.58.down_proj", "model.layers.20.mlp.experts.59.down_proj", "model.layers.20.mlp.experts.60.down_proj", "model.layers.20.mlp.experts.61.down_proj", "model.layers.20.mlp.experts.62.down_proj", "model.layers.20.mlp.experts.63.down_proj", "model.layers.20.mlp.experts.64.down_proj", "model.layers.20.mlp.experts.65.down_proj", "model.layers.20.mlp.experts.66.down_proj", "model.layers.20.mlp.experts.67.down_proj", "model.layers.20.mlp.experts.68.down_proj", "model.layers.20.mlp.experts.69.down_proj", "model.layers.20.mlp.experts.70.down_proj", "model.layers.20.mlp.experts.71.down_proj", "model.layers.20.mlp.experts.72.down_proj", "model.layers.20.mlp.experts.73.down_proj", "model.layers.20.mlp.experts.74.down_proj", "model.layers.20.mlp.experts.75.down_proj", "model.layers.20.mlp.experts.76.down_proj", "model.layers.20.mlp.experts.77.down_proj", "model.layers.20.mlp.experts.78.down_proj", "model.layers.20.mlp.experts.79.down_proj", "model.layers.20.mlp.experts.80.down_proj", "model.layers.20.mlp.experts.81.down_proj", "model.layers.20.mlp.experts.82.down_proj", "model.layers.20.mlp.experts.83.down_proj", "model.layers.20.mlp.experts.84.down_proj", "model.layers.20.mlp.experts.85.down_proj", "model.layers.20.mlp.experts.86.down_proj", "model.layers.20.mlp.experts.87.down_proj", "model.layers.20.mlp.experts.88.down_proj", "model.layers.20.mlp.experts.89.down_proj", "model.layers.20.mlp.experts.90.down_proj", "model.layers.20.mlp.experts.91.down_proj", "model.layers.20.mlp.experts.92.down_proj", "model.layers.20.mlp.experts.93.down_proj", "model.layers.20.mlp.experts.94.down_proj", "model.layers.20.mlp.experts.95.down_proj", "model.layers.20.mlp.experts.96.down_proj", "model.layers.20.mlp.experts.97.down_proj", "model.layers.20.mlp.experts.98.down_proj", "model.layers.20.mlp.experts.99.down_proj", "model.layers.20.mlp.experts.100.down_proj", "model.layers.20.mlp.experts.101.down_proj", "model.layers.20.mlp.experts.102.down_proj", "model.layers.20.mlp.experts.103.down_proj", "model.layers.20.mlp.experts.104.down_proj", "model.layers.20.mlp.experts.105.down_proj", "model.layers.20.mlp.experts.106.down_proj", "model.layers.20.mlp.experts.107.down_proj", "model.layers.20.mlp.experts.108.down_proj", "model.layers.20.mlp.experts.109.down_proj", "model.layers.20.mlp.experts.110.down_proj", "model.layers.20.mlp.experts.111.down_proj", "model.layers.20.mlp.experts.112.down_proj", "model.layers.20.mlp.experts.113.down_proj", "model.layers.20.mlp.experts.114.down_proj", "model.layers.20.mlp.experts.115.down_proj", "model.layers.20.mlp.experts.116.down_proj", "model.layers.20.mlp.experts.117.down_proj", "model.layers.20.mlp.experts.118.down_proj", "model.layers.20.mlp.experts.119.down_proj", "model.layers.20.mlp.experts.120.down_proj", "model.layers.20.mlp.experts.121.down_proj", "model.layers.20.mlp.experts.122.down_proj", "model.layers.20.mlp.experts.123.down_proj", "model.layers.20.mlp.experts.124.down_proj", "model.layers.20.mlp.experts.125.down_proj", "model.layers.20.mlp.experts.126.down_proj", "model.layers.20.mlp.experts.127.down_proj", "model.layers.20.mlp.experts.128.down_proj", "model.layers.20.mlp.experts.129.down_proj", "model.layers.20.mlp.experts.130.down_proj", "model.layers.20.mlp.experts.131.down_proj", "model.layers.20.mlp.experts.132.down_proj", "model.layers.20.mlp.experts.133.down_proj", "model.layers.20.mlp.experts.134.down_proj", "model.layers.20.mlp.experts.135.down_proj", "model.layers.20.mlp.experts.136.down_proj", "model.layers.20.mlp.experts.137.down_proj", "model.layers.20.mlp.experts.138.down_proj", "model.layers.20.mlp.experts.139.down_proj", "model.layers.20.mlp.experts.140.down_proj", "model.layers.20.mlp.experts.141.down_proj", "model.layers.20.mlp.experts.142.down_proj", "model.layers.20.mlp.experts.143.down_proj", "model.layers.20.mlp.experts.144.down_proj", "model.layers.20.mlp.experts.145.down_proj", "model.layers.20.mlp.experts.146.down_proj", "model.layers.20.mlp.experts.147.down_proj", "model.layers.20.mlp.experts.148.down_proj", "model.layers.20.mlp.experts.149.down_proj", "model.layers.20.mlp.experts.150.down_proj", "model.layers.20.mlp.experts.151.down_proj", "model.layers.20.mlp.experts.152.down_proj", "model.layers.20.mlp.experts.153.down_proj", "model.layers.20.mlp.experts.154.down_proj", "model.layers.20.mlp.experts.155.down_proj", "model.layers.20.mlp.experts.156.down_proj", "model.layers.20.mlp.experts.157.down_proj", "model.layers.20.mlp.experts.158.down_proj", "model.layers.20.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002659713476896308, "dbits": 1258291200 } ] }, { "idx": 123, "layers": [ "model.layers.21.self_attn.q_proj" ], "candidates": [ { "dkld": 0.00013540536165235206, "dbits": 62914560 } ] }, { "idx": 124, "layers": [ "model.layers.21.self_attn.k_proj", "model.layers.21.self_attn.v_proj" ], "candidates": [ { "dkld": -0.002683246880769763, "dbits": 10485760 } ] }, { "idx": 125, "layers": [ "model.layers.21.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0010687366127967723, "dbits": 62914560 } ] }, { "idx": 126, "layers": [ "model.layers.21.mlp.shared_experts.gate_proj", "model.layers.21.mlp.shared_experts.up_proj", "model.layers.21.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0032680824398994557, "dbits": 23592960 } ] }, { "idx": 127, "layers": [ "model.layers.21.mlp.experts.0.gate_proj", "model.layers.21.mlp.experts.1.gate_proj", "model.layers.21.mlp.experts.2.gate_proj", "model.layers.21.mlp.experts.3.gate_proj", "model.layers.21.mlp.experts.4.gate_proj", "model.layers.21.mlp.experts.5.gate_proj", "model.layers.21.mlp.experts.6.gate_proj", "model.layers.21.mlp.experts.7.gate_proj", "model.layers.21.mlp.experts.8.gate_proj", "model.layers.21.mlp.experts.9.gate_proj", "model.layers.21.mlp.experts.10.gate_proj", "model.layers.21.mlp.experts.11.gate_proj", "model.layers.21.mlp.experts.12.gate_proj", "model.layers.21.mlp.experts.13.gate_proj", "model.layers.21.mlp.experts.14.gate_proj", "model.layers.21.mlp.experts.15.gate_proj", "model.layers.21.mlp.experts.16.gate_proj", "model.layers.21.mlp.experts.17.gate_proj", "model.layers.21.mlp.experts.18.gate_proj", "model.layers.21.mlp.experts.19.gate_proj", "model.layers.21.mlp.experts.20.gate_proj", "model.layers.21.mlp.experts.21.gate_proj", "model.layers.21.mlp.experts.22.gate_proj", "model.layers.21.mlp.experts.23.gate_proj", "model.layers.21.mlp.experts.24.gate_proj", "model.layers.21.mlp.experts.25.gate_proj", "model.layers.21.mlp.experts.26.gate_proj", "model.layers.21.mlp.experts.27.gate_proj", "model.layers.21.mlp.experts.28.gate_proj", "model.layers.21.mlp.experts.29.gate_proj", "model.layers.21.mlp.experts.30.gate_proj", "model.layers.21.mlp.experts.31.gate_proj", "model.layers.21.mlp.experts.32.gate_proj", "model.layers.21.mlp.experts.33.gate_proj", "model.layers.21.mlp.experts.34.gate_proj", "model.layers.21.mlp.experts.35.gate_proj", "model.layers.21.mlp.experts.36.gate_proj", "model.layers.21.mlp.experts.37.gate_proj", "model.layers.21.mlp.experts.38.gate_proj", "model.layers.21.mlp.experts.39.gate_proj", "model.layers.21.mlp.experts.40.gate_proj", "model.layers.21.mlp.experts.41.gate_proj", "model.layers.21.mlp.experts.42.gate_proj", "model.layers.21.mlp.experts.43.gate_proj", "model.layers.21.mlp.experts.44.gate_proj", "model.layers.21.mlp.experts.45.gate_proj", "model.layers.21.mlp.experts.46.gate_proj", "model.layers.21.mlp.experts.47.gate_proj", "model.layers.21.mlp.experts.48.gate_proj", "model.layers.21.mlp.experts.49.gate_proj", "model.layers.21.mlp.experts.50.gate_proj", "model.layers.21.mlp.experts.51.gate_proj", "model.layers.21.mlp.experts.52.gate_proj", "model.layers.21.mlp.experts.53.gate_proj", "model.layers.21.mlp.experts.54.gate_proj", "model.layers.21.mlp.experts.55.gate_proj", "model.layers.21.mlp.experts.56.gate_proj", "model.layers.21.mlp.experts.57.gate_proj", "model.layers.21.mlp.experts.58.gate_proj", "model.layers.21.mlp.experts.59.gate_proj", "model.layers.21.mlp.experts.60.gate_proj", "model.layers.21.mlp.experts.61.gate_proj", "model.layers.21.mlp.experts.62.gate_proj", "model.layers.21.mlp.experts.63.gate_proj", "model.layers.21.mlp.experts.64.gate_proj", "model.layers.21.mlp.experts.65.gate_proj", "model.layers.21.mlp.experts.66.gate_proj", "model.layers.21.mlp.experts.67.gate_proj", "model.layers.21.mlp.experts.68.gate_proj", "model.layers.21.mlp.experts.69.gate_proj", "model.layers.21.mlp.experts.70.gate_proj", "model.layers.21.mlp.experts.71.gate_proj", "model.layers.21.mlp.experts.72.gate_proj", "model.layers.21.mlp.experts.73.gate_proj", "model.layers.21.mlp.experts.74.gate_proj", "model.layers.21.mlp.experts.75.gate_proj", "model.layers.21.mlp.experts.76.gate_proj", "model.layers.21.mlp.experts.77.gate_proj", "model.layers.21.mlp.experts.78.gate_proj", "model.layers.21.mlp.experts.79.gate_proj", "model.layers.21.mlp.experts.80.gate_proj", "model.layers.21.mlp.experts.81.gate_proj", "model.layers.21.mlp.experts.82.gate_proj", "model.layers.21.mlp.experts.83.gate_proj", "model.layers.21.mlp.experts.84.gate_proj", "model.layers.21.mlp.experts.85.gate_proj", "model.layers.21.mlp.experts.86.gate_proj", "model.layers.21.mlp.experts.87.gate_proj", "model.layers.21.mlp.experts.88.gate_proj", "model.layers.21.mlp.experts.89.gate_proj", "model.layers.21.mlp.experts.90.gate_proj", "model.layers.21.mlp.experts.91.gate_proj", "model.layers.21.mlp.experts.92.gate_proj", "model.layers.21.mlp.experts.93.gate_proj", "model.layers.21.mlp.experts.94.gate_proj", "model.layers.21.mlp.experts.95.gate_proj", "model.layers.21.mlp.experts.96.gate_proj", "model.layers.21.mlp.experts.97.gate_proj", "model.layers.21.mlp.experts.98.gate_proj", "model.layers.21.mlp.experts.99.gate_proj", "model.layers.21.mlp.experts.100.gate_proj", "model.layers.21.mlp.experts.101.gate_proj", "model.layers.21.mlp.experts.102.gate_proj", "model.layers.21.mlp.experts.103.gate_proj", "model.layers.21.mlp.experts.104.gate_proj", "model.layers.21.mlp.experts.105.gate_proj", "model.layers.21.mlp.experts.106.gate_proj", "model.layers.21.mlp.experts.107.gate_proj", "model.layers.21.mlp.experts.108.gate_proj", "model.layers.21.mlp.experts.109.gate_proj", "model.layers.21.mlp.experts.110.gate_proj", "model.layers.21.mlp.experts.111.gate_proj", "model.layers.21.mlp.experts.112.gate_proj", "model.layers.21.mlp.experts.113.gate_proj", "model.layers.21.mlp.experts.114.gate_proj", "model.layers.21.mlp.experts.115.gate_proj", "model.layers.21.mlp.experts.116.gate_proj", "model.layers.21.mlp.experts.117.gate_proj", "model.layers.21.mlp.experts.118.gate_proj", "model.layers.21.mlp.experts.119.gate_proj", "model.layers.21.mlp.experts.120.gate_proj", "model.layers.21.mlp.experts.121.gate_proj", "model.layers.21.mlp.experts.122.gate_proj", "model.layers.21.mlp.experts.123.gate_proj", "model.layers.21.mlp.experts.124.gate_proj", "model.layers.21.mlp.experts.125.gate_proj", "model.layers.21.mlp.experts.126.gate_proj", "model.layers.21.mlp.experts.127.gate_proj", "model.layers.21.mlp.experts.128.gate_proj", "model.layers.21.mlp.experts.129.gate_proj", "model.layers.21.mlp.experts.130.gate_proj", "model.layers.21.mlp.experts.131.gate_proj", "model.layers.21.mlp.experts.132.gate_proj", "model.layers.21.mlp.experts.133.gate_proj", "model.layers.21.mlp.experts.134.gate_proj", "model.layers.21.mlp.experts.135.gate_proj", "model.layers.21.mlp.experts.136.gate_proj", "model.layers.21.mlp.experts.137.gate_proj", "model.layers.21.mlp.experts.138.gate_proj", "model.layers.21.mlp.experts.139.gate_proj", "model.layers.21.mlp.experts.140.gate_proj", "model.layers.21.mlp.experts.141.gate_proj", "model.layers.21.mlp.experts.142.gate_proj", "model.layers.21.mlp.experts.143.gate_proj", "model.layers.21.mlp.experts.144.gate_proj", "model.layers.21.mlp.experts.145.gate_proj", "model.layers.21.mlp.experts.146.gate_proj", "model.layers.21.mlp.experts.147.gate_proj", "model.layers.21.mlp.experts.148.gate_proj", "model.layers.21.mlp.experts.149.gate_proj", "model.layers.21.mlp.experts.150.gate_proj", "model.layers.21.mlp.experts.151.gate_proj", "model.layers.21.mlp.experts.152.gate_proj", "model.layers.21.mlp.experts.153.gate_proj", "model.layers.21.mlp.experts.154.gate_proj", "model.layers.21.mlp.experts.155.gate_proj", "model.layers.21.mlp.experts.156.gate_proj", "model.layers.21.mlp.experts.157.gate_proj", "model.layers.21.mlp.experts.158.gate_proj", "model.layers.21.mlp.experts.159.gate_proj", "model.layers.21.mlp.experts.0.up_proj", "model.layers.21.mlp.experts.1.up_proj", "model.layers.21.mlp.experts.2.up_proj", "model.layers.21.mlp.experts.3.up_proj", "model.layers.21.mlp.experts.4.up_proj", "model.layers.21.mlp.experts.5.up_proj", "model.layers.21.mlp.experts.6.up_proj", "model.layers.21.mlp.experts.7.up_proj", "model.layers.21.mlp.experts.8.up_proj", "model.layers.21.mlp.experts.9.up_proj", "model.layers.21.mlp.experts.10.up_proj", "model.layers.21.mlp.experts.11.up_proj", "model.layers.21.mlp.experts.12.up_proj", "model.layers.21.mlp.experts.13.up_proj", "model.layers.21.mlp.experts.14.up_proj", "model.layers.21.mlp.experts.15.up_proj", "model.layers.21.mlp.experts.16.up_proj", "model.layers.21.mlp.experts.17.up_proj", "model.layers.21.mlp.experts.18.up_proj", "model.layers.21.mlp.experts.19.up_proj", "model.layers.21.mlp.experts.20.up_proj", "model.layers.21.mlp.experts.21.up_proj", "model.layers.21.mlp.experts.22.up_proj", "model.layers.21.mlp.experts.23.up_proj", "model.layers.21.mlp.experts.24.up_proj", "model.layers.21.mlp.experts.25.up_proj", "model.layers.21.mlp.experts.26.up_proj", "model.layers.21.mlp.experts.27.up_proj", "model.layers.21.mlp.experts.28.up_proj", "model.layers.21.mlp.experts.29.up_proj", "model.layers.21.mlp.experts.30.up_proj", "model.layers.21.mlp.experts.31.up_proj", "model.layers.21.mlp.experts.32.up_proj", "model.layers.21.mlp.experts.33.up_proj", "model.layers.21.mlp.experts.34.up_proj", "model.layers.21.mlp.experts.35.up_proj", "model.layers.21.mlp.experts.36.up_proj", "model.layers.21.mlp.experts.37.up_proj", "model.layers.21.mlp.experts.38.up_proj", "model.layers.21.mlp.experts.39.up_proj", "model.layers.21.mlp.experts.40.up_proj", "model.layers.21.mlp.experts.41.up_proj", "model.layers.21.mlp.experts.42.up_proj", "model.layers.21.mlp.experts.43.up_proj", "model.layers.21.mlp.experts.44.up_proj", "model.layers.21.mlp.experts.45.up_proj", "model.layers.21.mlp.experts.46.up_proj", "model.layers.21.mlp.experts.47.up_proj", "model.layers.21.mlp.experts.48.up_proj", "model.layers.21.mlp.experts.49.up_proj", "model.layers.21.mlp.experts.50.up_proj", "model.layers.21.mlp.experts.51.up_proj", "model.layers.21.mlp.experts.52.up_proj", "model.layers.21.mlp.experts.53.up_proj", "model.layers.21.mlp.experts.54.up_proj", "model.layers.21.mlp.experts.55.up_proj", "model.layers.21.mlp.experts.56.up_proj", "model.layers.21.mlp.experts.57.up_proj", "model.layers.21.mlp.experts.58.up_proj", "model.layers.21.mlp.experts.59.up_proj", "model.layers.21.mlp.experts.60.up_proj", "model.layers.21.mlp.experts.61.up_proj", "model.layers.21.mlp.experts.62.up_proj", "model.layers.21.mlp.experts.63.up_proj", "model.layers.21.mlp.experts.64.up_proj", "model.layers.21.mlp.experts.65.up_proj", "model.layers.21.mlp.experts.66.up_proj", "model.layers.21.mlp.experts.67.up_proj", "model.layers.21.mlp.experts.68.up_proj", "model.layers.21.mlp.experts.69.up_proj", "model.layers.21.mlp.experts.70.up_proj", "model.layers.21.mlp.experts.71.up_proj", "model.layers.21.mlp.experts.72.up_proj", "model.layers.21.mlp.experts.73.up_proj", "model.layers.21.mlp.experts.74.up_proj", "model.layers.21.mlp.experts.75.up_proj", "model.layers.21.mlp.experts.76.up_proj", "model.layers.21.mlp.experts.77.up_proj", "model.layers.21.mlp.experts.78.up_proj", "model.layers.21.mlp.experts.79.up_proj", "model.layers.21.mlp.experts.80.up_proj", "model.layers.21.mlp.experts.81.up_proj", "model.layers.21.mlp.experts.82.up_proj", "model.layers.21.mlp.experts.83.up_proj", "model.layers.21.mlp.experts.84.up_proj", "model.layers.21.mlp.experts.85.up_proj", "model.layers.21.mlp.experts.86.up_proj", "model.layers.21.mlp.experts.87.up_proj", "model.layers.21.mlp.experts.88.up_proj", "model.layers.21.mlp.experts.89.up_proj", "model.layers.21.mlp.experts.90.up_proj", "model.layers.21.mlp.experts.91.up_proj", "model.layers.21.mlp.experts.92.up_proj", "model.layers.21.mlp.experts.93.up_proj", "model.layers.21.mlp.experts.94.up_proj", "model.layers.21.mlp.experts.95.up_proj", "model.layers.21.mlp.experts.96.up_proj", "model.layers.21.mlp.experts.97.up_proj", "model.layers.21.mlp.experts.98.up_proj", "model.layers.21.mlp.experts.99.up_proj", "model.layers.21.mlp.experts.100.up_proj", "model.layers.21.mlp.experts.101.up_proj", "model.layers.21.mlp.experts.102.up_proj", "model.layers.21.mlp.experts.103.up_proj", "model.layers.21.mlp.experts.104.up_proj", "model.layers.21.mlp.experts.105.up_proj", "model.layers.21.mlp.experts.106.up_proj", "model.layers.21.mlp.experts.107.up_proj", "model.layers.21.mlp.experts.108.up_proj", "model.layers.21.mlp.experts.109.up_proj", "model.layers.21.mlp.experts.110.up_proj", "model.layers.21.mlp.experts.111.up_proj", "model.layers.21.mlp.experts.112.up_proj", "model.layers.21.mlp.experts.113.up_proj", "model.layers.21.mlp.experts.114.up_proj", "model.layers.21.mlp.experts.115.up_proj", "model.layers.21.mlp.experts.116.up_proj", "model.layers.21.mlp.experts.117.up_proj", "model.layers.21.mlp.experts.118.up_proj", "model.layers.21.mlp.experts.119.up_proj", "model.layers.21.mlp.experts.120.up_proj", "model.layers.21.mlp.experts.121.up_proj", "model.layers.21.mlp.experts.122.up_proj", "model.layers.21.mlp.experts.123.up_proj", "model.layers.21.mlp.experts.124.up_proj", "model.layers.21.mlp.experts.125.up_proj", "model.layers.21.mlp.experts.126.up_proj", "model.layers.21.mlp.experts.127.up_proj", "model.layers.21.mlp.experts.128.up_proj", "model.layers.21.mlp.experts.129.up_proj", "model.layers.21.mlp.experts.130.up_proj", "model.layers.21.mlp.experts.131.up_proj", "model.layers.21.mlp.experts.132.up_proj", "model.layers.21.mlp.experts.133.up_proj", "model.layers.21.mlp.experts.134.up_proj", "model.layers.21.mlp.experts.135.up_proj", "model.layers.21.mlp.experts.136.up_proj", "model.layers.21.mlp.experts.137.up_proj", "model.layers.21.mlp.experts.138.up_proj", "model.layers.21.mlp.experts.139.up_proj", "model.layers.21.mlp.experts.140.up_proj", "model.layers.21.mlp.experts.141.up_proj", "model.layers.21.mlp.experts.142.up_proj", "model.layers.21.mlp.experts.143.up_proj", "model.layers.21.mlp.experts.144.up_proj", "model.layers.21.mlp.experts.145.up_proj", "model.layers.21.mlp.experts.146.up_proj", "model.layers.21.mlp.experts.147.up_proj", "model.layers.21.mlp.experts.148.up_proj", "model.layers.21.mlp.experts.149.up_proj", "model.layers.21.mlp.experts.150.up_proj", "model.layers.21.mlp.experts.151.up_proj", "model.layers.21.mlp.experts.152.up_proj", "model.layers.21.mlp.experts.153.up_proj", "model.layers.21.mlp.experts.154.up_proj", "model.layers.21.mlp.experts.155.up_proj", "model.layers.21.mlp.experts.156.up_proj", "model.layers.21.mlp.experts.157.up_proj", "model.layers.21.mlp.experts.158.up_proj", "model.layers.21.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.004669029265642166, "dbits": 2516582400 } ] }, { "idx": 128, "layers": [ "model.layers.21.mlp.experts.0.down_proj", "model.layers.21.mlp.experts.1.down_proj", "model.layers.21.mlp.experts.2.down_proj", "model.layers.21.mlp.experts.3.down_proj", "model.layers.21.mlp.experts.4.down_proj", "model.layers.21.mlp.experts.5.down_proj", "model.layers.21.mlp.experts.6.down_proj", "model.layers.21.mlp.experts.7.down_proj", "model.layers.21.mlp.experts.8.down_proj", "model.layers.21.mlp.experts.9.down_proj", "model.layers.21.mlp.experts.10.down_proj", "model.layers.21.mlp.experts.11.down_proj", "model.layers.21.mlp.experts.12.down_proj", "model.layers.21.mlp.experts.13.down_proj", "model.layers.21.mlp.experts.14.down_proj", "model.layers.21.mlp.experts.15.down_proj", "model.layers.21.mlp.experts.16.down_proj", "model.layers.21.mlp.experts.17.down_proj", "model.layers.21.mlp.experts.18.down_proj", "model.layers.21.mlp.experts.19.down_proj", "model.layers.21.mlp.experts.20.down_proj", "model.layers.21.mlp.experts.21.down_proj", "model.layers.21.mlp.experts.22.down_proj", "model.layers.21.mlp.experts.23.down_proj", "model.layers.21.mlp.experts.24.down_proj", "model.layers.21.mlp.experts.25.down_proj", "model.layers.21.mlp.experts.26.down_proj", "model.layers.21.mlp.experts.27.down_proj", "model.layers.21.mlp.experts.28.down_proj", "model.layers.21.mlp.experts.29.down_proj", "model.layers.21.mlp.experts.30.down_proj", "model.layers.21.mlp.experts.31.down_proj", "model.layers.21.mlp.experts.32.down_proj", "model.layers.21.mlp.experts.33.down_proj", "model.layers.21.mlp.experts.34.down_proj", "model.layers.21.mlp.experts.35.down_proj", "model.layers.21.mlp.experts.36.down_proj", "model.layers.21.mlp.experts.37.down_proj", "model.layers.21.mlp.experts.38.down_proj", "model.layers.21.mlp.experts.39.down_proj", "model.layers.21.mlp.experts.40.down_proj", "model.layers.21.mlp.experts.41.down_proj", "model.layers.21.mlp.experts.42.down_proj", "model.layers.21.mlp.experts.43.down_proj", "model.layers.21.mlp.experts.44.down_proj", "model.layers.21.mlp.experts.45.down_proj", "model.layers.21.mlp.experts.46.down_proj", "model.layers.21.mlp.experts.47.down_proj", "model.layers.21.mlp.experts.48.down_proj", "model.layers.21.mlp.experts.49.down_proj", "model.layers.21.mlp.experts.50.down_proj", "model.layers.21.mlp.experts.51.down_proj", "model.layers.21.mlp.experts.52.down_proj", "model.layers.21.mlp.experts.53.down_proj", "model.layers.21.mlp.experts.54.down_proj", "model.layers.21.mlp.experts.55.down_proj", "model.layers.21.mlp.experts.56.down_proj", "model.layers.21.mlp.experts.57.down_proj", "model.layers.21.mlp.experts.58.down_proj", "model.layers.21.mlp.experts.59.down_proj", "model.layers.21.mlp.experts.60.down_proj", "model.layers.21.mlp.experts.61.down_proj", "model.layers.21.mlp.experts.62.down_proj", "model.layers.21.mlp.experts.63.down_proj", "model.layers.21.mlp.experts.64.down_proj", "model.layers.21.mlp.experts.65.down_proj", "model.layers.21.mlp.experts.66.down_proj", "model.layers.21.mlp.experts.67.down_proj", "model.layers.21.mlp.experts.68.down_proj", "model.layers.21.mlp.experts.69.down_proj", "model.layers.21.mlp.experts.70.down_proj", "model.layers.21.mlp.experts.71.down_proj", "model.layers.21.mlp.experts.72.down_proj", "model.layers.21.mlp.experts.73.down_proj", "model.layers.21.mlp.experts.74.down_proj", "model.layers.21.mlp.experts.75.down_proj", "model.layers.21.mlp.experts.76.down_proj", "model.layers.21.mlp.experts.77.down_proj", "model.layers.21.mlp.experts.78.down_proj", "model.layers.21.mlp.experts.79.down_proj", "model.layers.21.mlp.experts.80.down_proj", "model.layers.21.mlp.experts.81.down_proj", "model.layers.21.mlp.experts.82.down_proj", "model.layers.21.mlp.experts.83.down_proj", "model.layers.21.mlp.experts.84.down_proj", "model.layers.21.mlp.experts.85.down_proj", "model.layers.21.mlp.experts.86.down_proj", "model.layers.21.mlp.experts.87.down_proj", "model.layers.21.mlp.experts.88.down_proj", "model.layers.21.mlp.experts.89.down_proj", "model.layers.21.mlp.experts.90.down_proj", "model.layers.21.mlp.experts.91.down_proj", "model.layers.21.mlp.experts.92.down_proj", "model.layers.21.mlp.experts.93.down_proj", "model.layers.21.mlp.experts.94.down_proj", "model.layers.21.mlp.experts.95.down_proj", "model.layers.21.mlp.experts.96.down_proj", "model.layers.21.mlp.experts.97.down_proj", "model.layers.21.mlp.experts.98.down_proj", "model.layers.21.mlp.experts.99.down_proj", "model.layers.21.mlp.experts.100.down_proj", "model.layers.21.mlp.experts.101.down_proj", "model.layers.21.mlp.experts.102.down_proj", "model.layers.21.mlp.experts.103.down_proj", "model.layers.21.mlp.experts.104.down_proj", "model.layers.21.mlp.experts.105.down_proj", "model.layers.21.mlp.experts.106.down_proj", "model.layers.21.mlp.experts.107.down_proj", "model.layers.21.mlp.experts.108.down_proj", "model.layers.21.mlp.experts.109.down_proj", "model.layers.21.mlp.experts.110.down_proj", "model.layers.21.mlp.experts.111.down_proj", "model.layers.21.mlp.experts.112.down_proj", "model.layers.21.mlp.experts.113.down_proj", "model.layers.21.mlp.experts.114.down_proj", "model.layers.21.mlp.experts.115.down_proj", "model.layers.21.mlp.experts.116.down_proj", "model.layers.21.mlp.experts.117.down_proj", "model.layers.21.mlp.experts.118.down_proj", "model.layers.21.mlp.experts.119.down_proj", "model.layers.21.mlp.experts.120.down_proj", "model.layers.21.mlp.experts.121.down_proj", "model.layers.21.mlp.experts.122.down_proj", "model.layers.21.mlp.experts.123.down_proj", "model.layers.21.mlp.experts.124.down_proj", "model.layers.21.mlp.experts.125.down_proj", "model.layers.21.mlp.experts.126.down_proj", "model.layers.21.mlp.experts.127.down_proj", "model.layers.21.mlp.experts.128.down_proj", "model.layers.21.mlp.experts.129.down_proj", "model.layers.21.mlp.experts.130.down_proj", "model.layers.21.mlp.experts.131.down_proj", "model.layers.21.mlp.experts.132.down_proj", "model.layers.21.mlp.experts.133.down_proj", "model.layers.21.mlp.experts.134.down_proj", "model.layers.21.mlp.experts.135.down_proj", "model.layers.21.mlp.experts.136.down_proj", "model.layers.21.mlp.experts.137.down_proj", "model.layers.21.mlp.experts.138.down_proj", "model.layers.21.mlp.experts.139.down_proj", "model.layers.21.mlp.experts.140.down_proj", "model.layers.21.mlp.experts.141.down_proj", "model.layers.21.mlp.experts.142.down_proj", "model.layers.21.mlp.experts.143.down_proj", "model.layers.21.mlp.experts.144.down_proj", "model.layers.21.mlp.experts.145.down_proj", "model.layers.21.mlp.experts.146.down_proj", "model.layers.21.mlp.experts.147.down_proj", "model.layers.21.mlp.experts.148.down_proj", "model.layers.21.mlp.experts.149.down_proj", "model.layers.21.mlp.experts.150.down_proj", "model.layers.21.mlp.experts.151.down_proj", "model.layers.21.mlp.experts.152.down_proj", "model.layers.21.mlp.experts.153.down_proj", "model.layers.21.mlp.experts.154.down_proj", "model.layers.21.mlp.experts.155.down_proj", "model.layers.21.mlp.experts.156.down_proj", "model.layers.21.mlp.experts.157.down_proj", "model.layers.21.mlp.experts.158.down_proj", "model.layers.21.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0025527045130729897, "dbits": 1258291200 } ] }, { "idx": 129, "layers": [ "model.layers.22.self_attn.q_proj" ], "candidates": [ { "dkld": 0.00029709935188293457, "dbits": 62914560 } ] }, { "idx": 130, "layers": [ "model.layers.22.self_attn.k_proj", "model.layers.22.self_attn.v_proj" ], "candidates": [ { "dkld": -0.00047025084495544434, "dbits": 10485760 } ] }, { "idx": 131, "layers": [ "model.layers.22.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00015229806303979077, "dbits": 62914560 } ] }, { "idx": 132, "layers": [ "model.layers.22.mlp.shared_experts.gate_proj", "model.layers.22.mlp.shared_experts.up_proj", "model.layers.22.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0033725596964359617, "dbits": 23592960 } ] }, { "idx": 133, "layers": [ "model.layers.22.mlp.experts.0.gate_proj", "model.layers.22.mlp.experts.1.gate_proj", "model.layers.22.mlp.experts.2.gate_proj", "model.layers.22.mlp.experts.3.gate_proj", "model.layers.22.mlp.experts.4.gate_proj", "model.layers.22.mlp.experts.5.gate_proj", "model.layers.22.mlp.experts.6.gate_proj", "model.layers.22.mlp.experts.7.gate_proj", "model.layers.22.mlp.experts.8.gate_proj", "model.layers.22.mlp.experts.9.gate_proj", "model.layers.22.mlp.experts.10.gate_proj", "model.layers.22.mlp.experts.11.gate_proj", "model.layers.22.mlp.experts.12.gate_proj", "model.layers.22.mlp.experts.13.gate_proj", "model.layers.22.mlp.experts.14.gate_proj", "model.layers.22.mlp.experts.15.gate_proj", "model.layers.22.mlp.experts.16.gate_proj", "model.layers.22.mlp.experts.17.gate_proj", "model.layers.22.mlp.experts.18.gate_proj", "model.layers.22.mlp.experts.19.gate_proj", "model.layers.22.mlp.experts.20.gate_proj", "model.layers.22.mlp.experts.21.gate_proj", "model.layers.22.mlp.experts.22.gate_proj", "model.layers.22.mlp.experts.23.gate_proj", "model.layers.22.mlp.experts.24.gate_proj", "model.layers.22.mlp.experts.25.gate_proj", "model.layers.22.mlp.experts.26.gate_proj", "model.layers.22.mlp.experts.27.gate_proj", "model.layers.22.mlp.experts.28.gate_proj", "model.layers.22.mlp.experts.29.gate_proj", "model.layers.22.mlp.experts.30.gate_proj", "model.layers.22.mlp.experts.31.gate_proj", "model.layers.22.mlp.experts.32.gate_proj", "model.layers.22.mlp.experts.33.gate_proj", "model.layers.22.mlp.experts.34.gate_proj", "model.layers.22.mlp.experts.35.gate_proj", "model.layers.22.mlp.experts.36.gate_proj", "model.layers.22.mlp.experts.37.gate_proj", "model.layers.22.mlp.experts.38.gate_proj", "model.layers.22.mlp.experts.39.gate_proj", "model.layers.22.mlp.experts.40.gate_proj", "model.layers.22.mlp.experts.41.gate_proj", "model.layers.22.mlp.experts.42.gate_proj", "model.layers.22.mlp.experts.43.gate_proj", "model.layers.22.mlp.experts.44.gate_proj", "model.layers.22.mlp.experts.45.gate_proj", "model.layers.22.mlp.experts.46.gate_proj", "model.layers.22.mlp.experts.47.gate_proj", "model.layers.22.mlp.experts.48.gate_proj", "model.layers.22.mlp.experts.49.gate_proj", "model.layers.22.mlp.experts.50.gate_proj", "model.layers.22.mlp.experts.51.gate_proj", "model.layers.22.mlp.experts.52.gate_proj", "model.layers.22.mlp.experts.53.gate_proj", "model.layers.22.mlp.experts.54.gate_proj", "model.layers.22.mlp.experts.55.gate_proj", "model.layers.22.mlp.experts.56.gate_proj", "model.layers.22.mlp.experts.57.gate_proj", "model.layers.22.mlp.experts.58.gate_proj", "model.layers.22.mlp.experts.59.gate_proj", "model.layers.22.mlp.experts.60.gate_proj", "model.layers.22.mlp.experts.61.gate_proj", "model.layers.22.mlp.experts.62.gate_proj", "model.layers.22.mlp.experts.63.gate_proj", "model.layers.22.mlp.experts.64.gate_proj", "model.layers.22.mlp.experts.65.gate_proj", "model.layers.22.mlp.experts.66.gate_proj", "model.layers.22.mlp.experts.67.gate_proj", "model.layers.22.mlp.experts.68.gate_proj", "model.layers.22.mlp.experts.69.gate_proj", "model.layers.22.mlp.experts.70.gate_proj", "model.layers.22.mlp.experts.71.gate_proj", "model.layers.22.mlp.experts.72.gate_proj", "model.layers.22.mlp.experts.73.gate_proj", "model.layers.22.mlp.experts.74.gate_proj", "model.layers.22.mlp.experts.75.gate_proj", "model.layers.22.mlp.experts.76.gate_proj", "model.layers.22.mlp.experts.77.gate_proj", "model.layers.22.mlp.experts.78.gate_proj", "model.layers.22.mlp.experts.79.gate_proj", "model.layers.22.mlp.experts.80.gate_proj", "model.layers.22.mlp.experts.81.gate_proj", "model.layers.22.mlp.experts.82.gate_proj", "model.layers.22.mlp.experts.83.gate_proj", "model.layers.22.mlp.experts.84.gate_proj", "model.layers.22.mlp.experts.85.gate_proj", "model.layers.22.mlp.experts.86.gate_proj", "model.layers.22.mlp.experts.87.gate_proj", "model.layers.22.mlp.experts.88.gate_proj", "model.layers.22.mlp.experts.89.gate_proj", "model.layers.22.mlp.experts.90.gate_proj", "model.layers.22.mlp.experts.91.gate_proj", "model.layers.22.mlp.experts.92.gate_proj", "model.layers.22.mlp.experts.93.gate_proj", "model.layers.22.mlp.experts.94.gate_proj", "model.layers.22.mlp.experts.95.gate_proj", "model.layers.22.mlp.experts.96.gate_proj", "model.layers.22.mlp.experts.97.gate_proj", "model.layers.22.mlp.experts.98.gate_proj", "model.layers.22.mlp.experts.99.gate_proj", "model.layers.22.mlp.experts.100.gate_proj", "model.layers.22.mlp.experts.101.gate_proj", "model.layers.22.mlp.experts.102.gate_proj", "model.layers.22.mlp.experts.103.gate_proj", "model.layers.22.mlp.experts.104.gate_proj", "model.layers.22.mlp.experts.105.gate_proj", "model.layers.22.mlp.experts.106.gate_proj", "model.layers.22.mlp.experts.107.gate_proj", "model.layers.22.mlp.experts.108.gate_proj", "model.layers.22.mlp.experts.109.gate_proj", "model.layers.22.mlp.experts.110.gate_proj", "model.layers.22.mlp.experts.111.gate_proj", "model.layers.22.mlp.experts.112.gate_proj", "model.layers.22.mlp.experts.113.gate_proj", "model.layers.22.mlp.experts.114.gate_proj", "model.layers.22.mlp.experts.115.gate_proj", "model.layers.22.mlp.experts.116.gate_proj", "model.layers.22.mlp.experts.117.gate_proj", "model.layers.22.mlp.experts.118.gate_proj", "model.layers.22.mlp.experts.119.gate_proj", "model.layers.22.mlp.experts.120.gate_proj", "model.layers.22.mlp.experts.121.gate_proj", "model.layers.22.mlp.experts.122.gate_proj", "model.layers.22.mlp.experts.123.gate_proj", "model.layers.22.mlp.experts.124.gate_proj", "model.layers.22.mlp.experts.125.gate_proj", "model.layers.22.mlp.experts.126.gate_proj", "model.layers.22.mlp.experts.127.gate_proj", "model.layers.22.mlp.experts.128.gate_proj", "model.layers.22.mlp.experts.129.gate_proj", "model.layers.22.mlp.experts.130.gate_proj", "model.layers.22.mlp.experts.131.gate_proj", "model.layers.22.mlp.experts.132.gate_proj", "model.layers.22.mlp.experts.133.gate_proj", "model.layers.22.mlp.experts.134.gate_proj", "model.layers.22.mlp.experts.135.gate_proj", "model.layers.22.mlp.experts.136.gate_proj", "model.layers.22.mlp.experts.137.gate_proj", "model.layers.22.mlp.experts.138.gate_proj", "model.layers.22.mlp.experts.139.gate_proj", "model.layers.22.mlp.experts.140.gate_proj", "model.layers.22.mlp.experts.141.gate_proj", "model.layers.22.mlp.experts.142.gate_proj", "model.layers.22.mlp.experts.143.gate_proj", "model.layers.22.mlp.experts.144.gate_proj", "model.layers.22.mlp.experts.145.gate_proj", "model.layers.22.mlp.experts.146.gate_proj", "model.layers.22.mlp.experts.147.gate_proj", "model.layers.22.mlp.experts.148.gate_proj", "model.layers.22.mlp.experts.149.gate_proj", "model.layers.22.mlp.experts.150.gate_proj", "model.layers.22.mlp.experts.151.gate_proj", "model.layers.22.mlp.experts.152.gate_proj", "model.layers.22.mlp.experts.153.gate_proj", "model.layers.22.mlp.experts.154.gate_proj", "model.layers.22.mlp.experts.155.gate_proj", "model.layers.22.mlp.experts.156.gate_proj", "model.layers.22.mlp.experts.157.gate_proj", "model.layers.22.mlp.experts.158.gate_proj", "model.layers.22.mlp.experts.159.gate_proj", "model.layers.22.mlp.experts.0.up_proj", "model.layers.22.mlp.experts.1.up_proj", "model.layers.22.mlp.experts.2.up_proj", "model.layers.22.mlp.experts.3.up_proj", "model.layers.22.mlp.experts.4.up_proj", "model.layers.22.mlp.experts.5.up_proj", "model.layers.22.mlp.experts.6.up_proj", "model.layers.22.mlp.experts.7.up_proj", "model.layers.22.mlp.experts.8.up_proj", "model.layers.22.mlp.experts.9.up_proj", "model.layers.22.mlp.experts.10.up_proj", "model.layers.22.mlp.experts.11.up_proj", "model.layers.22.mlp.experts.12.up_proj", "model.layers.22.mlp.experts.13.up_proj", "model.layers.22.mlp.experts.14.up_proj", "model.layers.22.mlp.experts.15.up_proj", "model.layers.22.mlp.experts.16.up_proj", "model.layers.22.mlp.experts.17.up_proj", "model.layers.22.mlp.experts.18.up_proj", "model.layers.22.mlp.experts.19.up_proj", "model.layers.22.mlp.experts.20.up_proj", "model.layers.22.mlp.experts.21.up_proj", "model.layers.22.mlp.experts.22.up_proj", "model.layers.22.mlp.experts.23.up_proj", "model.layers.22.mlp.experts.24.up_proj", "model.layers.22.mlp.experts.25.up_proj", "model.layers.22.mlp.experts.26.up_proj", "model.layers.22.mlp.experts.27.up_proj", "model.layers.22.mlp.experts.28.up_proj", "model.layers.22.mlp.experts.29.up_proj", "model.layers.22.mlp.experts.30.up_proj", "model.layers.22.mlp.experts.31.up_proj", "model.layers.22.mlp.experts.32.up_proj", "model.layers.22.mlp.experts.33.up_proj", "model.layers.22.mlp.experts.34.up_proj", "model.layers.22.mlp.experts.35.up_proj", "model.layers.22.mlp.experts.36.up_proj", "model.layers.22.mlp.experts.37.up_proj", "model.layers.22.mlp.experts.38.up_proj", "model.layers.22.mlp.experts.39.up_proj", "model.layers.22.mlp.experts.40.up_proj", "model.layers.22.mlp.experts.41.up_proj", "model.layers.22.mlp.experts.42.up_proj", "model.layers.22.mlp.experts.43.up_proj", "model.layers.22.mlp.experts.44.up_proj", "model.layers.22.mlp.experts.45.up_proj", "model.layers.22.mlp.experts.46.up_proj", "model.layers.22.mlp.experts.47.up_proj", "model.layers.22.mlp.experts.48.up_proj", "model.layers.22.mlp.experts.49.up_proj", "model.layers.22.mlp.experts.50.up_proj", "model.layers.22.mlp.experts.51.up_proj", "model.layers.22.mlp.experts.52.up_proj", "model.layers.22.mlp.experts.53.up_proj", "model.layers.22.mlp.experts.54.up_proj", "model.layers.22.mlp.experts.55.up_proj", "model.layers.22.mlp.experts.56.up_proj", "model.layers.22.mlp.experts.57.up_proj", "model.layers.22.mlp.experts.58.up_proj", "model.layers.22.mlp.experts.59.up_proj", "model.layers.22.mlp.experts.60.up_proj", "model.layers.22.mlp.experts.61.up_proj", "model.layers.22.mlp.experts.62.up_proj", "model.layers.22.mlp.experts.63.up_proj", "model.layers.22.mlp.experts.64.up_proj", "model.layers.22.mlp.experts.65.up_proj", "model.layers.22.mlp.experts.66.up_proj", "model.layers.22.mlp.experts.67.up_proj", "model.layers.22.mlp.experts.68.up_proj", "model.layers.22.mlp.experts.69.up_proj", "model.layers.22.mlp.experts.70.up_proj", "model.layers.22.mlp.experts.71.up_proj", "model.layers.22.mlp.experts.72.up_proj", "model.layers.22.mlp.experts.73.up_proj", "model.layers.22.mlp.experts.74.up_proj", "model.layers.22.mlp.experts.75.up_proj", "model.layers.22.mlp.experts.76.up_proj", "model.layers.22.mlp.experts.77.up_proj", "model.layers.22.mlp.experts.78.up_proj", "model.layers.22.mlp.experts.79.up_proj", "model.layers.22.mlp.experts.80.up_proj", "model.layers.22.mlp.experts.81.up_proj", "model.layers.22.mlp.experts.82.up_proj", "model.layers.22.mlp.experts.83.up_proj", "model.layers.22.mlp.experts.84.up_proj", "model.layers.22.mlp.experts.85.up_proj", "model.layers.22.mlp.experts.86.up_proj", "model.layers.22.mlp.experts.87.up_proj", "model.layers.22.mlp.experts.88.up_proj", "model.layers.22.mlp.experts.89.up_proj", "model.layers.22.mlp.experts.90.up_proj", "model.layers.22.mlp.experts.91.up_proj", "model.layers.22.mlp.experts.92.up_proj", "model.layers.22.mlp.experts.93.up_proj", "model.layers.22.mlp.experts.94.up_proj", "model.layers.22.mlp.experts.95.up_proj", "model.layers.22.mlp.experts.96.up_proj", "model.layers.22.mlp.experts.97.up_proj", "model.layers.22.mlp.experts.98.up_proj", "model.layers.22.mlp.experts.99.up_proj", "model.layers.22.mlp.experts.100.up_proj", "model.layers.22.mlp.experts.101.up_proj", "model.layers.22.mlp.experts.102.up_proj", "model.layers.22.mlp.experts.103.up_proj", "model.layers.22.mlp.experts.104.up_proj", "model.layers.22.mlp.experts.105.up_proj", "model.layers.22.mlp.experts.106.up_proj", "model.layers.22.mlp.experts.107.up_proj", "model.layers.22.mlp.experts.108.up_proj", "model.layers.22.mlp.experts.109.up_proj", "model.layers.22.mlp.experts.110.up_proj", "model.layers.22.mlp.experts.111.up_proj", "model.layers.22.mlp.experts.112.up_proj", "model.layers.22.mlp.experts.113.up_proj", "model.layers.22.mlp.experts.114.up_proj", "model.layers.22.mlp.experts.115.up_proj", "model.layers.22.mlp.experts.116.up_proj", "model.layers.22.mlp.experts.117.up_proj", "model.layers.22.mlp.experts.118.up_proj", "model.layers.22.mlp.experts.119.up_proj", "model.layers.22.mlp.experts.120.up_proj", "model.layers.22.mlp.experts.121.up_proj", "model.layers.22.mlp.experts.122.up_proj", "model.layers.22.mlp.experts.123.up_proj", "model.layers.22.mlp.experts.124.up_proj", "model.layers.22.mlp.experts.125.up_proj", "model.layers.22.mlp.experts.126.up_proj", "model.layers.22.mlp.experts.127.up_proj", "model.layers.22.mlp.experts.128.up_proj", "model.layers.22.mlp.experts.129.up_proj", "model.layers.22.mlp.experts.130.up_proj", "model.layers.22.mlp.experts.131.up_proj", "model.layers.22.mlp.experts.132.up_proj", "model.layers.22.mlp.experts.133.up_proj", "model.layers.22.mlp.experts.134.up_proj", "model.layers.22.mlp.experts.135.up_proj", "model.layers.22.mlp.experts.136.up_proj", "model.layers.22.mlp.experts.137.up_proj", "model.layers.22.mlp.experts.138.up_proj", "model.layers.22.mlp.experts.139.up_proj", "model.layers.22.mlp.experts.140.up_proj", "model.layers.22.mlp.experts.141.up_proj", "model.layers.22.mlp.experts.142.up_proj", "model.layers.22.mlp.experts.143.up_proj", "model.layers.22.mlp.experts.144.up_proj", "model.layers.22.mlp.experts.145.up_proj", "model.layers.22.mlp.experts.146.up_proj", "model.layers.22.mlp.experts.147.up_proj", "model.layers.22.mlp.experts.148.up_proj", "model.layers.22.mlp.experts.149.up_proj", "model.layers.22.mlp.experts.150.up_proj", "model.layers.22.mlp.experts.151.up_proj", "model.layers.22.mlp.experts.152.up_proj", "model.layers.22.mlp.experts.153.up_proj", "model.layers.22.mlp.experts.154.up_proj", "model.layers.22.mlp.experts.155.up_proj", "model.layers.22.mlp.experts.156.up_proj", "model.layers.22.mlp.experts.157.up_proj", "model.layers.22.mlp.experts.158.up_proj", "model.layers.22.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0026817500591277965, "dbits": 2516582400 } ] }, { "idx": 134, "layers": [ "model.layers.22.mlp.experts.0.down_proj", "model.layers.22.mlp.experts.1.down_proj", "model.layers.22.mlp.experts.2.down_proj", "model.layers.22.mlp.experts.3.down_proj", "model.layers.22.mlp.experts.4.down_proj", "model.layers.22.mlp.experts.5.down_proj", "model.layers.22.mlp.experts.6.down_proj", "model.layers.22.mlp.experts.7.down_proj", "model.layers.22.mlp.experts.8.down_proj", "model.layers.22.mlp.experts.9.down_proj", "model.layers.22.mlp.experts.10.down_proj", "model.layers.22.mlp.experts.11.down_proj", "model.layers.22.mlp.experts.12.down_proj", "model.layers.22.mlp.experts.13.down_proj", "model.layers.22.mlp.experts.14.down_proj", "model.layers.22.mlp.experts.15.down_proj", "model.layers.22.mlp.experts.16.down_proj", "model.layers.22.mlp.experts.17.down_proj", "model.layers.22.mlp.experts.18.down_proj", "model.layers.22.mlp.experts.19.down_proj", "model.layers.22.mlp.experts.20.down_proj", "model.layers.22.mlp.experts.21.down_proj", "model.layers.22.mlp.experts.22.down_proj", "model.layers.22.mlp.experts.23.down_proj", "model.layers.22.mlp.experts.24.down_proj", "model.layers.22.mlp.experts.25.down_proj", "model.layers.22.mlp.experts.26.down_proj", "model.layers.22.mlp.experts.27.down_proj", "model.layers.22.mlp.experts.28.down_proj", "model.layers.22.mlp.experts.29.down_proj", "model.layers.22.mlp.experts.30.down_proj", "model.layers.22.mlp.experts.31.down_proj", "model.layers.22.mlp.experts.32.down_proj", "model.layers.22.mlp.experts.33.down_proj", "model.layers.22.mlp.experts.34.down_proj", "model.layers.22.mlp.experts.35.down_proj", "model.layers.22.mlp.experts.36.down_proj", "model.layers.22.mlp.experts.37.down_proj", "model.layers.22.mlp.experts.38.down_proj", "model.layers.22.mlp.experts.39.down_proj", "model.layers.22.mlp.experts.40.down_proj", "model.layers.22.mlp.experts.41.down_proj", "model.layers.22.mlp.experts.42.down_proj", "model.layers.22.mlp.experts.43.down_proj", "model.layers.22.mlp.experts.44.down_proj", "model.layers.22.mlp.experts.45.down_proj", "model.layers.22.mlp.experts.46.down_proj", "model.layers.22.mlp.experts.47.down_proj", "model.layers.22.mlp.experts.48.down_proj", "model.layers.22.mlp.experts.49.down_proj", "model.layers.22.mlp.experts.50.down_proj", "model.layers.22.mlp.experts.51.down_proj", "model.layers.22.mlp.experts.52.down_proj", "model.layers.22.mlp.experts.53.down_proj", "model.layers.22.mlp.experts.54.down_proj", "model.layers.22.mlp.experts.55.down_proj", "model.layers.22.mlp.experts.56.down_proj", "model.layers.22.mlp.experts.57.down_proj", "model.layers.22.mlp.experts.58.down_proj", "model.layers.22.mlp.experts.59.down_proj", "model.layers.22.mlp.experts.60.down_proj", "model.layers.22.mlp.experts.61.down_proj", "model.layers.22.mlp.experts.62.down_proj", "model.layers.22.mlp.experts.63.down_proj", "model.layers.22.mlp.experts.64.down_proj", "model.layers.22.mlp.experts.65.down_proj", "model.layers.22.mlp.experts.66.down_proj", "model.layers.22.mlp.experts.67.down_proj", "model.layers.22.mlp.experts.68.down_proj", "model.layers.22.mlp.experts.69.down_proj", "model.layers.22.mlp.experts.70.down_proj", "model.layers.22.mlp.experts.71.down_proj", "model.layers.22.mlp.experts.72.down_proj", "model.layers.22.mlp.experts.73.down_proj", "model.layers.22.mlp.experts.74.down_proj", "model.layers.22.mlp.experts.75.down_proj", "model.layers.22.mlp.experts.76.down_proj", "model.layers.22.mlp.experts.77.down_proj", "model.layers.22.mlp.experts.78.down_proj", "model.layers.22.mlp.experts.79.down_proj", "model.layers.22.mlp.experts.80.down_proj", "model.layers.22.mlp.experts.81.down_proj", "model.layers.22.mlp.experts.82.down_proj", "model.layers.22.mlp.experts.83.down_proj", "model.layers.22.mlp.experts.84.down_proj", "model.layers.22.mlp.experts.85.down_proj", "model.layers.22.mlp.experts.86.down_proj", "model.layers.22.mlp.experts.87.down_proj", "model.layers.22.mlp.experts.88.down_proj", "model.layers.22.mlp.experts.89.down_proj", "model.layers.22.mlp.experts.90.down_proj", "model.layers.22.mlp.experts.91.down_proj", "model.layers.22.mlp.experts.92.down_proj", "model.layers.22.mlp.experts.93.down_proj", "model.layers.22.mlp.experts.94.down_proj", "model.layers.22.mlp.experts.95.down_proj", "model.layers.22.mlp.experts.96.down_proj", "model.layers.22.mlp.experts.97.down_proj", "model.layers.22.mlp.experts.98.down_proj", "model.layers.22.mlp.experts.99.down_proj", "model.layers.22.mlp.experts.100.down_proj", "model.layers.22.mlp.experts.101.down_proj", "model.layers.22.mlp.experts.102.down_proj", "model.layers.22.mlp.experts.103.down_proj", "model.layers.22.mlp.experts.104.down_proj", "model.layers.22.mlp.experts.105.down_proj", "model.layers.22.mlp.experts.106.down_proj", "model.layers.22.mlp.experts.107.down_proj", "model.layers.22.mlp.experts.108.down_proj", "model.layers.22.mlp.experts.109.down_proj", "model.layers.22.mlp.experts.110.down_proj", "model.layers.22.mlp.experts.111.down_proj", "model.layers.22.mlp.experts.112.down_proj", "model.layers.22.mlp.experts.113.down_proj", "model.layers.22.mlp.experts.114.down_proj", "model.layers.22.mlp.experts.115.down_proj", "model.layers.22.mlp.experts.116.down_proj", "model.layers.22.mlp.experts.117.down_proj", "model.layers.22.mlp.experts.118.down_proj", "model.layers.22.mlp.experts.119.down_proj", "model.layers.22.mlp.experts.120.down_proj", "model.layers.22.mlp.experts.121.down_proj", "model.layers.22.mlp.experts.122.down_proj", "model.layers.22.mlp.experts.123.down_proj", "model.layers.22.mlp.experts.124.down_proj", "model.layers.22.mlp.experts.125.down_proj", "model.layers.22.mlp.experts.126.down_proj", "model.layers.22.mlp.experts.127.down_proj", "model.layers.22.mlp.experts.128.down_proj", "model.layers.22.mlp.experts.129.down_proj", "model.layers.22.mlp.experts.130.down_proj", "model.layers.22.mlp.experts.131.down_proj", "model.layers.22.mlp.experts.132.down_proj", "model.layers.22.mlp.experts.133.down_proj", "model.layers.22.mlp.experts.134.down_proj", "model.layers.22.mlp.experts.135.down_proj", "model.layers.22.mlp.experts.136.down_proj", "model.layers.22.mlp.experts.137.down_proj", "model.layers.22.mlp.experts.138.down_proj", "model.layers.22.mlp.experts.139.down_proj", "model.layers.22.mlp.experts.140.down_proj", "model.layers.22.mlp.experts.141.down_proj", "model.layers.22.mlp.experts.142.down_proj", "model.layers.22.mlp.experts.143.down_proj", "model.layers.22.mlp.experts.144.down_proj", "model.layers.22.mlp.experts.145.down_proj", "model.layers.22.mlp.experts.146.down_proj", "model.layers.22.mlp.experts.147.down_proj", "model.layers.22.mlp.experts.148.down_proj", "model.layers.22.mlp.experts.149.down_proj", "model.layers.22.mlp.experts.150.down_proj", "model.layers.22.mlp.experts.151.down_proj", "model.layers.22.mlp.experts.152.down_proj", "model.layers.22.mlp.experts.153.down_proj", "model.layers.22.mlp.experts.154.down_proj", "model.layers.22.mlp.experts.155.down_proj", "model.layers.22.mlp.experts.156.down_proj", "model.layers.22.mlp.experts.157.down_proj", "model.layers.22.mlp.experts.158.down_proj", "model.layers.22.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002654716372489929, "dbits": 1258291200 } ] }, { "idx": 135, "layers": [ "model.layers.23.self_attn.q_proj" ], "candidates": [ { "dkld": -0.003228016197681427, "dbits": 62914560 } ] }, { "idx": 136, "layers": [ "model.layers.23.self_attn.k_proj", "model.layers.23.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0026565581560134555, "dbits": 10485760 } ] }, { "idx": 137, "layers": [ "model.layers.23.self_attn.o_proj" ], "candidates": [ { "dkld": -0.000502978265285503, "dbits": 62914560 } ] }, { "idx": 138, "layers": [ "model.layers.23.mlp.shared_experts.gate_proj", "model.layers.23.mlp.shared_experts.up_proj", "model.layers.23.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.006236594915390026, "dbits": 23592960 } ] }, { "idx": 139, "layers": [ "model.layers.23.mlp.experts.0.gate_proj", "model.layers.23.mlp.experts.1.gate_proj", "model.layers.23.mlp.experts.2.gate_proj", "model.layers.23.mlp.experts.3.gate_proj", "model.layers.23.mlp.experts.4.gate_proj", "model.layers.23.mlp.experts.5.gate_proj", "model.layers.23.mlp.experts.6.gate_proj", "model.layers.23.mlp.experts.7.gate_proj", "model.layers.23.mlp.experts.8.gate_proj", "model.layers.23.mlp.experts.9.gate_proj", "model.layers.23.mlp.experts.10.gate_proj", "model.layers.23.mlp.experts.11.gate_proj", "model.layers.23.mlp.experts.12.gate_proj", "model.layers.23.mlp.experts.13.gate_proj", "model.layers.23.mlp.experts.14.gate_proj", "model.layers.23.mlp.experts.15.gate_proj", "model.layers.23.mlp.experts.16.gate_proj", "model.layers.23.mlp.experts.17.gate_proj", "model.layers.23.mlp.experts.18.gate_proj", "model.layers.23.mlp.experts.19.gate_proj", "model.layers.23.mlp.experts.20.gate_proj", "model.layers.23.mlp.experts.21.gate_proj", "model.layers.23.mlp.experts.22.gate_proj", "model.layers.23.mlp.experts.23.gate_proj", "model.layers.23.mlp.experts.24.gate_proj", "model.layers.23.mlp.experts.25.gate_proj", "model.layers.23.mlp.experts.26.gate_proj", "model.layers.23.mlp.experts.27.gate_proj", "model.layers.23.mlp.experts.28.gate_proj", "model.layers.23.mlp.experts.29.gate_proj", "model.layers.23.mlp.experts.30.gate_proj", "model.layers.23.mlp.experts.31.gate_proj", "model.layers.23.mlp.experts.32.gate_proj", "model.layers.23.mlp.experts.33.gate_proj", "model.layers.23.mlp.experts.34.gate_proj", "model.layers.23.mlp.experts.35.gate_proj", "model.layers.23.mlp.experts.36.gate_proj", "model.layers.23.mlp.experts.37.gate_proj", "model.layers.23.mlp.experts.38.gate_proj", "model.layers.23.mlp.experts.39.gate_proj", "model.layers.23.mlp.experts.40.gate_proj", "model.layers.23.mlp.experts.41.gate_proj", "model.layers.23.mlp.experts.42.gate_proj", "model.layers.23.mlp.experts.43.gate_proj", "model.layers.23.mlp.experts.44.gate_proj", "model.layers.23.mlp.experts.45.gate_proj", "model.layers.23.mlp.experts.46.gate_proj", "model.layers.23.mlp.experts.47.gate_proj", "model.layers.23.mlp.experts.48.gate_proj", "model.layers.23.mlp.experts.49.gate_proj", "model.layers.23.mlp.experts.50.gate_proj", "model.layers.23.mlp.experts.51.gate_proj", "model.layers.23.mlp.experts.52.gate_proj", "model.layers.23.mlp.experts.53.gate_proj", "model.layers.23.mlp.experts.54.gate_proj", "model.layers.23.mlp.experts.55.gate_proj", "model.layers.23.mlp.experts.56.gate_proj", "model.layers.23.mlp.experts.57.gate_proj", "model.layers.23.mlp.experts.58.gate_proj", "model.layers.23.mlp.experts.59.gate_proj", "model.layers.23.mlp.experts.60.gate_proj", "model.layers.23.mlp.experts.61.gate_proj", "model.layers.23.mlp.experts.62.gate_proj", "model.layers.23.mlp.experts.63.gate_proj", "model.layers.23.mlp.experts.64.gate_proj", "model.layers.23.mlp.experts.65.gate_proj", "model.layers.23.mlp.experts.66.gate_proj", "model.layers.23.mlp.experts.67.gate_proj", "model.layers.23.mlp.experts.68.gate_proj", "model.layers.23.mlp.experts.69.gate_proj", "model.layers.23.mlp.experts.70.gate_proj", "model.layers.23.mlp.experts.71.gate_proj", "model.layers.23.mlp.experts.72.gate_proj", "model.layers.23.mlp.experts.73.gate_proj", "model.layers.23.mlp.experts.74.gate_proj", "model.layers.23.mlp.experts.75.gate_proj", "model.layers.23.mlp.experts.76.gate_proj", "model.layers.23.mlp.experts.77.gate_proj", "model.layers.23.mlp.experts.78.gate_proj", "model.layers.23.mlp.experts.79.gate_proj", "model.layers.23.mlp.experts.80.gate_proj", "model.layers.23.mlp.experts.81.gate_proj", "model.layers.23.mlp.experts.82.gate_proj", "model.layers.23.mlp.experts.83.gate_proj", "model.layers.23.mlp.experts.84.gate_proj", "model.layers.23.mlp.experts.85.gate_proj", "model.layers.23.mlp.experts.86.gate_proj", "model.layers.23.mlp.experts.87.gate_proj", "model.layers.23.mlp.experts.88.gate_proj", "model.layers.23.mlp.experts.89.gate_proj", "model.layers.23.mlp.experts.90.gate_proj", "model.layers.23.mlp.experts.91.gate_proj", "model.layers.23.mlp.experts.92.gate_proj", "model.layers.23.mlp.experts.93.gate_proj", "model.layers.23.mlp.experts.94.gate_proj", "model.layers.23.mlp.experts.95.gate_proj", "model.layers.23.mlp.experts.96.gate_proj", "model.layers.23.mlp.experts.97.gate_proj", "model.layers.23.mlp.experts.98.gate_proj", "model.layers.23.mlp.experts.99.gate_proj", "model.layers.23.mlp.experts.100.gate_proj", "model.layers.23.mlp.experts.101.gate_proj", "model.layers.23.mlp.experts.102.gate_proj", "model.layers.23.mlp.experts.103.gate_proj", "model.layers.23.mlp.experts.104.gate_proj", "model.layers.23.mlp.experts.105.gate_proj", "model.layers.23.mlp.experts.106.gate_proj", "model.layers.23.mlp.experts.107.gate_proj", "model.layers.23.mlp.experts.108.gate_proj", "model.layers.23.mlp.experts.109.gate_proj", "model.layers.23.mlp.experts.110.gate_proj", "model.layers.23.mlp.experts.111.gate_proj", "model.layers.23.mlp.experts.112.gate_proj", "model.layers.23.mlp.experts.113.gate_proj", "model.layers.23.mlp.experts.114.gate_proj", "model.layers.23.mlp.experts.115.gate_proj", "model.layers.23.mlp.experts.116.gate_proj", "model.layers.23.mlp.experts.117.gate_proj", "model.layers.23.mlp.experts.118.gate_proj", "model.layers.23.mlp.experts.119.gate_proj", "model.layers.23.mlp.experts.120.gate_proj", "model.layers.23.mlp.experts.121.gate_proj", "model.layers.23.mlp.experts.122.gate_proj", "model.layers.23.mlp.experts.123.gate_proj", "model.layers.23.mlp.experts.124.gate_proj", "model.layers.23.mlp.experts.125.gate_proj", "model.layers.23.mlp.experts.126.gate_proj", "model.layers.23.mlp.experts.127.gate_proj", "model.layers.23.mlp.experts.128.gate_proj", "model.layers.23.mlp.experts.129.gate_proj", "model.layers.23.mlp.experts.130.gate_proj", "model.layers.23.mlp.experts.131.gate_proj", "model.layers.23.mlp.experts.132.gate_proj", "model.layers.23.mlp.experts.133.gate_proj", "model.layers.23.mlp.experts.134.gate_proj", "model.layers.23.mlp.experts.135.gate_proj", "model.layers.23.mlp.experts.136.gate_proj", "model.layers.23.mlp.experts.137.gate_proj", "model.layers.23.mlp.experts.138.gate_proj", "model.layers.23.mlp.experts.139.gate_proj", "model.layers.23.mlp.experts.140.gate_proj", "model.layers.23.mlp.experts.141.gate_proj", "model.layers.23.mlp.experts.142.gate_proj", "model.layers.23.mlp.experts.143.gate_proj", "model.layers.23.mlp.experts.144.gate_proj", "model.layers.23.mlp.experts.145.gate_proj", "model.layers.23.mlp.experts.146.gate_proj", "model.layers.23.mlp.experts.147.gate_proj", "model.layers.23.mlp.experts.148.gate_proj", "model.layers.23.mlp.experts.149.gate_proj", "model.layers.23.mlp.experts.150.gate_proj", "model.layers.23.mlp.experts.151.gate_proj", "model.layers.23.mlp.experts.152.gate_proj", "model.layers.23.mlp.experts.153.gate_proj", "model.layers.23.mlp.experts.154.gate_proj", "model.layers.23.mlp.experts.155.gate_proj", "model.layers.23.mlp.experts.156.gate_proj", "model.layers.23.mlp.experts.157.gate_proj", "model.layers.23.mlp.experts.158.gate_proj", "model.layers.23.mlp.experts.159.gate_proj", "model.layers.23.mlp.experts.0.up_proj", "model.layers.23.mlp.experts.1.up_proj", "model.layers.23.mlp.experts.2.up_proj", "model.layers.23.mlp.experts.3.up_proj", "model.layers.23.mlp.experts.4.up_proj", "model.layers.23.mlp.experts.5.up_proj", "model.layers.23.mlp.experts.6.up_proj", "model.layers.23.mlp.experts.7.up_proj", "model.layers.23.mlp.experts.8.up_proj", "model.layers.23.mlp.experts.9.up_proj", "model.layers.23.mlp.experts.10.up_proj", "model.layers.23.mlp.experts.11.up_proj", "model.layers.23.mlp.experts.12.up_proj", "model.layers.23.mlp.experts.13.up_proj", "model.layers.23.mlp.experts.14.up_proj", "model.layers.23.mlp.experts.15.up_proj", "model.layers.23.mlp.experts.16.up_proj", "model.layers.23.mlp.experts.17.up_proj", "model.layers.23.mlp.experts.18.up_proj", "model.layers.23.mlp.experts.19.up_proj", "model.layers.23.mlp.experts.20.up_proj", "model.layers.23.mlp.experts.21.up_proj", "model.layers.23.mlp.experts.22.up_proj", "model.layers.23.mlp.experts.23.up_proj", "model.layers.23.mlp.experts.24.up_proj", "model.layers.23.mlp.experts.25.up_proj", "model.layers.23.mlp.experts.26.up_proj", "model.layers.23.mlp.experts.27.up_proj", "model.layers.23.mlp.experts.28.up_proj", "model.layers.23.mlp.experts.29.up_proj", "model.layers.23.mlp.experts.30.up_proj", "model.layers.23.mlp.experts.31.up_proj", "model.layers.23.mlp.experts.32.up_proj", "model.layers.23.mlp.experts.33.up_proj", "model.layers.23.mlp.experts.34.up_proj", "model.layers.23.mlp.experts.35.up_proj", "model.layers.23.mlp.experts.36.up_proj", "model.layers.23.mlp.experts.37.up_proj", "model.layers.23.mlp.experts.38.up_proj", "model.layers.23.mlp.experts.39.up_proj", "model.layers.23.mlp.experts.40.up_proj", "model.layers.23.mlp.experts.41.up_proj", "model.layers.23.mlp.experts.42.up_proj", "model.layers.23.mlp.experts.43.up_proj", "model.layers.23.mlp.experts.44.up_proj", "model.layers.23.mlp.experts.45.up_proj", "model.layers.23.mlp.experts.46.up_proj", "model.layers.23.mlp.experts.47.up_proj", "model.layers.23.mlp.experts.48.up_proj", "model.layers.23.mlp.experts.49.up_proj", "model.layers.23.mlp.experts.50.up_proj", "model.layers.23.mlp.experts.51.up_proj", "model.layers.23.mlp.experts.52.up_proj", "model.layers.23.mlp.experts.53.up_proj", "model.layers.23.mlp.experts.54.up_proj", "model.layers.23.mlp.experts.55.up_proj", "model.layers.23.mlp.experts.56.up_proj", "model.layers.23.mlp.experts.57.up_proj", "model.layers.23.mlp.experts.58.up_proj", "model.layers.23.mlp.experts.59.up_proj", "model.layers.23.mlp.experts.60.up_proj", "model.layers.23.mlp.experts.61.up_proj", "model.layers.23.mlp.experts.62.up_proj", "model.layers.23.mlp.experts.63.up_proj", "model.layers.23.mlp.experts.64.up_proj", "model.layers.23.mlp.experts.65.up_proj", "model.layers.23.mlp.experts.66.up_proj", "model.layers.23.mlp.experts.67.up_proj", "model.layers.23.mlp.experts.68.up_proj", "model.layers.23.mlp.experts.69.up_proj", "model.layers.23.mlp.experts.70.up_proj", "model.layers.23.mlp.experts.71.up_proj", "model.layers.23.mlp.experts.72.up_proj", "model.layers.23.mlp.experts.73.up_proj", "model.layers.23.mlp.experts.74.up_proj", "model.layers.23.mlp.experts.75.up_proj", "model.layers.23.mlp.experts.76.up_proj", "model.layers.23.mlp.experts.77.up_proj", "model.layers.23.mlp.experts.78.up_proj", "model.layers.23.mlp.experts.79.up_proj", "model.layers.23.mlp.experts.80.up_proj", "model.layers.23.mlp.experts.81.up_proj", "model.layers.23.mlp.experts.82.up_proj", "model.layers.23.mlp.experts.83.up_proj", "model.layers.23.mlp.experts.84.up_proj", "model.layers.23.mlp.experts.85.up_proj", "model.layers.23.mlp.experts.86.up_proj", "model.layers.23.mlp.experts.87.up_proj", "model.layers.23.mlp.experts.88.up_proj", "model.layers.23.mlp.experts.89.up_proj", "model.layers.23.mlp.experts.90.up_proj", "model.layers.23.mlp.experts.91.up_proj", "model.layers.23.mlp.experts.92.up_proj", "model.layers.23.mlp.experts.93.up_proj", "model.layers.23.mlp.experts.94.up_proj", "model.layers.23.mlp.experts.95.up_proj", "model.layers.23.mlp.experts.96.up_proj", "model.layers.23.mlp.experts.97.up_proj", "model.layers.23.mlp.experts.98.up_proj", "model.layers.23.mlp.experts.99.up_proj", "model.layers.23.mlp.experts.100.up_proj", "model.layers.23.mlp.experts.101.up_proj", "model.layers.23.mlp.experts.102.up_proj", "model.layers.23.mlp.experts.103.up_proj", "model.layers.23.mlp.experts.104.up_proj", "model.layers.23.mlp.experts.105.up_proj", "model.layers.23.mlp.experts.106.up_proj", "model.layers.23.mlp.experts.107.up_proj", "model.layers.23.mlp.experts.108.up_proj", "model.layers.23.mlp.experts.109.up_proj", "model.layers.23.mlp.experts.110.up_proj", "model.layers.23.mlp.experts.111.up_proj", "model.layers.23.mlp.experts.112.up_proj", "model.layers.23.mlp.experts.113.up_proj", "model.layers.23.mlp.experts.114.up_proj", "model.layers.23.mlp.experts.115.up_proj", "model.layers.23.mlp.experts.116.up_proj", "model.layers.23.mlp.experts.117.up_proj", "model.layers.23.mlp.experts.118.up_proj", "model.layers.23.mlp.experts.119.up_proj", "model.layers.23.mlp.experts.120.up_proj", "model.layers.23.mlp.experts.121.up_proj", "model.layers.23.mlp.experts.122.up_proj", "model.layers.23.mlp.experts.123.up_proj", "model.layers.23.mlp.experts.124.up_proj", "model.layers.23.mlp.experts.125.up_proj", "model.layers.23.mlp.experts.126.up_proj", "model.layers.23.mlp.experts.127.up_proj", "model.layers.23.mlp.experts.128.up_proj", "model.layers.23.mlp.experts.129.up_proj", "model.layers.23.mlp.experts.130.up_proj", "model.layers.23.mlp.experts.131.up_proj", "model.layers.23.mlp.experts.132.up_proj", "model.layers.23.mlp.experts.133.up_proj", "model.layers.23.mlp.experts.134.up_proj", "model.layers.23.mlp.experts.135.up_proj", "model.layers.23.mlp.experts.136.up_proj", "model.layers.23.mlp.experts.137.up_proj", "model.layers.23.mlp.experts.138.up_proj", "model.layers.23.mlp.experts.139.up_proj", "model.layers.23.mlp.experts.140.up_proj", "model.layers.23.mlp.experts.141.up_proj", "model.layers.23.mlp.experts.142.up_proj", "model.layers.23.mlp.experts.143.up_proj", "model.layers.23.mlp.experts.144.up_proj", "model.layers.23.mlp.experts.145.up_proj", "model.layers.23.mlp.experts.146.up_proj", "model.layers.23.mlp.experts.147.up_proj", "model.layers.23.mlp.experts.148.up_proj", "model.layers.23.mlp.experts.149.up_proj", "model.layers.23.mlp.experts.150.up_proj", "model.layers.23.mlp.experts.151.up_proj", "model.layers.23.mlp.experts.152.up_proj", "model.layers.23.mlp.experts.153.up_proj", "model.layers.23.mlp.experts.154.up_proj", "model.layers.23.mlp.experts.155.up_proj", "model.layers.23.mlp.experts.156.up_proj", "model.layers.23.mlp.experts.157.up_proj", "model.layers.23.mlp.experts.158.up_proj", "model.layers.23.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0034894637763500103, "dbits": 2516582400 } ] }, { "idx": 140, "layers": [ "model.layers.23.mlp.experts.0.down_proj", "model.layers.23.mlp.experts.1.down_proj", "model.layers.23.mlp.experts.2.down_proj", "model.layers.23.mlp.experts.3.down_proj", "model.layers.23.mlp.experts.4.down_proj", "model.layers.23.mlp.experts.5.down_proj", "model.layers.23.mlp.experts.6.down_proj", "model.layers.23.mlp.experts.7.down_proj", "model.layers.23.mlp.experts.8.down_proj", "model.layers.23.mlp.experts.9.down_proj", "model.layers.23.mlp.experts.10.down_proj", "model.layers.23.mlp.experts.11.down_proj", "model.layers.23.mlp.experts.12.down_proj", "model.layers.23.mlp.experts.13.down_proj", "model.layers.23.mlp.experts.14.down_proj", "model.layers.23.mlp.experts.15.down_proj", "model.layers.23.mlp.experts.16.down_proj", "model.layers.23.mlp.experts.17.down_proj", "model.layers.23.mlp.experts.18.down_proj", "model.layers.23.mlp.experts.19.down_proj", "model.layers.23.mlp.experts.20.down_proj", "model.layers.23.mlp.experts.21.down_proj", "model.layers.23.mlp.experts.22.down_proj", "model.layers.23.mlp.experts.23.down_proj", "model.layers.23.mlp.experts.24.down_proj", "model.layers.23.mlp.experts.25.down_proj", "model.layers.23.mlp.experts.26.down_proj", "model.layers.23.mlp.experts.27.down_proj", "model.layers.23.mlp.experts.28.down_proj", "model.layers.23.mlp.experts.29.down_proj", "model.layers.23.mlp.experts.30.down_proj", "model.layers.23.mlp.experts.31.down_proj", "model.layers.23.mlp.experts.32.down_proj", "model.layers.23.mlp.experts.33.down_proj", "model.layers.23.mlp.experts.34.down_proj", "model.layers.23.mlp.experts.35.down_proj", "model.layers.23.mlp.experts.36.down_proj", "model.layers.23.mlp.experts.37.down_proj", "model.layers.23.mlp.experts.38.down_proj", "model.layers.23.mlp.experts.39.down_proj", "model.layers.23.mlp.experts.40.down_proj", "model.layers.23.mlp.experts.41.down_proj", "model.layers.23.mlp.experts.42.down_proj", "model.layers.23.mlp.experts.43.down_proj", "model.layers.23.mlp.experts.44.down_proj", "model.layers.23.mlp.experts.45.down_proj", "model.layers.23.mlp.experts.46.down_proj", "model.layers.23.mlp.experts.47.down_proj", "model.layers.23.mlp.experts.48.down_proj", "model.layers.23.mlp.experts.49.down_proj", "model.layers.23.mlp.experts.50.down_proj", "model.layers.23.mlp.experts.51.down_proj", "model.layers.23.mlp.experts.52.down_proj", "model.layers.23.mlp.experts.53.down_proj", "model.layers.23.mlp.experts.54.down_proj", "model.layers.23.mlp.experts.55.down_proj", "model.layers.23.mlp.experts.56.down_proj", "model.layers.23.mlp.experts.57.down_proj", "model.layers.23.mlp.experts.58.down_proj", "model.layers.23.mlp.experts.59.down_proj", "model.layers.23.mlp.experts.60.down_proj", "model.layers.23.mlp.experts.61.down_proj", "model.layers.23.mlp.experts.62.down_proj", "model.layers.23.mlp.experts.63.down_proj", "model.layers.23.mlp.experts.64.down_proj", "model.layers.23.mlp.experts.65.down_proj", "model.layers.23.mlp.experts.66.down_proj", "model.layers.23.mlp.experts.67.down_proj", "model.layers.23.mlp.experts.68.down_proj", "model.layers.23.mlp.experts.69.down_proj", "model.layers.23.mlp.experts.70.down_proj", "model.layers.23.mlp.experts.71.down_proj", "model.layers.23.mlp.experts.72.down_proj", "model.layers.23.mlp.experts.73.down_proj", "model.layers.23.mlp.experts.74.down_proj", "model.layers.23.mlp.experts.75.down_proj", "model.layers.23.mlp.experts.76.down_proj", "model.layers.23.mlp.experts.77.down_proj", "model.layers.23.mlp.experts.78.down_proj", "model.layers.23.mlp.experts.79.down_proj", "model.layers.23.mlp.experts.80.down_proj", "model.layers.23.mlp.experts.81.down_proj", "model.layers.23.mlp.experts.82.down_proj", "model.layers.23.mlp.experts.83.down_proj", "model.layers.23.mlp.experts.84.down_proj", "model.layers.23.mlp.experts.85.down_proj", "model.layers.23.mlp.experts.86.down_proj", "model.layers.23.mlp.experts.87.down_proj", "model.layers.23.mlp.experts.88.down_proj", "model.layers.23.mlp.experts.89.down_proj", "model.layers.23.mlp.experts.90.down_proj", "model.layers.23.mlp.experts.91.down_proj", "model.layers.23.mlp.experts.92.down_proj", "model.layers.23.mlp.experts.93.down_proj", "model.layers.23.mlp.experts.94.down_proj", "model.layers.23.mlp.experts.95.down_proj", "model.layers.23.mlp.experts.96.down_proj", "model.layers.23.mlp.experts.97.down_proj", "model.layers.23.mlp.experts.98.down_proj", "model.layers.23.mlp.experts.99.down_proj", "model.layers.23.mlp.experts.100.down_proj", "model.layers.23.mlp.experts.101.down_proj", "model.layers.23.mlp.experts.102.down_proj", "model.layers.23.mlp.experts.103.down_proj", "model.layers.23.mlp.experts.104.down_proj", "model.layers.23.mlp.experts.105.down_proj", "model.layers.23.mlp.experts.106.down_proj", "model.layers.23.mlp.experts.107.down_proj", "model.layers.23.mlp.experts.108.down_proj", "model.layers.23.mlp.experts.109.down_proj", "model.layers.23.mlp.experts.110.down_proj", "model.layers.23.mlp.experts.111.down_proj", "model.layers.23.mlp.experts.112.down_proj", "model.layers.23.mlp.experts.113.down_proj", "model.layers.23.mlp.experts.114.down_proj", "model.layers.23.mlp.experts.115.down_proj", "model.layers.23.mlp.experts.116.down_proj", "model.layers.23.mlp.experts.117.down_proj", "model.layers.23.mlp.experts.118.down_proj", "model.layers.23.mlp.experts.119.down_proj", "model.layers.23.mlp.experts.120.down_proj", "model.layers.23.mlp.experts.121.down_proj", "model.layers.23.mlp.experts.122.down_proj", "model.layers.23.mlp.experts.123.down_proj", "model.layers.23.mlp.experts.124.down_proj", "model.layers.23.mlp.experts.125.down_proj", "model.layers.23.mlp.experts.126.down_proj", "model.layers.23.mlp.experts.127.down_proj", "model.layers.23.mlp.experts.128.down_proj", "model.layers.23.mlp.experts.129.down_proj", "model.layers.23.mlp.experts.130.down_proj", "model.layers.23.mlp.experts.131.down_proj", "model.layers.23.mlp.experts.132.down_proj", "model.layers.23.mlp.experts.133.down_proj", "model.layers.23.mlp.experts.134.down_proj", "model.layers.23.mlp.experts.135.down_proj", "model.layers.23.mlp.experts.136.down_proj", "model.layers.23.mlp.experts.137.down_proj", "model.layers.23.mlp.experts.138.down_proj", "model.layers.23.mlp.experts.139.down_proj", "model.layers.23.mlp.experts.140.down_proj", "model.layers.23.mlp.experts.141.down_proj", "model.layers.23.mlp.experts.142.down_proj", "model.layers.23.mlp.experts.143.down_proj", "model.layers.23.mlp.experts.144.down_proj", "model.layers.23.mlp.experts.145.down_proj", "model.layers.23.mlp.experts.146.down_proj", "model.layers.23.mlp.experts.147.down_proj", "model.layers.23.mlp.experts.148.down_proj", "model.layers.23.mlp.experts.149.down_proj", "model.layers.23.mlp.experts.150.down_proj", "model.layers.23.mlp.experts.151.down_proj", "model.layers.23.mlp.experts.152.down_proj", "model.layers.23.mlp.experts.153.down_proj", "model.layers.23.mlp.experts.154.down_proj", "model.layers.23.mlp.experts.155.down_proj", "model.layers.23.mlp.experts.156.down_proj", "model.layers.23.mlp.experts.157.down_proj", "model.layers.23.mlp.experts.158.down_proj", "model.layers.23.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0015354841947555764, "dbits": 1258291200 } ] }, { "idx": 141, "layers": [ "model.layers.24.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0006864473223686107, "dbits": 62914560 } ] }, { "idx": 142, "layers": [ "model.layers.24.self_attn.k_proj", "model.layers.24.self_attn.v_proj" ], "candidates": [ { "dkld": 0.000934087485075008, "dbits": 10485760 } ] }, { "idx": 143, "layers": [ "model.layers.24.self_attn.o_proj" ], "candidates": [ { "dkld": -0.007718427479267154, "dbits": 62914560 } ] }, { "idx": 144, "layers": [ "model.layers.24.mlp.shared_experts.gate_proj", "model.layers.24.mlp.shared_experts.up_proj", "model.layers.24.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.002513144910335552, "dbits": 23592960 } ] }, { "idx": 145, "layers": [ "model.layers.24.mlp.experts.0.gate_proj", "model.layers.24.mlp.experts.1.gate_proj", "model.layers.24.mlp.experts.2.gate_proj", "model.layers.24.mlp.experts.3.gate_proj", "model.layers.24.mlp.experts.4.gate_proj", "model.layers.24.mlp.experts.5.gate_proj", "model.layers.24.mlp.experts.6.gate_proj", "model.layers.24.mlp.experts.7.gate_proj", "model.layers.24.mlp.experts.8.gate_proj", "model.layers.24.mlp.experts.9.gate_proj", "model.layers.24.mlp.experts.10.gate_proj", "model.layers.24.mlp.experts.11.gate_proj", "model.layers.24.mlp.experts.12.gate_proj", "model.layers.24.mlp.experts.13.gate_proj", "model.layers.24.mlp.experts.14.gate_proj", "model.layers.24.mlp.experts.15.gate_proj", "model.layers.24.mlp.experts.16.gate_proj", "model.layers.24.mlp.experts.17.gate_proj", "model.layers.24.mlp.experts.18.gate_proj", "model.layers.24.mlp.experts.19.gate_proj", "model.layers.24.mlp.experts.20.gate_proj", "model.layers.24.mlp.experts.21.gate_proj", "model.layers.24.mlp.experts.22.gate_proj", "model.layers.24.mlp.experts.23.gate_proj", "model.layers.24.mlp.experts.24.gate_proj", "model.layers.24.mlp.experts.25.gate_proj", "model.layers.24.mlp.experts.26.gate_proj", "model.layers.24.mlp.experts.27.gate_proj", "model.layers.24.mlp.experts.28.gate_proj", "model.layers.24.mlp.experts.29.gate_proj", "model.layers.24.mlp.experts.30.gate_proj", "model.layers.24.mlp.experts.31.gate_proj", "model.layers.24.mlp.experts.32.gate_proj", "model.layers.24.mlp.experts.33.gate_proj", "model.layers.24.mlp.experts.34.gate_proj", "model.layers.24.mlp.experts.35.gate_proj", "model.layers.24.mlp.experts.36.gate_proj", "model.layers.24.mlp.experts.37.gate_proj", "model.layers.24.mlp.experts.38.gate_proj", "model.layers.24.mlp.experts.39.gate_proj", "model.layers.24.mlp.experts.40.gate_proj", "model.layers.24.mlp.experts.41.gate_proj", "model.layers.24.mlp.experts.42.gate_proj", "model.layers.24.mlp.experts.43.gate_proj", "model.layers.24.mlp.experts.44.gate_proj", "model.layers.24.mlp.experts.45.gate_proj", "model.layers.24.mlp.experts.46.gate_proj", "model.layers.24.mlp.experts.47.gate_proj", "model.layers.24.mlp.experts.48.gate_proj", "model.layers.24.mlp.experts.49.gate_proj", "model.layers.24.mlp.experts.50.gate_proj", "model.layers.24.mlp.experts.51.gate_proj", "model.layers.24.mlp.experts.52.gate_proj", "model.layers.24.mlp.experts.53.gate_proj", "model.layers.24.mlp.experts.54.gate_proj", "model.layers.24.mlp.experts.55.gate_proj", "model.layers.24.mlp.experts.56.gate_proj", "model.layers.24.mlp.experts.57.gate_proj", "model.layers.24.mlp.experts.58.gate_proj", "model.layers.24.mlp.experts.59.gate_proj", "model.layers.24.mlp.experts.60.gate_proj", "model.layers.24.mlp.experts.61.gate_proj", "model.layers.24.mlp.experts.62.gate_proj", "model.layers.24.mlp.experts.63.gate_proj", "model.layers.24.mlp.experts.64.gate_proj", "model.layers.24.mlp.experts.65.gate_proj", "model.layers.24.mlp.experts.66.gate_proj", "model.layers.24.mlp.experts.67.gate_proj", "model.layers.24.mlp.experts.68.gate_proj", "model.layers.24.mlp.experts.69.gate_proj", "model.layers.24.mlp.experts.70.gate_proj", "model.layers.24.mlp.experts.71.gate_proj", "model.layers.24.mlp.experts.72.gate_proj", "model.layers.24.mlp.experts.73.gate_proj", "model.layers.24.mlp.experts.74.gate_proj", "model.layers.24.mlp.experts.75.gate_proj", "model.layers.24.mlp.experts.76.gate_proj", "model.layers.24.mlp.experts.77.gate_proj", "model.layers.24.mlp.experts.78.gate_proj", "model.layers.24.mlp.experts.79.gate_proj", "model.layers.24.mlp.experts.80.gate_proj", "model.layers.24.mlp.experts.81.gate_proj", "model.layers.24.mlp.experts.82.gate_proj", "model.layers.24.mlp.experts.83.gate_proj", "model.layers.24.mlp.experts.84.gate_proj", "model.layers.24.mlp.experts.85.gate_proj", "model.layers.24.mlp.experts.86.gate_proj", "model.layers.24.mlp.experts.87.gate_proj", "model.layers.24.mlp.experts.88.gate_proj", "model.layers.24.mlp.experts.89.gate_proj", "model.layers.24.mlp.experts.90.gate_proj", "model.layers.24.mlp.experts.91.gate_proj", "model.layers.24.mlp.experts.92.gate_proj", "model.layers.24.mlp.experts.93.gate_proj", "model.layers.24.mlp.experts.94.gate_proj", "model.layers.24.mlp.experts.95.gate_proj", "model.layers.24.mlp.experts.96.gate_proj", "model.layers.24.mlp.experts.97.gate_proj", "model.layers.24.mlp.experts.98.gate_proj", "model.layers.24.mlp.experts.99.gate_proj", "model.layers.24.mlp.experts.100.gate_proj", "model.layers.24.mlp.experts.101.gate_proj", "model.layers.24.mlp.experts.102.gate_proj", "model.layers.24.mlp.experts.103.gate_proj", "model.layers.24.mlp.experts.104.gate_proj", "model.layers.24.mlp.experts.105.gate_proj", "model.layers.24.mlp.experts.106.gate_proj", "model.layers.24.mlp.experts.107.gate_proj", "model.layers.24.mlp.experts.108.gate_proj", "model.layers.24.mlp.experts.109.gate_proj", "model.layers.24.mlp.experts.110.gate_proj", "model.layers.24.mlp.experts.111.gate_proj", "model.layers.24.mlp.experts.112.gate_proj", "model.layers.24.mlp.experts.113.gate_proj", "model.layers.24.mlp.experts.114.gate_proj", "model.layers.24.mlp.experts.115.gate_proj", "model.layers.24.mlp.experts.116.gate_proj", "model.layers.24.mlp.experts.117.gate_proj", "model.layers.24.mlp.experts.118.gate_proj", "model.layers.24.mlp.experts.119.gate_proj", "model.layers.24.mlp.experts.120.gate_proj", "model.layers.24.mlp.experts.121.gate_proj", "model.layers.24.mlp.experts.122.gate_proj", "model.layers.24.mlp.experts.123.gate_proj", "model.layers.24.mlp.experts.124.gate_proj", "model.layers.24.mlp.experts.125.gate_proj", "model.layers.24.mlp.experts.126.gate_proj", "model.layers.24.mlp.experts.127.gate_proj", "model.layers.24.mlp.experts.128.gate_proj", "model.layers.24.mlp.experts.129.gate_proj", "model.layers.24.mlp.experts.130.gate_proj", "model.layers.24.mlp.experts.131.gate_proj", "model.layers.24.mlp.experts.132.gate_proj", "model.layers.24.mlp.experts.133.gate_proj", "model.layers.24.mlp.experts.134.gate_proj", "model.layers.24.mlp.experts.135.gate_proj", "model.layers.24.mlp.experts.136.gate_proj", "model.layers.24.mlp.experts.137.gate_proj", "model.layers.24.mlp.experts.138.gate_proj", "model.layers.24.mlp.experts.139.gate_proj", "model.layers.24.mlp.experts.140.gate_proj", "model.layers.24.mlp.experts.141.gate_proj", "model.layers.24.mlp.experts.142.gate_proj", "model.layers.24.mlp.experts.143.gate_proj", "model.layers.24.mlp.experts.144.gate_proj", "model.layers.24.mlp.experts.145.gate_proj", "model.layers.24.mlp.experts.146.gate_proj", "model.layers.24.mlp.experts.147.gate_proj", "model.layers.24.mlp.experts.148.gate_proj", "model.layers.24.mlp.experts.149.gate_proj", "model.layers.24.mlp.experts.150.gate_proj", "model.layers.24.mlp.experts.151.gate_proj", "model.layers.24.mlp.experts.152.gate_proj", "model.layers.24.mlp.experts.153.gate_proj", "model.layers.24.mlp.experts.154.gate_proj", "model.layers.24.mlp.experts.155.gate_proj", "model.layers.24.mlp.experts.156.gate_proj", "model.layers.24.mlp.experts.157.gate_proj", "model.layers.24.mlp.experts.158.gate_proj", "model.layers.24.mlp.experts.159.gate_proj", "model.layers.24.mlp.experts.0.up_proj", "model.layers.24.mlp.experts.1.up_proj", "model.layers.24.mlp.experts.2.up_proj", "model.layers.24.mlp.experts.3.up_proj", "model.layers.24.mlp.experts.4.up_proj", "model.layers.24.mlp.experts.5.up_proj", "model.layers.24.mlp.experts.6.up_proj", "model.layers.24.mlp.experts.7.up_proj", "model.layers.24.mlp.experts.8.up_proj", "model.layers.24.mlp.experts.9.up_proj", "model.layers.24.mlp.experts.10.up_proj", "model.layers.24.mlp.experts.11.up_proj", "model.layers.24.mlp.experts.12.up_proj", "model.layers.24.mlp.experts.13.up_proj", "model.layers.24.mlp.experts.14.up_proj", "model.layers.24.mlp.experts.15.up_proj", "model.layers.24.mlp.experts.16.up_proj", "model.layers.24.mlp.experts.17.up_proj", "model.layers.24.mlp.experts.18.up_proj", "model.layers.24.mlp.experts.19.up_proj", "model.layers.24.mlp.experts.20.up_proj", "model.layers.24.mlp.experts.21.up_proj", "model.layers.24.mlp.experts.22.up_proj", "model.layers.24.mlp.experts.23.up_proj", "model.layers.24.mlp.experts.24.up_proj", "model.layers.24.mlp.experts.25.up_proj", "model.layers.24.mlp.experts.26.up_proj", "model.layers.24.mlp.experts.27.up_proj", "model.layers.24.mlp.experts.28.up_proj", "model.layers.24.mlp.experts.29.up_proj", "model.layers.24.mlp.experts.30.up_proj", "model.layers.24.mlp.experts.31.up_proj", "model.layers.24.mlp.experts.32.up_proj", "model.layers.24.mlp.experts.33.up_proj", "model.layers.24.mlp.experts.34.up_proj", "model.layers.24.mlp.experts.35.up_proj", "model.layers.24.mlp.experts.36.up_proj", "model.layers.24.mlp.experts.37.up_proj", "model.layers.24.mlp.experts.38.up_proj", "model.layers.24.mlp.experts.39.up_proj", "model.layers.24.mlp.experts.40.up_proj", "model.layers.24.mlp.experts.41.up_proj", "model.layers.24.mlp.experts.42.up_proj", "model.layers.24.mlp.experts.43.up_proj", "model.layers.24.mlp.experts.44.up_proj", "model.layers.24.mlp.experts.45.up_proj", "model.layers.24.mlp.experts.46.up_proj", "model.layers.24.mlp.experts.47.up_proj", "model.layers.24.mlp.experts.48.up_proj", "model.layers.24.mlp.experts.49.up_proj", "model.layers.24.mlp.experts.50.up_proj", "model.layers.24.mlp.experts.51.up_proj", "model.layers.24.mlp.experts.52.up_proj", "model.layers.24.mlp.experts.53.up_proj", "model.layers.24.mlp.experts.54.up_proj", "model.layers.24.mlp.experts.55.up_proj", "model.layers.24.mlp.experts.56.up_proj", "model.layers.24.mlp.experts.57.up_proj", "model.layers.24.mlp.experts.58.up_proj", "model.layers.24.mlp.experts.59.up_proj", "model.layers.24.mlp.experts.60.up_proj", "model.layers.24.mlp.experts.61.up_proj", "model.layers.24.mlp.experts.62.up_proj", "model.layers.24.mlp.experts.63.up_proj", "model.layers.24.mlp.experts.64.up_proj", "model.layers.24.mlp.experts.65.up_proj", "model.layers.24.mlp.experts.66.up_proj", "model.layers.24.mlp.experts.67.up_proj", "model.layers.24.mlp.experts.68.up_proj", "model.layers.24.mlp.experts.69.up_proj", "model.layers.24.mlp.experts.70.up_proj", "model.layers.24.mlp.experts.71.up_proj", "model.layers.24.mlp.experts.72.up_proj", "model.layers.24.mlp.experts.73.up_proj", "model.layers.24.mlp.experts.74.up_proj", "model.layers.24.mlp.experts.75.up_proj", "model.layers.24.mlp.experts.76.up_proj", "model.layers.24.mlp.experts.77.up_proj", "model.layers.24.mlp.experts.78.up_proj", "model.layers.24.mlp.experts.79.up_proj", "model.layers.24.mlp.experts.80.up_proj", "model.layers.24.mlp.experts.81.up_proj", "model.layers.24.mlp.experts.82.up_proj", "model.layers.24.mlp.experts.83.up_proj", "model.layers.24.mlp.experts.84.up_proj", "model.layers.24.mlp.experts.85.up_proj", "model.layers.24.mlp.experts.86.up_proj", "model.layers.24.mlp.experts.87.up_proj", "model.layers.24.mlp.experts.88.up_proj", "model.layers.24.mlp.experts.89.up_proj", "model.layers.24.mlp.experts.90.up_proj", "model.layers.24.mlp.experts.91.up_proj", "model.layers.24.mlp.experts.92.up_proj", "model.layers.24.mlp.experts.93.up_proj", "model.layers.24.mlp.experts.94.up_proj", "model.layers.24.mlp.experts.95.up_proj", "model.layers.24.mlp.experts.96.up_proj", "model.layers.24.mlp.experts.97.up_proj", "model.layers.24.mlp.experts.98.up_proj", "model.layers.24.mlp.experts.99.up_proj", "model.layers.24.mlp.experts.100.up_proj", "model.layers.24.mlp.experts.101.up_proj", "model.layers.24.mlp.experts.102.up_proj", "model.layers.24.mlp.experts.103.up_proj", "model.layers.24.mlp.experts.104.up_proj", "model.layers.24.mlp.experts.105.up_proj", "model.layers.24.mlp.experts.106.up_proj", "model.layers.24.mlp.experts.107.up_proj", "model.layers.24.mlp.experts.108.up_proj", "model.layers.24.mlp.experts.109.up_proj", "model.layers.24.mlp.experts.110.up_proj", "model.layers.24.mlp.experts.111.up_proj", "model.layers.24.mlp.experts.112.up_proj", "model.layers.24.mlp.experts.113.up_proj", "model.layers.24.mlp.experts.114.up_proj", "model.layers.24.mlp.experts.115.up_proj", "model.layers.24.mlp.experts.116.up_proj", "model.layers.24.mlp.experts.117.up_proj", "model.layers.24.mlp.experts.118.up_proj", "model.layers.24.mlp.experts.119.up_proj", "model.layers.24.mlp.experts.120.up_proj", "model.layers.24.mlp.experts.121.up_proj", "model.layers.24.mlp.experts.122.up_proj", "model.layers.24.mlp.experts.123.up_proj", "model.layers.24.mlp.experts.124.up_proj", "model.layers.24.mlp.experts.125.up_proj", "model.layers.24.mlp.experts.126.up_proj", "model.layers.24.mlp.experts.127.up_proj", "model.layers.24.mlp.experts.128.up_proj", "model.layers.24.mlp.experts.129.up_proj", "model.layers.24.mlp.experts.130.up_proj", "model.layers.24.mlp.experts.131.up_proj", "model.layers.24.mlp.experts.132.up_proj", "model.layers.24.mlp.experts.133.up_proj", "model.layers.24.mlp.experts.134.up_proj", "model.layers.24.mlp.experts.135.up_proj", "model.layers.24.mlp.experts.136.up_proj", "model.layers.24.mlp.experts.137.up_proj", "model.layers.24.mlp.experts.138.up_proj", "model.layers.24.mlp.experts.139.up_proj", "model.layers.24.mlp.experts.140.up_proj", "model.layers.24.mlp.experts.141.up_proj", "model.layers.24.mlp.experts.142.up_proj", "model.layers.24.mlp.experts.143.up_proj", "model.layers.24.mlp.experts.144.up_proj", "model.layers.24.mlp.experts.145.up_proj", "model.layers.24.mlp.experts.146.up_proj", "model.layers.24.mlp.experts.147.up_proj", "model.layers.24.mlp.experts.148.up_proj", "model.layers.24.mlp.experts.149.up_proj", "model.layers.24.mlp.experts.150.up_proj", "model.layers.24.mlp.experts.151.up_proj", "model.layers.24.mlp.experts.152.up_proj", "model.layers.24.mlp.experts.153.up_proj", "model.layers.24.mlp.experts.154.up_proj", "model.layers.24.mlp.experts.155.up_proj", "model.layers.24.mlp.experts.156.up_proj", "model.layers.24.mlp.experts.157.up_proj", "model.layers.24.mlp.experts.158.up_proj", "model.layers.24.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.002274114638566971, "dbits": 2516582400 } ] }, { "idx": 146, "layers": [ "model.layers.24.mlp.experts.0.down_proj", "model.layers.24.mlp.experts.1.down_proj", "model.layers.24.mlp.experts.2.down_proj", "model.layers.24.mlp.experts.3.down_proj", "model.layers.24.mlp.experts.4.down_proj", "model.layers.24.mlp.experts.5.down_proj", "model.layers.24.mlp.experts.6.down_proj", "model.layers.24.mlp.experts.7.down_proj", "model.layers.24.mlp.experts.8.down_proj", "model.layers.24.mlp.experts.9.down_proj", "model.layers.24.mlp.experts.10.down_proj", "model.layers.24.mlp.experts.11.down_proj", "model.layers.24.mlp.experts.12.down_proj", "model.layers.24.mlp.experts.13.down_proj", "model.layers.24.mlp.experts.14.down_proj", "model.layers.24.mlp.experts.15.down_proj", "model.layers.24.mlp.experts.16.down_proj", "model.layers.24.mlp.experts.17.down_proj", "model.layers.24.mlp.experts.18.down_proj", "model.layers.24.mlp.experts.19.down_proj", "model.layers.24.mlp.experts.20.down_proj", "model.layers.24.mlp.experts.21.down_proj", "model.layers.24.mlp.experts.22.down_proj", "model.layers.24.mlp.experts.23.down_proj", "model.layers.24.mlp.experts.24.down_proj", "model.layers.24.mlp.experts.25.down_proj", "model.layers.24.mlp.experts.26.down_proj", "model.layers.24.mlp.experts.27.down_proj", "model.layers.24.mlp.experts.28.down_proj", "model.layers.24.mlp.experts.29.down_proj", "model.layers.24.mlp.experts.30.down_proj", "model.layers.24.mlp.experts.31.down_proj", "model.layers.24.mlp.experts.32.down_proj", "model.layers.24.mlp.experts.33.down_proj", "model.layers.24.mlp.experts.34.down_proj", "model.layers.24.mlp.experts.35.down_proj", "model.layers.24.mlp.experts.36.down_proj", "model.layers.24.mlp.experts.37.down_proj", "model.layers.24.mlp.experts.38.down_proj", "model.layers.24.mlp.experts.39.down_proj", "model.layers.24.mlp.experts.40.down_proj", "model.layers.24.mlp.experts.41.down_proj", "model.layers.24.mlp.experts.42.down_proj", "model.layers.24.mlp.experts.43.down_proj", "model.layers.24.mlp.experts.44.down_proj", "model.layers.24.mlp.experts.45.down_proj", "model.layers.24.mlp.experts.46.down_proj", "model.layers.24.mlp.experts.47.down_proj", "model.layers.24.mlp.experts.48.down_proj", "model.layers.24.mlp.experts.49.down_proj", "model.layers.24.mlp.experts.50.down_proj", "model.layers.24.mlp.experts.51.down_proj", "model.layers.24.mlp.experts.52.down_proj", "model.layers.24.mlp.experts.53.down_proj", "model.layers.24.mlp.experts.54.down_proj", "model.layers.24.mlp.experts.55.down_proj", "model.layers.24.mlp.experts.56.down_proj", "model.layers.24.mlp.experts.57.down_proj", "model.layers.24.mlp.experts.58.down_proj", "model.layers.24.mlp.experts.59.down_proj", "model.layers.24.mlp.experts.60.down_proj", "model.layers.24.mlp.experts.61.down_proj", "model.layers.24.mlp.experts.62.down_proj", "model.layers.24.mlp.experts.63.down_proj", "model.layers.24.mlp.experts.64.down_proj", "model.layers.24.mlp.experts.65.down_proj", "model.layers.24.mlp.experts.66.down_proj", "model.layers.24.mlp.experts.67.down_proj", "model.layers.24.mlp.experts.68.down_proj", "model.layers.24.mlp.experts.69.down_proj", "model.layers.24.mlp.experts.70.down_proj", "model.layers.24.mlp.experts.71.down_proj", "model.layers.24.mlp.experts.72.down_proj", "model.layers.24.mlp.experts.73.down_proj", "model.layers.24.mlp.experts.74.down_proj", "model.layers.24.mlp.experts.75.down_proj", "model.layers.24.mlp.experts.76.down_proj", "model.layers.24.mlp.experts.77.down_proj", "model.layers.24.mlp.experts.78.down_proj", "model.layers.24.mlp.experts.79.down_proj", "model.layers.24.mlp.experts.80.down_proj", "model.layers.24.mlp.experts.81.down_proj", "model.layers.24.mlp.experts.82.down_proj", "model.layers.24.mlp.experts.83.down_proj", "model.layers.24.mlp.experts.84.down_proj", "model.layers.24.mlp.experts.85.down_proj", "model.layers.24.mlp.experts.86.down_proj", "model.layers.24.mlp.experts.87.down_proj", "model.layers.24.mlp.experts.88.down_proj", "model.layers.24.mlp.experts.89.down_proj", "model.layers.24.mlp.experts.90.down_proj", "model.layers.24.mlp.experts.91.down_proj", "model.layers.24.mlp.experts.92.down_proj", "model.layers.24.mlp.experts.93.down_proj", "model.layers.24.mlp.experts.94.down_proj", "model.layers.24.mlp.experts.95.down_proj", "model.layers.24.mlp.experts.96.down_proj", "model.layers.24.mlp.experts.97.down_proj", "model.layers.24.mlp.experts.98.down_proj", "model.layers.24.mlp.experts.99.down_proj", "model.layers.24.mlp.experts.100.down_proj", "model.layers.24.mlp.experts.101.down_proj", "model.layers.24.mlp.experts.102.down_proj", "model.layers.24.mlp.experts.103.down_proj", "model.layers.24.mlp.experts.104.down_proj", "model.layers.24.mlp.experts.105.down_proj", "model.layers.24.mlp.experts.106.down_proj", "model.layers.24.mlp.experts.107.down_proj", "model.layers.24.mlp.experts.108.down_proj", "model.layers.24.mlp.experts.109.down_proj", "model.layers.24.mlp.experts.110.down_proj", "model.layers.24.mlp.experts.111.down_proj", "model.layers.24.mlp.experts.112.down_proj", "model.layers.24.mlp.experts.113.down_proj", "model.layers.24.mlp.experts.114.down_proj", "model.layers.24.mlp.experts.115.down_proj", "model.layers.24.mlp.experts.116.down_proj", "model.layers.24.mlp.experts.117.down_proj", "model.layers.24.mlp.experts.118.down_proj", "model.layers.24.mlp.experts.119.down_proj", "model.layers.24.mlp.experts.120.down_proj", "model.layers.24.mlp.experts.121.down_proj", "model.layers.24.mlp.experts.122.down_proj", "model.layers.24.mlp.experts.123.down_proj", "model.layers.24.mlp.experts.124.down_proj", "model.layers.24.mlp.experts.125.down_proj", "model.layers.24.mlp.experts.126.down_proj", "model.layers.24.mlp.experts.127.down_proj", "model.layers.24.mlp.experts.128.down_proj", "model.layers.24.mlp.experts.129.down_proj", "model.layers.24.mlp.experts.130.down_proj", "model.layers.24.mlp.experts.131.down_proj", "model.layers.24.mlp.experts.132.down_proj", "model.layers.24.mlp.experts.133.down_proj", "model.layers.24.mlp.experts.134.down_proj", "model.layers.24.mlp.experts.135.down_proj", "model.layers.24.mlp.experts.136.down_proj", "model.layers.24.mlp.experts.137.down_proj", "model.layers.24.mlp.experts.138.down_proj", "model.layers.24.mlp.experts.139.down_proj", "model.layers.24.mlp.experts.140.down_proj", "model.layers.24.mlp.experts.141.down_proj", "model.layers.24.mlp.experts.142.down_proj", "model.layers.24.mlp.experts.143.down_proj", "model.layers.24.mlp.experts.144.down_proj", "model.layers.24.mlp.experts.145.down_proj", "model.layers.24.mlp.experts.146.down_proj", "model.layers.24.mlp.experts.147.down_proj", "model.layers.24.mlp.experts.148.down_proj", "model.layers.24.mlp.experts.149.down_proj", "model.layers.24.mlp.experts.150.down_proj", "model.layers.24.mlp.experts.151.down_proj", "model.layers.24.mlp.experts.152.down_proj", "model.layers.24.mlp.experts.153.down_proj", "model.layers.24.mlp.experts.154.down_proj", "model.layers.24.mlp.experts.155.down_proj", "model.layers.24.mlp.experts.156.down_proj", "model.layers.24.mlp.experts.157.down_proj", "model.layers.24.mlp.experts.158.down_proj", "model.layers.24.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0025529444217681774, "dbits": 1258291200 } ] }, { "idx": 147, "layers": [ "model.layers.25.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0020675353705882915, "dbits": 62914560 } ] }, { "idx": 148, "layers": [ "model.layers.25.self_attn.k_proj", "model.layers.25.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0012882798910140991, "dbits": 10485760 } ] }, { "idx": 149, "layers": [ "model.layers.25.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0031835936009884103, "dbits": 62914560 } ] }, { "idx": 150, "layers": [ "model.layers.25.mlp.shared_experts.gate_proj", "model.layers.25.mlp.shared_experts.up_proj", "model.layers.25.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.003732319176197041, "dbits": 23592960 } ] }, { "idx": 151, "layers": [ "model.layers.25.mlp.experts.0.gate_proj", "model.layers.25.mlp.experts.1.gate_proj", "model.layers.25.mlp.experts.2.gate_proj", "model.layers.25.mlp.experts.3.gate_proj", "model.layers.25.mlp.experts.4.gate_proj", "model.layers.25.mlp.experts.5.gate_proj", "model.layers.25.mlp.experts.6.gate_proj", "model.layers.25.mlp.experts.7.gate_proj", "model.layers.25.mlp.experts.8.gate_proj", "model.layers.25.mlp.experts.9.gate_proj", "model.layers.25.mlp.experts.10.gate_proj", "model.layers.25.mlp.experts.11.gate_proj", "model.layers.25.mlp.experts.12.gate_proj", "model.layers.25.mlp.experts.13.gate_proj", "model.layers.25.mlp.experts.14.gate_proj", "model.layers.25.mlp.experts.15.gate_proj", "model.layers.25.mlp.experts.16.gate_proj", "model.layers.25.mlp.experts.17.gate_proj", "model.layers.25.mlp.experts.18.gate_proj", "model.layers.25.mlp.experts.19.gate_proj", "model.layers.25.mlp.experts.20.gate_proj", "model.layers.25.mlp.experts.21.gate_proj", "model.layers.25.mlp.experts.22.gate_proj", "model.layers.25.mlp.experts.23.gate_proj", "model.layers.25.mlp.experts.24.gate_proj", "model.layers.25.mlp.experts.25.gate_proj", "model.layers.25.mlp.experts.26.gate_proj", "model.layers.25.mlp.experts.27.gate_proj", "model.layers.25.mlp.experts.28.gate_proj", "model.layers.25.mlp.experts.29.gate_proj", "model.layers.25.mlp.experts.30.gate_proj", "model.layers.25.mlp.experts.31.gate_proj", "model.layers.25.mlp.experts.32.gate_proj", "model.layers.25.mlp.experts.33.gate_proj", "model.layers.25.mlp.experts.34.gate_proj", "model.layers.25.mlp.experts.35.gate_proj", "model.layers.25.mlp.experts.36.gate_proj", "model.layers.25.mlp.experts.37.gate_proj", "model.layers.25.mlp.experts.38.gate_proj", "model.layers.25.mlp.experts.39.gate_proj", "model.layers.25.mlp.experts.40.gate_proj", "model.layers.25.mlp.experts.41.gate_proj", "model.layers.25.mlp.experts.42.gate_proj", "model.layers.25.mlp.experts.43.gate_proj", "model.layers.25.mlp.experts.44.gate_proj", "model.layers.25.mlp.experts.45.gate_proj", "model.layers.25.mlp.experts.46.gate_proj", "model.layers.25.mlp.experts.47.gate_proj", "model.layers.25.mlp.experts.48.gate_proj", "model.layers.25.mlp.experts.49.gate_proj", "model.layers.25.mlp.experts.50.gate_proj", "model.layers.25.mlp.experts.51.gate_proj", "model.layers.25.mlp.experts.52.gate_proj", "model.layers.25.mlp.experts.53.gate_proj", "model.layers.25.mlp.experts.54.gate_proj", "model.layers.25.mlp.experts.55.gate_proj", "model.layers.25.mlp.experts.56.gate_proj", "model.layers.25.mlp.experts.57.gate_proj", "model.layers.25.mlp.experts.58.gate_proj", "model.layers.25.mlp.experts.59.gate_proj", "model.layers.25.mlp.experts.60.gate_proj", "model.layers.25.mlp.experts.61.gate_proj", "model.layers.25.mlp.experts.62.gate_proj", "model.layers.25.mlp.experts.63.gate_proj", "model.layers.25.mlp.experts.64.gate_proj", "model.layers.25.mlp.experts.65.gate_proj", "model.layers.25.mlp.experts.66.gate_proj", "model.layers.25.mlp.experts.67.gate_proj", "model.layers.25.mlp.experts.68.gate_proj", "model.layers.25.mlp.experts.69.gate_proj", "model.layers.25.mlp.experts.70.gate_proj", "model.layers.25.mlp.experts.71.gate_proj", "model.layers.25.mlp.experts.72.gate_proj", "model.layers.25.mlp.experts.73.gate_proj", "model.layers.25.mlp.experts.74.gate_proj", "model.layers.25.mlp.experts.75.gate_proj", "model.layers.25.mlp.experts.76.gate_proj", "model.layers.25.mlp.experts.77.gate_proj", "model.layers.25.mlp.experts.78.gate_proj", "model.layers.25.mlp.experts.79.gate_proj", "model.layers.25.mlp.experts.80.gate_proj", "model.layers.25.mlp.experts.81.gate_proj", "model.layers.25.mlp.experts.82.gate_proj", "model.layers.25.mlp.experts.83.gate_proj", "model.layers.25.mlp.experts.84.gate_proj", "model.layers.25.mlp.experts.85.gate_proj", "model.layers.25.mlp.experts.86.gate_proj", "model.layers.25.mlp.experts.87.gate_proj", "model.layers.25.mlp.experts.88.gate_proj", "model.layers.25.mlp.experts.89.gate_proj", "model.layers.25.mlp.experts.90.gate_proj", "model.layers.25.mlp.experts.91.gate_proj", "model.layers.25.mlp.experts.92.gate_proj", "model.layers.25.mlp.experts.93.gate_proj", "model.layers.25.mlp.experts.94.gate_proj", "model.layers.25.mlp.experts.95.gate_proj", "model.layers.25.mlp.experts.96.gate_proj", "model.layers.25.mlp.experts.97.gate_proj", "model.layers.25.mlp.experts.98.gate_proj", "model.layers.25.mlp.experts.99.gate_proj", "model.layers.25.mlp.experts.100.gate_proj", "model.layers.25.mlp.experts.101.gate_proj", "model.layers.25.mlp.experts.102.gate_proj", "model.layers.25.mlp.experts.103.gate_proj", "model.layers.25.mlp.experts.104.gate_proj", "model.layers.25.mlp.experts.105.gate_proj", "model.layers.25.mlp.experts.106.gate_proj", "model.layers.25.mlp.experts.107.gate_proj", "model.layers.25.mlp.experts.108.gate_proj", "model.layers.25.mlp.experts.109.gate_proj", "model.layers.25.mlp.experts.110.gate_proj", "model.layers.25.mlp.experts.111.gate_proj", "model.layers.25.mlp.experts.112.gate_proj", "model.layers.25.mlp.experts.113.gate_proj", "model.layers.25.mlp.experts.114.gate_proj", "model.layers.25.mlp.experts.115.gate_proj", "model.layers.25.mlp.experts.116.gate_proj", "model.layers.25.mlp.experts.117.gate_proj", "model.layers.25.mlp.experts.118.gate_proj", "model.layers.25.mlp.experts.119.gate_proj", "model.layers.25.mlp.experts.120.gate_proj", "model.layers.25.mlp.experts.121.gate_proj", "model.layers.25.mlp.experts.122.gate_proj", "model.layers.25.mlp.experts.123.gate_proj", "model.layers.25.mlp.experts.124.gate_proj", "model.layers.25.mlp.experts.125.gate_proj", "model.layers.25.mlp.experts.126.gate_proj", "model.layers.25.mlp.experts.127.gate_proj", "model.layers.25.mlp.experts.128.gate_proj", "model.layers.25.mlp.experts.129.gate_proj", "model.layers.25.mlp.experts.130.gate_proj", "model.layers.25.mlp.experts.131.gate_proj", "model.layers.25.mlp.experts.132.gate_proj", "model.layers.25.mlp.experts.133.gate_proj", "model.layers.25.mlp.experts.134.gate_proj", "model.layers.25.mlp.experts.135.gate_proj", "model.layers.25.mlp.experts.136.gate_proj", "model.layers.25.mlp.experts.137.gate_proj", "model.layers.25.mlp.experts.138.gate_proj", "model.layers.25.mlp.experts.139.gate_proj", "model.layers.25.mlp.experts.140.gate_proj", "model.layers.25.mlp.experts.141.gate_proj", "model.layers.25.mlp.experts.142.gate_proj", "model.layers.25.mlp.experts.143.gate_proj", "model.layers.25.mlp.experts.144.gate_proj", "model.layers.25.mlp.experts.145.gate_proj", "model.layers.25.mlp.experts.146.gate_proj", "model.layers.25.mlp.experts.147.gate_proj", "model.layers.25.mlp.experts.148.gate_proj", "model.layers.25.mlp.experts.149.gate_proj", "model.layers.25.mlp.experts.150.gate_proj", "model.layers.25.mlp.experts.151.gate_proj", "model.layers.25.mlp.experts.152.gate_proj", "model.layers.25.mlp.experts.153.gate_proj", "model.layers.25.mlp.experts.154.gate_proj", "model.layers.25.mlp.experts.155.gate_proj", "model.layers.25.mlp.experts.156.gate_proj", "model.layers.25.mlp.experts.157.gate_proj", "model.layers.25.mlp.experts.158.gate_proj", "model.layers.25.mlp.experts.159.gate_proj", "model.layers.25.mlp.experts.0.up_proj", "model.layers.25.mlp.experts.1.up_proj", "model.layers.25.mlp.experts.2.up_proj", "model.layers.25.mlp.experts.3.up_proj", "model.layers.25.mlp.experts.4.up_proj", "model.layers.25.mlp.experts.5.up_proj", "model.layers.25.mlp.experts.6.up_proj", "model.layers.25.mlp.experts.7.up_proj", "model.layers.25.mlp.experts.8.up_proj", "model.layers.25.mlp.experts.9.up_proj", "model.layers.25.mlp.experts.10.up_proj", "model.layers.25.mlp.experts.11.up_proj", "model.layers.25.mlp.experts.12.up_proj", "model.layers.25.mlp.experts.13.up_proj", "model.layers.25.mlp.experts.14.up_proj", "model.layers.25.mlp.experts.15.up_proj", "model.layers.25.mlp.experts.16.up_proj", "model.layers.25.mlp.experts.17.up_proj", "model.layers.25.mlp.experts.18.up_proj", "model.layers.25.mlp.experts.19.up_proj", "model.layers.25.mlp.experts.20.up_proj", "model.layers.25.mlp.experts.21.up_proj", "model.layers.25.mlp.experts.22.up_proj", "model.layers.25.mlp.experts.23.up_proj", "model.layers.25.mlp.experts.24.up_proj", "model.layers.25.mlp.experts.25.up_proj", "model.layers.25.mlp.experts.26.up_proj", "model.layers.25.mlp.experts.27.up_proj", "model.layers.25.mlp.experts.28.up_proj", "model.layers.25.mlp.experts.29.up_proj", "model.layers.25.mlp.experts.30.up_proj", "model.layers.25.mlp.experts.31.up_proj", "model.layers.25.mlp.experts.32.up_proj", "model.layers.25.mlp.experts.33.up_proj", "model.layers.25.mlp.experts.34.up_proj", "model.layers.25.mlp.experts.35.up_proj", "model.layers.25.mlp.experts.36.up_proj", "model.layers.25.mlp.experts.37.up_proj", "model.layers.25.mlp.experts.38.up_proj", "model.layers.25.mlp.experts.39.up_proj", "model.layers.25.mlp.experts.40.up_proj", "model.layers.25.mlp.experts.41.up_proj", "model.layers.25.mlp.experts.42.up_proj", "model.layers.25.mlp.experts.43.up_proj", "model.layers.25.mlp.experts.44.up_proj", "model.layers.25.mlp.experts.45.up_proj", "model.layers.25.mlp.experts.46.up_proj", "model.layers.25.mlp.experts.47.up_proj", "model.layers.25.mlp.experts.48.up_proj", "model.layers.25.mlp.experts.49.up_proj", "model.layers.25.mlp.experts.50.up_proj", "model.layers.25.mlp.experts.51.up_proj", "model.layers.25.mlp.experts.52.up_proj", "model.layers.25.mlp.experts.53.up_proj", "model.layers.25.mlp.experts.54.up_proj", "model.layers.25.mlp.experts.55.up_proj", "model.layers.25.mlp.experts.56.up_proj", "model.layers.25.mlp.experts.57.up_proj", "model.layers.25.mlp.experts.58.up_proj", "model.layers.25.mlp.experts.59.up_proj", "model.layers.25.mlp.experts.60.up_proj", "model.layers.25.mlp.experts.61.up_proj", "model.layers.25.mlp.experts.62.up_proj", "model.layers.25.mlp.experts.63.up_proj", "model.layers.25.mlp.experts.64.up_proj", "model.layers.25.mlp.experts.65.up_proj", "model.layers.25.mlp.experts.66.up_proj", "model.layers.25.mlp.experts.67.up_proj", "model.layers.25.mlp.experts.68.up_proj", "model.layers.25.mlp.experts.69.up_proj", "model.layers.25.mlp.experts.70.up_proj", "model.layers.25.mlp.experts.71.up_proj", "model.layers.25.mlp.experts.72.up_proj", "model.layers.25.mlp.experts.73.up_proj", "model.layers.25.mlp.experts.74.up_proj", "model.layers.25.mlp.experts.75.up_proj", "model.layers.25.mlp.experts.76.up_proj", "model.layers.25.mlp.experts.77.up_proj", "model.layers.25.mlp.experts.78.up_proj", "model.layers.25.mlp.experts.79.up_proj", "model.layers.25.mlp.experts.80.up_proj", "model.layers.25.mlp.experts.81.up_proj", "model.layers.25.mlp.experts.82.up_proj", "model.layers.25.mlp.experts.83.up_proj", "model.layers.25.mlp.experts.84.up_proj", "model.layers.25.mlp.experts.85.up_proj", "model.layers.25.mlp.experts.86.up_proj", "model.layers.25.mlp.experts.87.up_proj", "model.layers.25.mlp.experts.88.up_proj", "model.layers.25.mlp.experts.89.up_proj", "model.layers.25.mlp.experts.90.up_proj", "model.layers.25.mlp.experts.91.up_proj", "model.layers.25.mlp.experts.92.up_proj", "model.layers.25.mlp.experts.93.up_proj", "model.layers.25.mlp.experts.94.up_proj", "model.layers.25.mlp.experts.95.up_proj", "model.layers.25.mlp.experts.96.up_proj", "model.layers.25.mlp.experts.97.up_proj", "model.layers.25.mlp.experts.98.up_proj", "model.layers.25.mlp.experts.99.up_proj", "model.layers.25.mlp.experts.100.up_proj", "model.layers.25.mlp.experts.101.up_proj", "model.layers.25.mlp.experts.102.up_proj", "model.layers.25.mlp.experts.103.up_proj", "model.layers.25.mlp.experts.104.up_proj", "model.layers.25.mlp.experts.105.up_proj", "model.layers.25.mlp.experts.106.up_proj", "model.layers.25.mlp.experts.107.up_proj", "model.layers.25.mlp.experts.108.up_proj", "model.layers.25.mlp.experts.109.up_proj", "model.layers.25.mlp.experts.110.up_proj", "model.layers.25.mlp.experts.111.up_proj", "model.layers.25.mlp.experts.112.up_proj", "model.layers.25.mlp.experts.113.up_proj", "model.layers.25.mlp.experts.114.up_proj", "model.layers.25.mlp.experts.115.up_proj", "model.layers.25.mlp.experts.116.up_proj", "model.layers.25.mlp.experts.117.up_proj", "model.layers.25.mlp.experts.118.up_proj", "model.layers.25.mlp.experts.119.up_proj", "model.layers.25.mlp.experts.120.up_proj", "model.layers.25.mlp.experts.121.up_proj", "model.layers.25.mlp.experts.122.up_proj", "model.layers.25.mlp.experts.123.up_proj", "model.layers.25.mlp.experts.124.up_proj", "model.layers.25.mlp.experts.125.up_proj", "model.layers.25.mlp.experts.126.up_proj", "model.layers.25.mlp.experts.127.up_proj", "model.layers.25.mlp.experts.128.up_proj", "model.layers.25.mlp.experts.129.up_proj", "model.layers.25.mlp.experts.130.up_proj", "model.layers.25.mlp.experts.131.up_proj", "model.layers.25.mlp.experts.132.up_proj", "model.layers.25.mlp.experts.133.up_proj", "model.layers.25.mlp.experts.134.up_proj", "model.layers.25.mlp.experts.135.up_proj", "model.layers.25.mlp.experts.136.up_proj", "model.layers.25.mlp.experts.137.up_proj", "model.layers.25.mlp.experts.138.up_proj", "model.layers.25.mlp.experts.139.up_proj", "model.layers.25.mlp.experts.140.up_proj", "model.layers.25.mlp.experts.141.up_proj", "model.layers.25.mlp.experts.142.up_proj", "model.layers.25.mlp.experts.143.up_proj", "model.layers.25.mlp.experts.144.up_proj", "model.layers.25.mlp.experts.145.up_proj", "model.layers.25.mlp.experts.146.up_proj", "model.layers.25.mlp.experts.147.up_proj", "model.layers.25.mlp.experts.148.up_proj", "model.layers.25.mlp.experts.149.up_proj", "model.layers.25.mlp.experts.150.up_proj", "model.layers.25.mlp.experts.151.up_proj", "model.layers.25.mlp.experts.152.up_proj", "model.layers.25.mlp.experts.153.up_proj", "model.layers.25.mlp.experts.154.up_proj", "model.layers.25.mlp.experts.155.up_proj", "model.layers.25.mlp.experts.156.up_proj", "model.layers.25.mlp.experts.157.up_proj", "model.layers.25.mlp.experts.158.up_proj", "model.layers.25.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.002924625575542461, "dbits": 2516582400 } ] }, { "idx": 152, "layers": [ "model.layers.25.mlp.experts.0.down_proj", "model.layers.25.mlp.experts.1.down_proj", "model.layers.25.mlp.experts.2.down_proj", "model.layers.25.mlp.experts.3.down_proj", "model.layers.25.mlp.experts.4.down_proj", "model.layers.25.mlp.experts.5.down_proj", "model.layers.25.mlp.experts.6.down_proj", "model.layers.25.mlp.experts.7.down_proj", "model.layers.25.mlp.experts.8.down_proj", "model.layers.25.mlp.experts.9.down_proj", "model.layers.25.mlp.experts.10.down_proj", "model.layers.25.mlp.experts.11.down_proj", "model.layers.25.mlp.experts.12.down_proj", "model.layers.25.mlp.experts.13.down_proj", "model.layers.25.mlp.experts.14.down_proj", "model.layers.25.mlp.experts.15.down_proj", "model.layers.25.mlp.experts.16.down_proj", "model.layers.25.mlp.experts.17.down_proj", "model.layers.25.mlp.experts.18.down_proj", "model.layers.25.mlp.experts.19.down_proj", "model.layers.25.mlp.experts.20.down_proj", "model.layers.25.mlp.experts.21.down_proj", "model.layers.25.mlp.experts.22.down_proj", "model.layers.25.mlp.experts.23.down_proj", "model.layers.25.mlp.experts.24.down_proj", "model.layers.25.mlp.experts.25.down_proj", "model.layers.25.mlp.experts.26.down_proj", "model.layers.25.mlp.experts.27.down_proj", "model.layers.25.mlp.experts.28.down_proj", "model.layers.25.mlp.experts.29.down_proj", "model.layers.25.mlp.experts.30.down_proj", "model.layers.25.mlp.experts.31.down_proj", "model.layers.25.mlp.experts.32.down_proj", "model.layers.25.mlp.experts.33.down_proj", "model.layers.25.mlp.experts.34.down_proj", "model.layers.25.mlp.experts.35.down_proj", "model.layers.25.mlp.experts.36.down_proj", "model.layers.25.mlp.experts.37.down_proj", "model.layers.25.mlp.experts.38.down_proj", "model.layers.25.mlp.experts.39.down_proj", "model.layers.25.mlp.experts.40.down_proj", "model.layers.25.mlp.experts.41.down_proj", "model.layers.25.mlp.experts.42.down_proj", "model.layers.25.mlp.experts.43.down_proj", "model.layers.25.mlp.experts.44.down_proj", "model.layers.25.mlp.experts.45.down_proj", "model.layers.25.mlp.experts.46.down_proj", "model.layers.25.mlp.experts.47.down_proj", "model.layers.25.mlp.experts.48.down_proj", "model.layers.25.mlp.experts.49.down_proj", "model.layers.25.mlp.experts.50.down_proj", "model.layers.25.mlp.experts.51.down_proj", "model.layers.25.mlp.experts.52.down_proj", "model.layers.25.mlp.experts.53.down_proj", "model.layers.25.mlp.experts.54.down_proj", "model.layers.25.mlp.experts.55.down_proj", "model.layers.25.mlp.experts.56.down_proj", "model.layers.25.mlp.experts.57.down_proj", "model.layers.25.mlp.experts.58.down_proj", "model.layers.25.mlp.experts.59.down_proj", "model.layers.25.mlp.experts.60.down_proj", "model.layers.25.mlp.experts.61.down_proj", "model.layers.25.mlp.experts.62.down_proj", "model.layers.25.mlp.experts.63.down_proj", "model.layers.25.mlp.experts.64.down_proj", "model.layers.25.mlp.experts.65.down_proj", "model.layers.25.mlp.experts.66.down_proj", "model.layers.25.mlp.experts.67.down_proj", "model.layers.25.mlp.experts.68.down_proj", "model.layers.25.mlp.experts.69.down_proj", "model.layers.25.mlp.experts.70.down_proj", "model.layers.25.mlp.experts.71.down_proj", "model.layers.25.mlp.experts.72.down_proj", "model.layers.25.mlp.experts.73.down_proj", "model.layers.25.mlp.experts.74.down_proj", "model.layers.25.mlp.experts.75.down_proj", "model.layers.25.mlp.experts.76.down_proj", "model.layers.25.mlp.experts.77.down_proj", "model.layers.25.mlp.experts.78.down_proj", "model.layers.25.mlp.experts.79.down_proj", "model.layers.25.mlp.experts.80.down_proj", "model.layers.25.mlp.experts.81.down_proj", "model.layers.25.mlp.experts.82.down_proj", "model.layers.25.mlp.experts.83.down_proj", "model.layers.25.mlp.experts.84.down_proj", "model.layers.25.mlp.experts.85.down_proj", "model.layers.25.mlp.experts.86.down_proj", "model.layers.25.mlp.experts.87.down_proj", "model.layers.25.mlp.experts.88.down_proj", "model.layers.25.mlp.experts.89.down_proj", "model.layers.25.mlp.experts.90.down_proj", "model.layers.25.mlp.experts.91.down_proj", "model.layers.25.mlp.experts.92.down_proj", "model.layers.25.mlp.experts.93.down_proj", "model.layers.25.mlp.experts.94.down_proj", "model.layers.25.mlp.experts.95.down_proj", "model.layers.25.mlp.experts.96.down_proj", "model.layers.25.mlp.experts.97.down_proj", "model.layers.25.mlp.experts.98.down_proj", "model.layers.25.mlp.experts.99.down_proj", "model.layers.25.mlp.experts.100.down_proj", "model.layers.25.mlp.experts.101.down_proj", "model.layers.25.mlp.experts.102.down_proj", "model.layers.25.mlp.experts.103.down_proj", "model.layers.25.mlp.experts.104.down_proj", "model.layers.25.mlp.experts.105.down_proj", "model.layers.25.mlp.experts.106.down_proj", "model.layers.25.mlp.experts.107.down_proj", "model.layers.25.mlp.experts.108.down_proj", "model.layers.25.mlp.experts.109.down_proj", "model.layers.25.mlp.experts.110.down_proj", "model.layers.25.mlp.experts.111.down_proj", "model.layers.25.mlp.experts.112.down_proj", "model.layers.25.mlp.experts.113.down_proj", "model.layers.25.mlp.experts.114.down_proj", "model.layers.25.mlp.experts.115.down_proj", "model.layers.25.mlp.experts.116.down_proj", "model.layers.25.mlp.experts.117.down_proj", "model.layers.25.mlp.experts.118.down_proj", "model.layers.25.mlp.experts.119.down_proj", "model.layers.25.mlp.experts.120.down_proj", "model.layers.25.mlp.experts.121.down_proj", "model.layers.25.mlp.experts.122.down_proj", "model.layers.25.mlp.experts.123.down_proj", "model.layers.25.mlp.experts.124.down_proj", "model.layers.25.mlp.experts.125.down_proj", "model.layers.25.mlp.experts.126.down_proj", "model.layers.25.mlp.experts.127.down_proj", "model.layers.25.mlp.experts.128.down_proj", "model.layers.25.mlp.experts.129.down_proj", "model.layers.25.mlp.experts.130.down_proj", "model.layers.25.mlp.experts.131.down_proj", "model.layers.25.mlp.experts.132.down_proj", "model.layers.25.mlp.experts.133.down_proj", "model.layers.25.mlp.experts.134.down_proj", "model.layers.25.mlp.experts.135.down_proj", "model.layers.25.mlp.experts.136.down_proj", "model.layers.25.mlp.experts.137.down_proj", "model.layers.25.mlp.experts.138.down_proj", "model.layers.25.mlp.experts.139.down_proj", "model.layers.25.mlp.experts.140.down_proj", "model.layers.25.mlp.experts.141.down_proj", "model.layers.25.mlp.experts.142.down_proj", "model.layers.25.mlp.experts.143.down_proj", "model.layers.25.mlp.experts.144.down_proj", "model.layers.25.mlp.experts.145.down_proj", "model.layers.25.mlp.experts.146.down_proj", "model.layers.25.mlp.experts.147.down_proj", "model.layers.25.mlp.experts.148.down_proj", "model.layers.25.mlp.experts.149.down_proj", "model.layers.25.mlp.experts.150.down_proj", "model.layers.25.mlp.experts.151.down_proj", "model.layers.25.mlp.experts.152.down_proj", "model.layers.25.mlp.experts.153.down_proj", "model.layers.25.mlp.experts.154.down_proj", "model.layers.25.mlp.experts.155.down_proj", "model.layers.25.mlp.experts.156.down_proj", "model.layers.25.mlp.experts.157.down_proj", "model.layers.25.mlp.experts.158.down_proj", "model.layers.25.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0021624580025673135, "dbits": 1258291200 } ] }, { "idx": 153, "layers": [ "model.layers.26.self_attn.q_proj" ], "candidates": [ { "dkld": 0.000395800918340683, "dbits": 62914560 } ] }, { "idx": 154, "layers": [ "model.layers.26.self_attn.k_proj", "model.layers.26.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0007043384015559817, "dbits": 10485760 } ] }, { "idx": 155, "layers": [ "model.layers.26.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0004433877766132355, "dbits": 62914560 } ] }, { "idx": 156, "layers": [ "model.layers.26.mlp.shared_experts.gate_proj", "model.layers.26.mlp.shared_experts.up_proj", "model.layers.26.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.008344546705484401, "dbits": 23592960 } ] }, { "idx": 157, "layers": [ "model.layers.26.mlp.experts.0.gate_proj", "model.layers.26.mlp.experts.1.gate_proj", "model.layers.26.mlp.experts.2.gate_proj", "model.layers.26.mlp.experts.3.gate_proj", "model.layers.26.mlp.experts.4.gate_proj", "model.layers.26.mlp.experts.5.gate_proj", "model.layers.26.mlp.experts.6.gate_proj", "model.layers.26.mlp.experts.7.gate_proj", "model.layers.26.mlp.experts.8.gate_proj", "model.layers.26.mlp.experts.9.gate_proj", "model.layers.26.mlp.experts.10.gate_proj", "model.layers.26.mlp.experts.11.gate_proj", "model.layers.26.mlp.experts.12.gate_proj", "model.layers.26.mlp.experts.13.gate_proj", "model.layers.26.mlp.experts.14.gate_proj", "model.layers.26.mlp.experts.15.gate_proj", "model.layers.26.mlp.experts.16.gate_proj", "model.layers.26.mlp.experts.17.gate_proj", "model.layers.26.mlp.experts.18.gate_proj", "model.layers.26.mlp.experts.19.gate_proj", "model.layers.26.mlp.experts.20.gate_proj", "model.layers.26.mlp.experts.21.gate_proj", "model.layers.26.mlp.experts.22.gate_proj", "model.layers.26.mlp.experts.23.gate_proj", "model.layers.26.mlp.experts.24.gate_proj", "model.layers.26.mlp.experts.25.gate_proj", "model.layers.26.mlp.experts.26.gate_proj", "model.layers.26.mlp.experts.27.gate_proj", "model.layers.26.mlp.experts.28.gate_proj", "model.layers.26.mlp.experts.29.gate_proj", "model.layers.26.mlp.experts.30.gate_proj", "model.layers.26.mlp.experts.31.gate_proj", "model.layers.26.mlp.experts.32.gate_proj", "model.layers.26.mlp.experts.33.gate_proj", "model.layers.26.mlp.experts.34.gate_proj", "model.layers.26.mlp.experts.35.gate_proj", "model.layers.26.mlp.experts.36.gate_proj", "model.layers.26.mlp.experts.37.gate_proj", "model.layers.26.mlp.experts.38.gate_proj", "model.layers.26.mlp.experts.39.gate_proj", "model.layers.26.mlp.experts.40.gate_proj", "model.layers.26.mlp.experts.41.gate_proj", "model.layers.26.mlp.experts.42.gate_proj", "model.layers.26.mlp.experts.43.gate_proj", "model.layers.26.mlp.experts.44.gate_proj", "model.layers.26.mlp.experts.45.gate_proj", "model.layers.26.mlp.experts.46.gate_proj", "model.layers.26.mlp.experts.47.gate_proj", "model.layers.26.mlp.experts.48.gate_proj", "model.layers.26.mlp.experts.49.gate_proj", "model.layers.26.mlp.experts.50.gate_proj", "model.layers.26.mlp.experts.51.gate_proj", "model.layers.26.mlp.experts.52.gate_proj", "model.layers.26.mlp.experts.53.gate_proj", "model.layers.26.mlp.experts.54.gate_proj", "model.layers.26.mlp.experts.55.gate_proj", "model.layers.26.mlp.experts.56.gate_proj", "model.layers.26.mlp.experts.57.gate_proj", "model.layers.26.mlp.experts.58.gate_proj", "model.layers.26.mlp.experts.59.gate_proj", "model.layers.26.mlp.experts.60.gate_proj", "model.layers.26.mlp.experts.61.gate_proj", "model.layers.26.mlp.experts.62.gate_proj", "model.layers.26.mlp.experts.63.gate_proj", "model.layers.26.mlp.experts.64.gate_proj", "model.layers.26.mlp.experts.65.gate_proj", "model.layers.26.mlp.experts.66.gate_proj", "model.layers.26.mlp.experts.67.gate_proj", "model.layers.26.mlp.experts.68.gate_proj", "model.layers.26.mlp.experts.69.gate_proj", "model.layers.26.mlp.experts.70.gate_proj", "model.layers.26.mlp.experts.71.gate_proj", "model.layers.26.mlp.experts.72.gate_proj", "model.layers.26.mlp.experts.73.gate_proj", "model.layers.26.mlp.experts.74.gate_proj", "model.layers.26.mlp.experts.75.gate_proj", "model.layers.26.mlp.experts.76.gate_proj", "model.layers.26.mlp.experts.77.gate_proj", "model.layers.26.mlp.experts.78.gate_proj", "model.layers.26.mlp.experts.79.gate_proj", "model.layers.26.mlp.experts.80.gate_proj", "model.layers.26.mlp.experts.81.gate_proj", "model.layers.26.mlp.experts.82.gate_proj", "model.layers.26.mlp.experts.83.gate_proj", "model.layers.26.mlp.experts.84.gate_proj", "model.layers.26.mlp.experts.85.gate_proj", "model.layers.26.mlp.experts.86.gate_proj", "model.layers.26.mlp.experts.87.gate_proj", "model.layers.26.mlp.experts.88.gate_proj", "model.layers.26.mlp.experts.89.gate_proj", "model.layers.26.mlp.experts.90.gate_proj", "model.layers.26.mlp.experts.91.gate_proj", "model.layers.26.mlp.experts.92.gate_proj", "model.layers.26.mlp.experts.93.gate_proj", "model.layers.26.mlp.experts.94.gate_proj", "model.layers.26.mlp.experts.95.gate_proj", "model.layers.26.mlp.experts.96.gate_proj", "model.layers.26.mlp.experts.97.gate_proj", "model.layers.26.mlp.experts.98.gate_proj", "model.layers.26.mlp.experts.99.gate_proj", "model.layers.26.mlp.experts.100.gate_proj", "model.layers.26.mlp.experts.101.gate_proj", "model.layers.26.mlp.experts.102.gate_proj", "model.layers.26.mlp.experts.103.gate_proj", "model.layers.26.mlp.experts.104.gate_proj", "model.layers.26.mlp.experts.105.gate_proj", "model.layers.26.mlp.experts.106.gate_proj", "model.layers.26.mlp.experts.107.gate_proj", "model.layers.26.mlp.experts.108.gate_proj", "model.layers.26.mlp.experts.109.gate_proj", "model.layers.26.mlp.experts.110.gate_proj", "model.layers.26.mlp.experts.111.gate_proj", "model.layers.26.mlp.experts.112.gate_proj", "model.layers.26.mlp.experts.113.gate_proj", "model.layers.26.mlp.experts.114.gate_proj", "model.layers.26.mlp.experts.115.gate_proj", "model.layers.26.mlp.experts.116.gate_proj", "model.layers.26.mlp.experts.117.gate_proj", "model.layers.26.mlp.experts.118.gate_proj", "model.layers.26.mlp.experts.119.gate_proj", "model.layers.26.mlp.experts.120.gate_proj", "model.layers.26.mlp.experts.121.gate_proj", "model.layers.26.mlp.experts.122.gate_proj", "model.layers.26.mlp.experts.123.gate_proj", "model.layers.26.mlp.experts.124.gate_proj", "model.layers.26.mlp.experts.125.gate_proj", "model.layers.26.mlp.experts.126.gate_proj", "model.layers.26.mlp.experts.127.gate_proj", "model.layers.26.mlp.experts.128.gate_proj", "model.layers.26.mlp.experts.129.gate_proj", "model.layers.26.mlp.experts.130.gate_proj", "model.layers.26.mlp.experts.131.gate_proj", "model.layers.26.mlp.experts.132.gate_proj", "model.layers.26.mlp.experts.133.gate_proj", "model.layers.26.mlp.experts.134.gate_proj", "model.layers.26.mlp.experts.135.gate_proj", "model.layers.26.mlp.experts.136.gate_proj", "model.layers.26.mlp.experts.137.gate_proj", "model.layers.26.mlp.experts.138.gate_proj", "model.layers.26.mlp.experts.139.gate_proj", "model.layers.26.mlp.experts.140.gate_proj", "model.layers.26.mlp.experts.141.gate_proj", "model.layers.26.mlp.experts.142.gate_proj", "model.layers.26.mlp.experts.143.gate_proj", "model.layers.26.mlp.experts.144.gate_proj", "model.layers.26.mlp.experts.145.gate_proj", "model.layers.26.mlp.experts.146.gate_proj", "model.layers.26.mlp.experts.147.gate_proj", "model.layers.26.mlp.experts.148.gate_proj", "model.layers.26.mlp.experts.149.gate_proj", "model.layers.26.mlp.experts.150.gate_proj", "model.layers.26.mlp.experts.151.gate_proj", "model.layers.26.mlp.experts.152.gate_proj", "model.layers.26.mlp.experts.153.gate_proj", "model.layers.26.mlp.experts.154.gate_proj", "model.layers.26.mlp.experts.155.gate_proj", "model.layers.26.mlp.experts.156.gate_proj", "model.layers.26.mlp.experts.157.gate_proj", "model.layers.26.mlp.experts.158.gate_proj", "model.layers.26.mlp.experts.159.gate_proj", "model.layers.26.mlp.experts.0.up_proj", "model.layers.26.mlp.experts.1.up_proj", "model.layers.26.mlp.experts.2.up_proj", "model.layers.26.mlp.experts.3.up_proj", "model.layers.26.mlp.experts.4.up_proj", "model.layers.26.mlp.experts.5.up_proj", "model.layers.26.mlp.experts.6.up_proj", "model.layers.26.mlp.experts.7.up_proj", "model.layers.26.mlp.experts.8.up_proj", "model.layers.26.mlp.experts.9.up_proj", "model.layers.26.mlp.experts.10.up_proj", "model.layers.26.mlp.experts.11.up_proj", "model.layers.26.mlp.experts.12.up_proj", "model.layers.26.mlp.experts.13.up_proj", "model.layers.26.mlp.experts.14.up_proj", "model.layers.26.mlp.experts.15.up_proj", "model.layers.26.mlp.experts.16.up_proj", "model.layers.26.mlp.experts.17.up_proj", "model.layers.26.mlp.experts.18.up_proj", "model.layers.26.mlp.experts.19.up_proj", "model.layers.26.mlp.experts.20.up_proj", "model.layers.26.mlp.experts.21.up_proj", "model.layers.26.mlp.experts.22.up_proj", "model.layers.26.mlp.experts.23.up_proj", "model.layers.26.mlp.experts.24.up_proj", "model.layers.26.mlp.experts.25.up_proj", "model.layers.26.mlp.experts.26.up_proj", "model.layers.26.mlp.experts.27.up_proj", "model.layers.26.mlp.experts.28.up_proj", "model.layers.26.mlp.experts.29.up_proj", "model.layers.26.mlp.experts.30.up_proj", "model.layers.26.mlp.experts.31.up_proj", "model.layers.26.mlp.experts.32.up_proj", "model.layers.26.mlp.experts.33.up_proj", "model.layers.26.mlp.experts.34.up_proj", "model.layers.26.mlp.experts.35.up_proj", "model.layers.26.mlp.experts.36.up_proj", "model.layers.26.mlp.experts.37.up_proj", "model.layers.26.mlp.experts.38.up_proj", "model.layers.26.mlp.experts.39.up_proj", "model.layers.26.mlp.experts.40.up_proj", "model.layers.26.mlp.experts.41.up_proj", "model.layers.26.mlp.experts.42.up_proj", "model.layers.26.mlp.experts.43.up_proj", "model.layers.26.mlp.experts.44.up_proj", "model.layers.26.mlp.experts.45.up_proj", "model.layers.26.mlp.experts.46.up_proj", "model.layers.26.mlp.experts.47.up_proj", "model.layers.26.mlp.experts.48.up_proj", "model.layers.26.mlp.experts.49.up_proj", "model.layers.26.mlp.experts.50.up_proj", "model.layers.26.mlp.experts.51.up_proj", "model.layers.26.mlp.experts.52.up_proj", "model.layers.26.mlp.experts.53.up_proj", "model.layers.26.mlp.experts.54.up_proj", "model.layers.26.mlp.experts.55.up_proj", "model.layers.26.mlp.experts.56.up_proj", "model.layers.26.mlp.experts.57.up_proj", "model.layers.26.mlp.experts.58.up_proj", "model.layers.26.mlp.experts.59.up_proj", "model.layers.26.mlp.experts.60.up_proj", "model.layers.26.mlp.experts.61.up_proj", "model.layers.26.mlp.experts.62.up_proj", "model.layers.26.mlp.experts.63.up_proj", "model.layers.26.mlp.experts.64.up_proj", "model.layers.26.mlp.experts.65.up_proj", "model.layers.26.mlp.experts.66.up_proj", "model.layers.26.mlp.experts.67.up_proj", "model.layers.26.mlp.experts.68.up_proj", "model.layers.26.mlp.experts.69.up_proj", "model.layers.26.mlp.experts.70.up_proj", "model.layers.26.mlp.experts.71.up_proj", "model.layers.26.mlp.experts.72.up_proj", "model.layers.26.mlp.experts.73.up_proj", "model.layers.26.mlp.experts.74.up_proj", "model.layers.26.mlp.experts.75.up_proj", "model.layers.26.mlp.experts.76.up_proj", "model.layers.26.mlp.experts.77.up_proj", "model.layers.26.mlp.experts.78.up_proj", "model.layers.26.mlp.experts.79.up_proj", "model.layers.26.mlp.experts.80.up_proj", "model.layers.26.mlp.experts.81.up_proj", "model.layers.26.mlp.experts.82.up_proj", "model.layers.26.mlp.experts.83.up_proj", "model.layers.26.mlp.experts.84.up_proj", "model.layers.26.mlp.experts.85.up_proj", "model.layers.26.mlp.experts.86.up_proj", "model.layers.26.mlp.experts.87.up_proj", "model.layers.26.mlp.experts.88.up_proj", "model.layers.26.mlp.experts.89.up_proj", "model.layers.26.mlp.experts.90.up_proj", "model.layers.26.mlp.experts.91.up_proj", "model.layers.26.mlp.experts.92.up_proj", "model.layers.26.mlp.experts.93.up_proj", "model.layers.26.mlp.experts.94.up_proj", "model.layers.26.mlp.experts.95.up_proj", "model.layers.26.mlp.experts.96.up_proj", "model.layers.26.mlp.experts.97.up_proj", "model.layers.26.mlp.experts.98.up_proj", "model.layers.26.mlp.experts.99.up_proj", "model.layers.26.mlp.experts.100.up_proj", "model.layers.26.mlp.experts.101.up_proj", "model.layers.26.mlp.experts.102.up_proj", "model.layers.26.mlp.experts.103.up_proj", "model.layers.26.mlp.experts.104.up_proj", "model.layers.26.mlp.experts.105.up_proj", "model.layers.26.mlp.experts.106.up_proj", "model.layers.26.mlp.experts.107.up_proj", "model.layers.26.mlp.experts.108.up_proj", "model.layers.26.mlp.experts.109.up_proj", "model.layers.26.mlp.experts.110.up_proj", "model.layers.26.mlp.experts.111.up_proj", "model.layers.26.mlp.experts.112.up_proj", "model.layers.26.mlp.experts.113.up_proj", "model.layers.26.mlp.experts.114.up_proj", "model.layers.26.mlp.experts.115.up_proj", "model.layers.26.mlp.experts.116.up_proj", "model.layers.26.mlp.experts.117.up_proj", "model.layers.26.mlp.experts.118.up_proj", "model.layers.26.mlp.experts.119.up_proj", "model.layers.26.mlp.experts.120.up_proj", "model.layers.26.mlp.experts.121.up_proj", "model.layers.26.mlp.experts.122.up_proj", "model.layers.26.mlp.experts.123.up_proj", "model.layers.26.mlp.experts.124.up_proj", "model.layers.26.mlp.experts.125.up_proj", "model.layers.26.mlp.experts.126.up_proj", "model.layers.26.mlp.experts.127.up_proj", "model.layers.26.mlp.experts.128.up_proj", "model.layers.26.mlp.experts.129.up_proj", "model.layers.26.mlp.experts.130.up_proj", "model.layers.26.mlp.experts.131.up_proj", "model.layers.26.mlp.experts.132.up_proj", "model.layers.26.mlp.experts.133.up_proj", "model.layers.26.mlp.experts.134.up_proj", "model.layers.26.mlp.experts.135.up_proj", "model.layers.26.mlp.experts.136.up_proj", "model.layers.26.mlp.experts.137.up_proj", "model.layers.26.mlp.experts.138.up_proj", "model.layers.26.mlp.experts.139.up_proj", "model.layers.26.mlp.experts.140.up_proj", "model.layers.26.mlp.experts.141.up_proj", "model.layers.26.mlp.experts.142.up_proj", "model.layers.26.mlp.experts.143.up_proj", "model.layers.26.mlp.experts.144.up_proj", "model.layers.26.mlp.experts.145.up_proj", "model.layers.26.mlp.experts.146.up_proj", "model.layers.26.mlp.experts.147.up_proj", "model.layers.26.mlp.experts.148.up_proj", "model.layers.26.mlp.experts.149.up_proj", "model.layers.26.mlp.experts.150.up_proj", "model.layers.26.mlp.experts.151.up_proj", "model.layers.26.mlp.experts.152.up_proj", "model.layers.26.mlp.experts.153.up_proj", "model.layers.26.mlp.experts.154.up_proj", "model.layers.26.mlp.experts.155.up_proj", "model.layers.26.mlp.experts.156.up_proj", "model.layers.26.mlp.experts.157.up_proj", "model.layers.26.mlp.experts.158.up_proj", "model.layers.26.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0024757556617260312, "dbits": 2516582400 } ] }, { "idx": 158, "layers": [ "model.layers.26.mlp.experts.0.down_proj", "model.layers.26.mlp.experts.1.down_proj", "model.layers.26.mlp.experts.2.down_proj", "model.layers.26.mlp.experts.3.down_proj", "model.layers.26.mlp.experts.4.down_proj", "model.layers.26.mlp.experts.5.down_proj", "model.layers.26.mlp.experts.6.down_proj", "model.layers.26.mlp.experts.7.down_proj", "model.layers.26.mlp.experts.8.down_proj", "model.layers.26.mlp.experts.9.down_proj", "model.layers.26.mlp.experts.10.down_proj", "model.layers.26.mlp.experts.11.down_proj", "model.layers.26.mlp.experts.12.down_proj", "model.layers.26.mlp.experts.13.down_proj", "model.layers.26.mlp.experts.14.down_proj", "model.layers.26.mlp.experts.15.down_proj", "model.layers.26.mlp.experts.16.down_proj", "model.layers.26.mlp.experts.17.down_proj", "model.layers.26.mlp.experts.18.down_proj", "model.layers.26.mlp.experts.19.down_proj", "model.layers.26.mlp.experts.20.down_proj", "model.layers.26.mlp.experts.21.down_proj", "model.layers.26.mlp.experts.22.down_proj", "model.layers.26.mlp.experts.23.down_proj", "model.layers.26.mlp.experts.24.down_proj", "model.layers.26.mlp.experts.25.down_proj", "model.layers.26.mlp.experts.26.down_proj", "model.layers.26.mlp.experts.27.down_proj", "model.layers.26.mlp.experts.28.down_proj", "model.layers.26.mlp.experts.29.down_proj", "model.layers.26.mlp.experts.30.down_proj", "model.layers.26.mlp.experts.31.down_proj", "model.layers.26.mlp.experts.32.down_proj", "model.layers.26.mlp.experts.33.down_proj", "model.layers.26.mlp.experts.34.down_proj", "model.layers.26.mlp.experts.35.down_proj", "model.layers.26.mlp.experts.36.down_proj", "model.layers.26.mlp.experts.37.down_proj", "model.layers.26.mlp.experts.38.down_proj", "model.layers.26.mlp.experts.39.down_proj", "model.layers.26.mlp.experts.40.down_proj", "model.layers.26.mlp.experts.41.down_proj", "model.layers.26.mlp.experts.42.down_proj", "model.layers.26.mlp.experts.43.down_proj", "model.layers.26.mlp.experts.44.down_proj", "model.layers.26.mlp.experts.45.down_proj", "model.layers.26.mlp.experts.46.down_proj", "model.layers.26.mlp.experts.47.down_proj", "model.layers.26.mlp.experts.48.down_proj", "model.layers.26.mlp.experts.49.down_proj", "model.layers.26.mlp.experts.50.down_proj", "model.layers.26.mlp.experts.51.down_proj", "model.layers.26.mlp.experts.52.down_proj", "model.layers.26.mlp.experts.53.down_proj", "model.layers.26.mlp.experts.54.down_proj", "model.layers.26.mlp.experts.55.down_proj", "model.layers.26.mlp.experts.56.down_proj", "model.layers.26.mlp.experts.57.down_proj", "model.layers.26.mlp.experts.58.down_proj", "model.layers.26.mlp.experts.59.down_proj", "model.layers.26.mlp.experts.60.down_proj", "model.layers.26.mlp.experts.61.down_proj", "model.layers.26.mlp.experts.62.down_proj", "model.layers.26.mlp.experts.63.down_proj", "model.layers.26.mlp.experts.64.down_proj", "model.layers.26.mlp.experts.65.down_proj", "model.layers.26.mlp.experts.66.down_proj", "model.layers.26.mlp.experts.67.down_proj", "model.layers.26.mlp.experts.68.down_proj", "model.layers.26.mlp.experts.69.down_proj", "model.layers.26.mlp.experts.70.down_proj", "model.layers.26.mlp.experts.71.down_proj", "model.layers.26.mlp.experts.72.down_proj", "model.layers.26.mlp.experts.73.down_proj", "model.layers.26.mlp.experts.74.down_proj", "model.layers.26.mlp.experts.75.down_proj", "model.layers.26.mlp.experts.76.down_proj", "model.layers.26.mlp.experts.77.down_proj", "model.layers.26.mlp.experts.78.down_proj", "model.layers.26.mlp.experts.79.down_proj", "model.layers.26.mlp.experts.80.down_proj", "model.layers.26.mlp.experts.81.down_proj", "model.layers.26.mlp.experts.82.down_proj", "model.layers.26.mlp.experts.83.down_proj", "model.layers.26.mlp.experts.84.down_proj", "model.layers.26.mlp.experts.85.down_proj", "model.layers.26.mlp.experts.86.down_proj", "model.layers.26.mlp.experts.87.down_proj", "model.layers.26.mlp.experts.88.down_proj", "model.layers.26.mlp.experts.89.down_proj", "model.layers.26.mlp.experts.90.down_proj", "model.layers.26.mlp.experts.91.down_proj", "model.layers.26.mlp.experts.92.down_proj", "model.layers.26.mlp.experts.93.down_proj", "model.layers.26.mlp.experts.94.down_proj", "model.layers.26.mlp.experts.95.down_proj", "model.layers.26.mlp.experts.96.down_proj", "model.layers.26.mlp.experts.97.down_proj", "model.layers.26.mlp.experts.98.down_proj", "model.layers.26.mlp.experts.99.down_proj", "model.layers.26.mlp.experts.100.down_proj", "model.layers.26.mlp.experts.101.down_proj", "model.layers.26.mlp.experts.102.down_proj", "model.layers.26.mlp.experts.103.down_proj", "model.layers.26.mlp.experts.104.down_proj", "model.layers.26.mlp.experts.105.down_proj", "model.layers.26.mlp.experts.106.down_proj", "model.layers.26.mlp.experts.107.down_proj", "model.layers.26.mlp.experts.108.down_proj", "model.layers.26.mlp.experts.109.down_proj", "model.layers.26.mlp.experts.110.down_proj", "model.layers.26.mlp.experts.111.down_proj", "model.layers.26.mlp.experts.112.down_proj", "model.layers.26.mlp.experts.113.down_proj", "model.layers.26.mlp.experts.114.down_proj", "model.layers.26.mlp.experts.115.down_proj", "model.layers.26.mlp.experts.116.down_proj", "model.layers.26.mlp.experts.117.down_proj", "model.layers.26.mlp.experts.118.down_proj", "model.layers.26.mlp.experts.119.down_proj", "model.layers.26.mlp.experts.120.down_proj", "model.layers.26.mlp.experts.121.down_proj", "model.layers.26.mlp.experts.122.down_proj", "model.layers.26.mlp.experts.123.down_proj", "model.layers.26.mlp.experts.124.down_proj", "model.layers.26.mlp.experts.125.down_proj", "model.layers.26.mlp.experts.126.down_proj", "model.layers.26.mlp.experts.127.down_proj", "model.layers.26.mlp.experts.128.down_proj", "model.layers.26.mlp.experts.129.down_proj", "model.layers.26.mlp.experts.130.down_proj", "model.layers.26.mlp.experts.131.down_proj", "model.layers.26.mlp.experts.132.down_proj", "model.layers.26.mlp.experts.133.down_proj", "model.layers.26.mlp.experts.134.down_proj", "model.layers.26.mlp.experts.135.down_proj", "model.layers.26.mlp.experts.136.down_proj", "model.layers.26.mlp.experts.137.down_proj", "model.layers.26.mlp.experts.138.down_proj", "model.layers.26.mlp.experts.139.down_proj", "model.layers.26.mlp.experts.140.down_proj", "model.layers.26.mlp.experts.141.down_proj", "model.layers.26.mlp.experts.142.down_proj", "model.layers.26.mlp.experts.143.down_proj", "model.layers.26.mlp.experts.144.down_proj", "model.layers.26.mlp.experts.145.down_proj", "model.layers.26.mlp.experts.146.down_proj", "model.layers.26.mlp.experts.147.down_proj", "model.layers.26.mlp.experts.148.down_proj", "model.layers.26.mlp.experts.149.down_proj", "model.layers.26.mlp.experts.150.down_proj", "model.layers.26.mlp.experts.151.down_proj", "model.layers.26.mlp.experts.152.down_proj", "model.layers.26.mlp.experts.153.down_proj", "model.layers.26.mlp.experts.154.down_proj", "model.layers.26.mlp.experts.155.down_proj", "model.layers.26.mlp.experts.156.down_proj", "model.layers.26.mlp.experts.157.down_proj", "model.layers.26.mlp.experts.158.down_proj", "model.layers.26.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002219223976135276, "dbits": 1258291200 } ] }, { "idx": 159, "layers": [ "model.layers.27.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0005519770085811726, "dbits": 62914560 } ] }, { "idx": 160, "layers": [ "model.layers.27.self_attn.k_proj", "model.layers.27.self_attn.v_proj" ], "candidates": [ { "dkld": 0.006316351890563954, "dbits": 10485760 } ] }, { "idx": 161, "layers": [ "model.layers.27.self_attn.o_proj" ], "candidates": [ { "dkld": 5.168467760052753e-06, "dbits": 62914560 } ] }, { "idx": 162, "layers": [ "model.layers.27.mlp.shared_experts.gate_proj", "model.layers.27.mlp.shared_experts.up_proj", "model.layers.27.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.003834831714630116, "dbits": 23592960 } ] }, { "idx": 163, "layers": [ "model.layers.27.mlp.experts.0.gate_proj", "model.layers.27.mlp.experts.1.gate_proj", "model.layers.27.mlp.experts.2.gate_proj", "model.layers.27.mlp.experts.3.gate_proj", "model.layers.27.mlp.experts.4.gate_proj", "model.layers.27.mlp.experts.5.gate_proj", "model.layers.27.mlp.experts.6.gate_proj", "model.layers.27.mlp.experts.7.gate_proj", "model.layers.27.mlp.experts.8.gate_proj", "model.layers.27.mlp.experts.9.gate_proj", "model.layers.27.mlp.experts.10.gate_proj", "model.layers.27.mlp.experts.11.gate_proj", "model.layers.27.mlp.experts.12.gate_proj", "model.layers.27.mlp.experts.13.gate_proj", "model.layers.27.mlp.experts.14.gate_proj", "model.layers.27.mlp.experts.15.gate_proj", "model.layers.27.mlp.experts.16.gate_proj", "model.layers.27.mlp.experts.17.gate_proj", "model.layers.27.mlp.experts.18.gate_proj", "model.layers.27.mlp.experts.19.gate_proj", "model.layers.27.mlp.experts.20.gate_proj", "model.layers.27.mlp.experts.21.gate_proj", "model.layers.27.mlp.experts.22.gate_proj", "model.layers.27.mlp.experts.23.gate_proj", "model.layers.27.mlp.experts.24.gate_proj", "model.layers.27.mlp.experts.25.gate_proj", "model.layers.27.mlp.experts.26.gate_proj", "model.layers.27.mlp.experts.27.gate_proj", "model.layers.27.mlp.experts.28.gate_proj", "model.layers.27.mlp.experts.29.gate_proj", "model.layers.27.mlp.experts.30.gate_proj", "model.layers.27.mlp.experts.31.gate_proj", "model.layers.27.mlp.experts.32.gate_proj", "model.layers.27.mlp.experts.33.gate_proj", "model.layers.27.mlp.experts.34.gate_proj", "model.layers.27.mlp.experts.35.gate_proj", "model.layers.27.mlp.experts.36.gate_proj", "model.layers.27.mlp.experts.37.gate_proj", "model.layers.27.mlp.experts.38.gate_proj", "model.layers.27.mlp.experts.39.gate_proj", "model.layers.27.mlp.experts.40.gate_proj", "model.layers.27.mlp.experts.41.gate_proj", "model.layers.27.mlp.experts.42.gate_proj", "model.layers.27.mlp.experts.43.gate_proj", "model.layers.27.mlp.experts.44.gate_proj", "model.layers.27.mlp.experts.45.gate_proj", "model.layers.27.mlp.experts.46.gate_proj", "model.layers.27.mlp.experts.47.gate_proj", "model.layers.27.mlp.experts.48.gate_proj", "model.layers.27.mlp.experts.49.gate_proj", "model.layers.27.mlp.experts.50.gate_proj", "model.layers.27.mlp.experts.51.gate_proj", "model.layers.27.mlp.experts.52.gate_proj", "model.layers.27.mlp.experts.53.gate_proj", "model.layers.27.mlp.experts.54.gate_proj", "model.layers.27.mlp.experts.55.gate_proj", "model.layers.27.mlp.experts.56.gate_proj", "model.layers.27.mlp.experts.57.gate_proj", "model.layers.27.mlp.experts.58.gate_proj", "model.layers.27.mlp.experts.59.gate_proj", "model.layers.27.mlp.experts.60.gate_proj", "model.layers.27.mlp.experts.61.gate_proj", "model.layers.27.mlp.experts.62.gate_proj", "model.layers.27.mlp.experts.63.gate_proj", "model.layers.27.mlp.experts.64.gate_proj", "model.layers.27.mlp.experts.65.gate_proj", "model.layers.27.mlp.experts.66.gate_proj", "model.layers.27.mlp.experts.67.gate_proj", "model.layers.27.mlp.experts.68.gate_proj", "model.layers.27.mlp.experts.69.gate_proj", "model.layers.27.mlp.experts.70.gate_proj", "model.layers.27.mlp.experts.71.gate_proj", "model.layers.27.mlp.experts.72.gate_proj", "model.layers.27.mlp.experts.73.gate_proj", "model.layers.27.mlp.experts.74.gate_proj", "model.layers.27.mlp.experts.75.gate_proj", "model.layers.27.mlp.experts.76.gate_proj", "model.layers.27.mlp.experts.77.gate_proj", "model.layers.27.mlp.experts.78.gate_proj", "model.layers.27.mlp.experts.79.gate_proj", "model.layers.27.mlp.experts.80.gate_proj", "model.layers.27.mlp.experts.81.gate_proj", "model.layers.27.mlp.experts.82.gate_proj", "model.layers.27.mlp.experts.83.gate_proj", "model.layers.27.mlp.experts.84.gate_proj", "model.layers.27.mlp.experts.85.gate_proj", "model.layers.27.mlp.experts.86.gate_proj", "model.layers.27.mlp.experts.87.gate_proj", "model.layers.27.mlp.experts.88.gate_proj", "model.layers.27.mlp.experts.89.gate_proj", "model.layers.27.mlp.experts.90.gate_proj", "model.layers.27.mlp.experts.91.gate_proj", "model.layers.27.mlp.experts.92.gate_proj", "model.layers.27.mlp.experts.93.gate_proj", "model.layers.27.mlp.experts.94.gate_proj", "model.layers.27.mlp.experts.95.gate_proj", "model.layers.27.mlp.experts.96.gate_proj", "model.layers.27.mlp.experts.97.gate_proj", "model.layers.27.mlp.experts.98.gate_proj", "model.layers.27.mlp.experts.99.gate_proj", "model.layers.27.mlp.experts.100.gate_proj", "model.layers.27.mlp.experts.101.gate_proj", "model.layers.27.mlp.experts.102.gate_proj", "model.layers.27.mlp.experts.103.gate_proj", "model.layers.27.mlp.experts.104.gate_proj", "model.layers.27.mlp.experts.105.gate_proj", "model.layers.27.mlp.experts.106.gate_proj", "model.layers.27.mlp.experts.107.gate_proj", "model.layers.27.mlp.experts.108.gate_proj", "model.layers.27.mlp.experts.109.gate_proj", "model.layers.27.mlp.experts.110.gate_proj", "model.layers.27.mlp.experts.111.gate_proj", "model.layers.27.mlp.experts.112.gate_proj", "model.layers.27.mlp.experts.113.gate_proj", "model.layers.27.mlp.experts.114.gate_proj", "model.layers.27.mlp.experts.115.gate_proj", "model.layers.27.mlp.experts.116.gate_proj", "model.layers.27.mlp.experts.117.gate_proj", "model.layers.27.mlp.experts.118.gate_proj", "model.layers.27.mlp.experts.119.gate_proj", "model.layers.27.mlp.experts.120.gate_proj", "model.layers.27.mlp.experts.121.gate_proj", "model.layers.27.mlp.experts.122.gate_proj", "model.layers.27.mlp.experts.123.gate_proj", "model.layers.27.mlp.experts.124.gate_proj", "model.layers.27.mlp.experts.125.gate_proj", "model.layers.27.mlp.experts.126.gate_proj", "model.layers.27.mlp.experts.127.gate_proj", "model.layers.27.mlp.experts.128.gate_proj", "model.layers.27.mlp.experts.129.gate_proj", "model.layers.27.mlp.experts.130.gate_proj", "model.layers.27.mlp.experts.131.gate_proj", "model.layers.27.mlp.experts.132.gate_proj", "model.layers.27.mlp.experts.133.gate_proj", "model.layers.27.mlp.experts.134.gate_proj", "model.layers.27.mlp.experts.135.gate_proj", "model.layers.27.mlp.experts.136.gate_proj", "model.layers.27.mlp.experts.137.gate_proj", "model.layers.27.mlp.experts.138.gate_proj", "model.layers.27.mlp.experts.139.gate_proj", "model.layers.27.mlp.experts.140.gate_proj", "model.layers.27.mlp.experts.141.gate_proj", "model.layers.27.mlp.experts.142.gate_proj", "model.layers.27.mlp.experts.143.gate_proj", "model.layers.27.mlp.experts.144.gate_proj", "model.layers.27.mlp.experts.145.gate_proj", "model.layers.27.mlp.experts.146.gate_proj", "model.layers.27.mlp.experts.147.gate_proj", "model.layers.27.mlp.experts.148.gate_proj", "model.layers.27.mlp.experts.149.gate_proj", "model.layers.27.mlp.experts.150.gate_proj", "model.layers.27.mlp.experts.151.gate_proj", "model.layers.27.mlp.experts.152.gate_proj", "model.layers.27.mlp.experts.153.gate_proj", "model.layers.27.mlp.experts.154.gate_proj", "model.layers.27.mlp.experts.155.gate_proj", "model.layers.27.mlp.experts.156.gate_proj", "model.layers.27.mlp.experts.157.gate_proj", "model.layers.27.mlp.experts.158.gate_proj", "model.layers.27.mlp.experts.159.gate_proj", "model.layers.27.mlp.experts.0.up_proj", "model.layers.27.mlp.experts.1.up_proj", "model.layers.27.mlp.experts.2.up_proj", "model.layers.27.mlp.experts.3.up_proj", "model.layers.27.mlp.experts.4.up_proj", "model.layers.27.mlp.experts.5.up_proj", "model.layers.27.mlp.experts.6.up_proj", "model.layers.27.mlp.experts.7.up_proj", "model.layers.27.mlp.experts.8.up_proj", "model.layers.27.mlp.experts.9.up_proj", "model.layers.27.mlp.experts.10.up_proj", "model.layers.27.mlp.experts.11.up_proj", "model.layers.27.mlp.experts.12.up_proj", "model.layers.27.mlp.experts.13.up_proj", "model.layers.27.mlp.experts.14.up_proj", "model.layers.27.mlp.experts.15.up_proj", "model.layers.27.mlp.experts.16.up_proj", "model.layers.27.mlp.experts.17.up_proj", "model.layers.27.mlp.experts.18.up_proj", "model.layers.27.mlp.experts.19.up_proj", "model.layers.27.mlp.experts.20.up_proj", "model.layers.27.mlp.experts.21.up_proj", "model.layers.27.mlp.experts.22.up_proj", "model.layers.27.mlp.experts.23.up_proj", "model.layers.27.mlp.experts.24.up_proj", "model.layers.27.mlp.experts.25.up_proj", "model.layers.27.mlp.experts.26.up_proj", "model.layers.27.mlp.experts.27.up_proj", "model.layers.27.mlp.experts.28.up_proj", "model.layers.27.mlp.experts.29.up_proj", "model.layers.27.mlp.experts.30.up_proj", "model.layers.27.mlp.experts.31.up_proj", "model.layers.27.mlp.experts.32.up_proj", "model.layers.27.mlp.experts.33.up_proj", "model.layers.27.mlp.experts.34.up_proj", "model.layers.27.mlp.experts.35.up_proj", "model.layers.27.mlp.experts.36.up_proj", "model.layers.27.mlp.experts.37.up_proj", "model.layers.27.mlp.experts.38.up_proj", "model.layers.27.mlp.experts.39.up_proj", "model.layers.27.mlp.experts.40.up_proj", "model.layers.27.mlp.experts.41.up_proj", "model.layers.27.mlp.experts.42.up_proj", "model.layers.27.mlp.experts.43.up_proj", "model.layers.27.mlp.experts.44.up_proj", "model.layers.27.mlp.experts.45.up_proj", "model.layers.27.mlp.experts.46.up_proj", "model.layers.27.mlp.experts.47.up_proj", "model.layers.27.mlp.experts.48.up_proj", "model.layers.27.mlp.experts.49.up_proj", "model.layers.27.mlp.experts.50.up_proj", "model.layers.27.mlp.experts.51.up_proj", "model.layers.27.mlp.experts.52.up_proj", "model.layers.27.mlp.experts.53.up_proj", "model.layers.27.mlp.experts.54.up_proj", "model.layers.27.mlp.experts.55.up_proj", "model.layers.27.mlp.experts.56.up_proj", "model.layers.27.mlp.experts.57.up_proj", "model.layers.27.mlp.experts.58.up_proj", "model.layers.27.mlp.experts.59.up_proj", "model.layers.27.mlp.experts.60.up_proj", "model.layers.27.mlp.experts.61.up_proj", "model.layers.27.mlp.experts.62.up_proj", "model.layers.27.mlp.experts.63.up_proj", "model.layers.27.mlp.experts.64.up_proj", "model.layers.27.mlp.experts.65.up_proj", "model.layers.27.mlp.experts.66.up_proj", "model.layers.27.mlp.experts.67.up_proj", "model.layers.27.mlp.experts.68.up_proj", "model.layers.27.mlp.experts.69.up_proj", "model.layers.27.mlp.experts.70.up_proj", "model.layers.27.mlp.experts.71.up_proj", "model.layers.27.mlp.experts.72.up_proj", "model.layers.27.mlp.experts.73.up_proj", "model.layers.27.mlp.experts.74.up_proj", "model.layers.27.mlp.experts.75.up_proj", "model.layers.27.mlp.experts.76.up_proj", "model.layers.27.mlp.experts.77.up_proj", "model.layers.27.mlp.experts.78.up_proj", "model.layers.27.mlp.experts.79.up_proj", "model.layers.27.mlp.experts.80.up_proj", "model.layers.27.mlp.experts.81.up_proj", "model.layers.27.mlp.experts.82.up_proj", "model.layers.27.mlp.experts.83.up_proj", "model.layers.27.mlp.experts.84.up_proj", "model.layers.27.mlp.experts.85.up_proj", "model.layers.27.mlp.experts.86.up_proj", "model.layers.27.mlp.experts.87.up_proj", "model.layers.27.mlp.experts.88.up_proj", "model.layers.27.mlp.experts.89.up_proj", "model.layers.27.mlp.experts.90.up_proj", "model.layers.27.mlp.experts.91.up_proj", "model.layers.27.mlp.experts.92.up_proj", "model.layers.27.mlp.experts.93.up_proj", "model.layers.27.mlp.experts.94.up_proj", "model.layers.27.mlp.experts.95.up_proj", "model.layers.27.mlp.experts.96.up_proj", "model.layers.27.mlp.experts.97.up_proj", "model.layers.27.mlp.experts.98.up_proj", "model.layers.27.mlp.experts.99.up_proj", "model.layers.27.mlp.experts.100.up_proj", "model.layers.27.mlp.experts.101.up_proj", "model.layers.27.mlp.experts.102.up_proj", "model.layers.27.mlp.experts.103.up_proj", "model.layers.27.mlp.experts.104.up_proj", "model.layers.27.mlp.experts.105.up_proj", "model.layers.27.mlp.experts.106.up_proj", "model.layers.27.mlp.experts.107.up_proj", "model.layers.27.mlp.experts.108.up_proj", "model.layers.27.mlp.experts.109.up_proj", "model.layers.27.mlp.experts.110.up_proj", "model.layers.27.mlp.experts.111.up_proj", "model.layers.27.mlp.experts.112.up_proj", "model.layers.27.mlp.experts.113.up_proj", "model.layers.27.mlp.experts.114.up_proj", "model.layers.27.mlp.experts.115.up_proj", "model.layers.27.mlp.experts.116.up_proj", "model.layers.27.mlp.experts.117.up_proj", "model.layers.27.mlp.experts.118.up_proj", "model.layers.27.mlp.experts.119.up_proj", "model.layers.27.mlp.experts.120.up_proj", "model.layers.27.mlp.experts.121.up_proj", "model.layers.27.mlp.experts.122.up_proj", "model.layers.27.mlp.experts.123.up_proj", "model.layers.27.mlp.experts.124.up_proj", "model.layers.27.mlp.experts.125.up_proj", "model.layers.27.mlp.experts.126.up_proj", "model.layers.27.mlp.experts.127.up_proj", "model.layers.27.mlp.experts.128.up_proj", "model.layers.27.mlp.experts.129.up_proj", "model.layers.27.mlp.experts.130.up_proj", "model.layers.27.mlp.experts.131.up_proj", "model.layers.27.mlp.experts.132.up_proj", "model.layers.27.mlp.experts.133.up_proj", "model.layers.27.mlp.experts.134.up_proj", "model.layers.27.mlp.experts.135.up_proj", "model.layers.27.mlp.experts.136.up_proj", "model.layers.27.mlp.experts.137.up_proj", "model.layers.27.mlp.experts.138.up_proj", "model.layers.27.mlp.experts.139.up_proj", "model.layers.27.mlp.experts.140.up_proj", "model.layers.27.mlp.experts.141.up_proj", "model.layers.27.mlp.experts.142.up_proj", "model.layers.27.mlp.experts.143.up_proj", "model.layers.27.mlp.experts.144.up_proj", "model.layers.27.mlp.experts.145.up_proj", "model.layers.27.mlp.experts.146.up_proj", "model.layers.27.mlp.experts.147.up_proj", "model.layers.27.mlp.experts.148.up_proj", "model.layers.27.mlp.experts.149.up_proj", "model.layers.27.mlp.experts.150.up_proj", "model.layers.27.mlp.experts.151.up_proj", "model.layers.27.mlp.experts.152.up_proj", "model.layers.27.mlp.experts.153.up_proj", "model.layers.27.mlp.experts.154.up_proj", "model.layers.27.mlp.experts.155.up_proj", "model.layers.27.mlp.experts.156.up_proj", "model.layers.27.mlp.experts.157.up_proj", "model.layers.27.mlp.experts.158.up_proj", "model.layers.27.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.002048788964748416, "dbits": 2516582400 } ] }, { "idx": 164, "layers": [ "model.layers.27.mlp.experts.0.down_proj", "model.layers.27.mlp.experts.1.down_proj", "model.layers.27.mlp.experts.2.down_proj", "model.layers.27.mlp.experts.3.down_proj", "model.layers.27.mlp.experts.4.down_proj", "model.layers.27.mlp.experts.5.down_proj", "model.layers.27.mlp.experts.6.down_proj", "model.layers.27.mlp.experts.7.down_proj", "model.layers.27.mlp.experts.8.down_proj", "model.layers.27.mlp.experts.9.down_proj", "model.layers.27.mlp.experts.10.down_proj", "model.layers.27.mlp.experts.11.down_proj", "model.layers.27.mlp.experts.12.down_proj", "model.layers.27.mlp.experts.13.down_proj", "model.layers.27.mlp.experts.14.down_proj", "model.layers.27.mlp.experts.15.down_proj", "model.layers.27.mlp.experts.16.down_proj", "model.layers.27.mlp.experts.17.down_proj", "model.layers.27.mlp.experts.18.down_proj", "model.layers.27.mlp.experts.19.down_proj", "model.layers.27.mlp.experts.20.down_proj", "model.layers.27.mlp.experts.21.down_proj", "model.layers.27.mlp.experts.22.down_proj", "model.layers.27.mlp.experts.23.down_proj", "model.layers.27.mlp.experts.24.down_proj", "model.layers.27.mlp.experts.25.down_proj", "model.layers.27.mlp.experts.26.down_proj", "model.layers.27.mlp.experts.27.down_proj", "model.layers.27.mlp.experts.28.down_proj", "model.layers.27.mlp.experts.29.down_proj", "model.layers.27.mlp.experts.30.down_proj", "model.layers.27.mlp.experts.31.down_proj", "model.layers.27.mlp.experts.32.down_proj", "model.layers.27.mlp.experts.33.down_proj", "model.layers.27.mlp.experts.34.down_proj", "model.layers.27.mlp.experts.35.down_proj", "model.layers.27.mlp.experts.36.down_proj", "model.layers.27.mlp.experts.37.down_proj", "model.layers.27.mlp.experts.38.down_proj", "model.layers.27.mlp.experts.39.down_proj", "model.layers.27.mlp.experts.40.down_proj", "model.layers.27.mlp.experts.41.down_proj", "model.layers.27.mlp.experts.42.down_proj", "model.layers.27.mlp.experts.43.down_proj", "model.layers.27.mlp.experts.44.down_proj", "model.layers.27.mlp.experts.45.down_proj", "model.layers.27.mlp.experts.46.down_proj", "model.layers.27.mlp.experts.47.down_proj", "model.layers.27.mlp.experts.48.down_proj", "model.layers.27.mlp.experts.49.down_proj", "model.layers.27.mlp.experts.50.down_proj", "model.layers.27.mlp.experts.51.down_proj", "model.layers.27.mlp.experts.52.down_proj", "model.layers.27.mlp.experts.53.down_proj", "model.layers.27.mlp.experts.54.down_proj", "model.layers.27.mlp.experts.55.down_proj", "model.layers.27.mlp.experts.56.down_proj", "model.layers.27.mlp.experts.57.down_proj", "model.layers.27.mlp.experts.58.down_proj", "model.layers.27.mlp.experts.59.down_proj", "model.layers.27.mlp.experts.60.down_proj", "model.layers.27.mlp.experts.61.down_proj", "model.layers.27.mlp.experts.62.down_proj", "model.layers.27.mlp.experts.63.down_proj", "model.layers.27.mlp.experts.64.down_proj", "model.layers.27.mlp.experts.65.down_proj", "model.layers.27.mlp.experts.66.down_proj", "model.layers.27.mlp.experts.67.down_proj", "model.layers.27.mlp.experts.68.down_proj", "model.layers.27.mlp.experts.69.down_proj", "model.layers.27.mlp.experts.70.down_proj", "model.layers.27.mlp.experts.71.down_proj", "model.layers.27.mlp.experts.72.down_proj", "model.layers.27.mlp.experts.73.down_proj", "model.layers.27.mlp.experts.74.down_proj", "model.layers.27.mlp.experts.75.down_proj", "model.layers.27.mlp.experts.76.down_proj", "model.layers.27.mlp.experts.77.down_proj", "model.layers.27.mlp.experts.78.down_proj", "model.layers.27.mlp.experts.79.down_proj", "model.layers.27.mlp.experts.80.down_proj", "model.layers.27.mlp.experts.81.down_proj", "model.layers.27.mlp.experts.82.down_proj", "model.layers.27.mlp.experts.83.down_proj", "model.layers.27.mlp.experts.84.down_proj", "model.layers.27.mlp.experts.85.down_proj", "model.layers.27.mlp.experts.86.down_proj", "model.layers.27.mlp.experts.87.down_proj", "model.layers.27.mlp.experts.88.down_proj", "model.layers.27.mlp.experts.89.down_proj", "model.layers.27.mlp.experts.90.down_proj", "model.layers.27.mlp.experts.91.down_proj", "model.layers.27.mlp.experts.92.down_proj", "model.layers.27.mlp.experts.93.down_proj", "model.layers.27.mlp.experts.94.down_proj", "model.layers.27.mlp.experts.95.down_proj", "model.layers.27.mlp.experts.96.down_proj", "model.layers.27.mlp.experts.97.down_proj", "model.layers.27.mlp.experts.98.down_proj", "model.layers.27.mlp.experts.99.down_proj", "model.layers.27.mlp.experts.100.down_proj", "model.layers.27.mlp.experts.101.down_proj", "model.layers.27.mlp.experts.102.down_proj", "model.layers.27.mlp.experts.103.down_proj", "model.layers.27.mlp.experts.104.down_proj", "model.layers.27.mlp.experts.105.down_proj", "model.layers.27.mlp.experts.106.down_proj", "model.layers.27.mlp.experts.107.down_proj", "model.layers.27.mlp.experts.108.down_proj", "model.layers.27.mlp.experts.109.down_proj", "model.layers.27.mlp.experts.110.down_proj", "model.layers.27.mlp.experts.111.down_proj", "model.layers.27.mlp.experts.112.down_proj", "model.layers.27.mlp.experts.113.down_proj", "model.layers.27.mlp.experts.114.down_proj", "model.layers.27.mlp.experts.115.down_proj", "model.layers.27.mlp.experts.116.down_proj", "model.layers.27.mlp.experts.117.down_proj", "model.layers.27.mlp.experts.118.down_proj", "model.layers.27.mlp.experts.119.down_proj", "model.layers.27.mlp.experts.120.down_proj", "model.layers.27.mlp.experts.121.down_proj", "model.layers.27.mlp.experts.122.down_proj", "model.layers.27.mlp.experts.123.down_proj", "model.layers.27.mlp.experts.124.down_proj", "model.layers.27.mlp.experts.125.down_proj", "model.layers.27.mlp.experts.126.down_proj", "model.layers.27.mlp.experts.127.down_proj", "model.layers.27.mlp.experts.128.down_proj", "model.layers.27.mlp.experts.129.down_proj", "model.layers.27.mlp.experts.130.down_proj", "model.layers.27.mlp.experts.131.down_proj", "model.layers.27.mlp.experts.132.down_proj", "model.layers.27.mlp.experts.133.down_proj", "model.layers.27.mlp.experts.134.down_proj", "model.layers.27.mlp.experts.135.down_proj", "model.layers.27.mlp.experts.136.down_proj", "model.layers.27.mlp.experts.137.down_proj", "model.layers.27.mlp.experts.138.down_proj", "model.layers.27.mlp.experts.139.down_proj", "model.layers.27.mlp.experts.140.down_proj", "model.layers.27.mlp.experts.141.down_proj", "model.layers.27.mlp.experts.142.down_proj", "model.layers.27.mlp.experts.143.down_proj", "model.layers.27.mlp.experts.144.down_proj", "model.layers.27.mlp.experts.145.down_proj", "model.layers.27.mlp.experts.146.down_proj", "model.layers.27.mlp.experts.147.down_proj", "model.layers.27.mlp.experts.148.down_proj", "model.layers.27.mlp.experts.149.down_proj", "model.layers.27.mlp.experts.150.down_proj", "model.layers.27.mlp.experts.151.down_proj", "model.layers.27.mlp.experts.152.down_proj", "model.layers.27.mlp.experts.153.down_proj", "model.layers.27.mlp.experts.154.down_proj", "model.layers.27.mlp.experts.155.down_proj", "model.layers.27.mlp.experts.156.down_proj", "model.layers.27.mlp.experts.157.down_proj", "model.layers.27.mlp.experts.158.down_proj", "model.layers.27.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002293948829174053, "dbits": 1258291200 } ] }, { "idx": 165, "layers": [ "model.layers.28.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0016066402196884266, "dbits": 62914560 } ] }, { "idx": 166, "layers": [ "model.layers.28.self_attn.k_proj", "model.layers.28.self_attn.v_proj" ], "candidates": [ { "dkld": 0.004597499221563328, "dbits": 10485760 } ] }, { "idx": 167, "layers": [ "model.layers.28.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0005196593701839447, "dbits": 62914560 } ] }, { "idx": 168, "layers": [ "model.layers.28.mlp.shared_experts.gate_proj", "model.layers.28.mlp.shared_experts.up_proj", "model.layers.28.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.007889974862337112, "dbits": 23592960 } ] }, { "idx": 169, "layers": [ "model.layers.28.mlp.experts.0.gate_proj", "model.layers.28.mlp.experts.1.gate_proj", "model.layers.28.mlp.experts.2.gate_proj", "model.layers.28.mlp.experts.3.gate_proj", "model.layers.28.mlp.experts.4.gate_proj", "model.layers.28.mlp.experts.5.gate_proj", "model.layers.28.mlp.experts.6.gate_proj", "model.layers.28.mlp.experts.7.gate_proj", "model.layers.28.mlp.experts.8.gate_proj", "model.layers.28.mlp.experts.9.gate_proj", "model.layers.28.mlp.experts.10.gate_proj", "model.layers.28.mlp.experts.11.gate_proj", "model.layers.28.mlp.experts.12.gate_proj", "model.layers.28.mlp.experts.13.gate_proj", "model.layers.28.mlp.experts.14.gate_proj", "model.layers.28.mlp.experts.15.gate_proj", "model.layers.28.mlp.experts.16.gate_proj", "model.layers.28.mlp.experts.17.gate_proj", "model.layers.28.mlp.experts.18.gate_proj", "model.layers.28.mlp.experts.19.gate_proj", "model.layers.28.mlp.experts.20.gate_proj", "model.layers.28.mlp.experts.21.gate_proj", "model.layers.28.mlp.experts.22.gate_proj", "model.layers.28.mlp.experts.23.gate_proj", "model.layers.28.mlp.experts.24.gate_proj", "model.layers.28.mlp.experts.25.gate_proj", "model.layers.28.mlp.experts.26.gate_proj", "model.layers.28.mlp.experts.27.gate_proj", "model.layers.28.mlp.experts.28.gate_proj", "model.layers.28.mlp.experts.29.gate_proj", "model.layers.28.mlp.experts.30.gate_proj", "model.layers.28.mlp.experts.31.gate_proj", "model.layers.28.mlp.experts.32.gate_proj", "model.layers.28.mlp.experts.33.gate_proj", "model.layers.28.mlp.experts.34.gate_proj", "model.layers.28.mlp.experts.35.gate_proj", "model.layers.28.mlp.experts.36.gate_proj", "model.layers.28.mlp.experts.37.gate_proj", "model.layers.28.mlp.experts.38.gate_proj", "model.layers.28.mlp.experts.39.gate_proj", "model.layers.28.mlp.experts.40.gate_proj", "model.layers.28.mlp.experts.41.gate_proj", "model.layers.28.mlp.experts.42.gate_proj", "model.layers.28.mlp.experts.43.gate_proj", "model.layers.28.mlp.experts.44.gate_proj", "model.layers.28.mlp.experts.45.gate_proj", "model.layers.28.mlp.experts.46.gate_proj", "model.layers.28.mlp.experts.47.gate_proj", "model.layers.28.mlp.experts.48.gate_proj", "model.layers.28.mlp.experts.49.gate_proj", "model.layers.28.mlp.experts.50.gate_proj", "model.layers.28.mlp.experts.51.gate_proj", "model.layers.28.mlp.experts.52.gate_proj", "model.layers.28.mlp.experts.53.gate_proj", "model.layers.28.mlp.experts.54.gate_proj", "model.layers.28.mlp.experts.55.gate_proj", "model.layers.28.mlp.experts.56.gate_proj", "model.layers.28.mlp.experts.57.gate_proj", "model.layers.28.mlp.experts.58.gate_proj", "model.layers.28.mlp.experts.59.gate_proj", "model.layers.28.mlp.experts.60.gate_proj", "model.layers.28.mlp.experts.61.gate_proj", "model.layers.28.mlp.experts.62.gate_proj", "model.layers.28.mlp.experts.63.gate_proj", "model.layers.28.mlp.experts.64.gate_proj", "model.layers.28.mlp.experts.65.gate_proj", "model.layers.28.mlp.experts.66.gate_proj", "model.layers.28.mlp.experts.67.gate_proj", "model.layers.28.mlp.experts.68.gate_proj", "model.layers.28.mlp.experts.69.gate_proj", "model.layers.28.mlp.experts.70.gate_proj", "model.layers.28.mlp.experts.71.gate_proj", "model.layers.28.mlp.experts.72.gate_proj", "model.layers.28.mlp.experts.73.gate_proj", "model.layers.28.mlp.experts.74.gate_proj", "model.layers.28.mlp.experts.75.gate_proj", "model.layers.28.mlp.experts.76.gate_proj", "model.layers.28.mlp.experts.77.gate_proj", "model.layers.28.mlp.experts.78.gate_proj", "model.layers.28.mlp.experts.79.gate_proj", "model.layers.28.mlp.experts.80.gate_proj", "model.layers.28.mlp.experts.81.gate_proj", "model.layers.28.mlp.experts.82.gate_proj", "model.layers.28.mlp.experts.83.gate_proj", "model.layers.28.mlp.experts.84.gate_proj", "model.layers.28.mlp.experts.85.gate_proj", "model.layers.28.mlp.experts.86.gate_proj", "model.layers.28.mlp.experts.87.gate_proj", "model.layers.28.mlp.experts.88.gate_proj", "model.layers.28.mlp.experts.89.gate_proj", "model.layers.28.mlp.experts.90.gate_proj", "model.layers.28.mlp.experts.91.gate_proj", "model.layers.28.mlp.experts.92.gate_proj", "model.layers.28.mlp.experts.93.gate_proj", "model.layers.28.mlp.experts.94.gate_proj", "model.layers.28.mlp.experts.95.gate_proj", "model.layers.28.mlp.experts.96.gate_proj", "model.layers.28.mlp.experts.97.gate_proj", "model.layers.28.mlp.experts.98.gate_proj", "model.layers.28.mlp.experts.99.gate_proj", "model.layers.28.mlp.experts.100.gate_proj", "model.layers.28.mlp.experts.101.gate_proj", "model.layers.28.mlp.experts.102.gate_proj", "model.layers.28.mlp.experts.103.gate_proj", "model.layers.28.mlp.experts.104.gate_proj", "model.layers.28.mlp.experts.105.gate_proj", "model.layers.28.mlp.experts.106.gate_proj", "model.layers.28.mlp.experts.107.gate_proj", "model.layers.28.mlp.experts.108.gate_proj", "model.layers.28.mlp.experts.109.gate_proj", "model.layers.28.mlp.experts.110.gate_proj", "model.layers.28.mlp.experts.111.gate_proj", "model.layers.28.mlp.experts.112.gate_proj", "model.layers.28.mlp.experts.113.gate_proj", "model.layers.28.mlp.experts.114.gate_proj", "model.layers.28.mlp.experts.115.gate_proj", "model.layers.28.mlp.experts.116.gate_proj", "model.layers.28.mlp.experts.117.gate_proj", "model.layers.28.mlp.experts.118.gate_proj", "model.layers.28.mlp.experts.119.gate_proj", "model.layers.28.mlp.experts.120.gate_proj", "model.layers.28.mlp.experts.121.gate_proj", "model.layers.28.mlp.experts.122.gate_proj", "model.layers.28.mlp.experts.123.gate_proj", "model.layers.28.mlp.experts.124.gate_proj", "model.layers.28.mlp.experts.125.gate_proj", "model.layers.28.mlp.experts.126.gate_proj", "model.layers.28.mlp.experts.127.gate_proj", "model.layers.28.mlp.experts.128.gate_proj", "model.layers.28.mlp.experts.129.gate_proj", "model.layers.28.mlp.experts.130.gate_proj", "model.layers.28.mlp.experts.131.gate_proj", "model.layers.28.mlp.experts.132.gate_proj", "model.layers.28.mlp.experts.133.gate_proj", "model.layers.28.mlp.experts.134.gate_proj", "model.layers.28.mlp.experts.135.gate_proj", "model.layers.28.mlp.experts.136.gate_proj", "model.layers.28.mlp.experts.137.gate_proj", "model.layers.28.mlp.experts.138.gate_proj", "model.layers.28.mlp.experts.139.gate_proj", "model.layers.28.mlp.experts.140.gate_proj", "model.layers.28.mlp.experts.141.gate_proj", "model.layers.28.mlp.experts.142.gate_proj", "model.layers.28.mlp.experts.143.gate_proj", "model.layers.28.mlp.experts.144.gate_proj", "model.layers.28.mlp.experts.145.gate_proj", "model.layers.28.mlp.experts.146.gate_proj", "model.layers.28.mlp.experts.147.gate_proj", "model.layers.28.mlp.experts.148.gate_proj", "model.layers.28.mlp.experts.149.gate_proj", "model.layers.28.mlp.experts.150.gate_proj", "model.layers.28.mlp.experts.151.gate_proj", "model.layers.28.mlp.experts.152.gate_proj", "model.layers.28.mlp.experts.153.gate_proj", "model.layers.28.mlp.experts.154.gate_proj", "model.layers.28.mlp.experts.155.gate_proj", "model.layers.28.mlp.experts.156.gate_proj", "model.layers.28.mlp.experts.157.gate_proj", "model.layers.28.mlp.experts.158.gate_proj", "model.layers.28.mlp.experts.159.gate_proj", "model.layers.28.mlp.experts.0.up_proj", "model.layers.28.mlp.experts.1.up_proj", "model.layers.28.mlp.experts.2.up_proj", "model.layers.28.mlp.experts.3.up_proj", "model.layers.28.mlp.experts.4.up_proj", "model.layers.28.mlp.experts.5.up_proj", "model.layers.28.mlp.experts.6.up_proj", "model.layers.28.mlp.experts.7.up_proj", "model.layers.28.mlp.experts.8.up_proj", "model.layers.28.mlp.experts.9.up_proj", "model.layers.28.mlp.experts.10.up_proj", "model.layers.28.mlp.experts.11.up_proj", "model.layers.28.mlp.experts.12.up_proj", "model.layers.28.mlp.experts.13.up_proj", "model.layers.28.mlp.experts.14.up_proj", "model.layers.28.mlp.experts.15.up_proj", "model.layers.28.mlp.experts.16.up_proj", "model.layers.28.mlp.experts.17.up_proj", "model.layers.28.mlp.experts.18.up_proj", "model.layers.28.mlp.experts.19.up_proj", "model.layers.28.mlp.experts.20.up_proj", "model.layers.28.mlp.experts.21.up_proj", "model.layers.28.mlp.experts.22.up_proj", "model.layers.28.mlp.experts.23.up_proj", "model.layers.28.mlp.experts.24.up_proj", "model.layers.28.mlp.experts.25.up_proj", "model.layers.28.mlp.experts.26.up_proj", "model.layers.28.mlp.experts.27.up_proj", "model.layers.28.mlp.experts.28.up_proj", "model.layers.28.mlp.experts.29.up_proj", "model.layers.28.mlp.experts.30.up_proj", "model.layers.28.mlp.experts.31.up_proj", "model.layers.28.mlp.experts.32.up_proj", "model.layers.28.mlp.experts.33.up_proj", "model.layers.28.mlp.experts.34.up_proj", "model.layers.28.mlp.experts.35.up_proj", "model.layers.28.mlp.experts.36.up_proj", "model.layers.28.mlp.experts.37.up_proj", "model.layers.28.mlp.experts.38.up_proj", "model.layers.28.mlp.experts.39.up_proj", "model.layers.28.mlp.experts.40.up_proj", "model.layers.28.mlp.experts.41.up_proj", "model.layers.28.mlp.experts.42.up_proj", "model.layers.28.mlp.experts.43.up_proj", "model.layers.28.mlp.experts.44.up_proj", "model.layers.28.mlp.experts.45.up_proj", "model.layers.28.mlp.experts.46.up_proj", "model.layers.28.mlp.experts.47.up_proj", "model.layers.28.mlp.experts.48.up_proj", "model.layers.28.mlp.experts.49.up_proj", "model.layers.28.mlp.experts.50.up_proj", "model.layers.28.mlp.experts.51.up_proj", "model.layers.28.mlp.experts.52.up_proj", "model.layers.28.mlp.experts.53.up_proj", "model.layers.28.mlp.experts.54.up_proj", "model.layers.28.mlp.experts.55.up_proj", "model.layers.28.mlp.experts.56.up_proj", "model.layers.28.mlp.experts.57.up_proj", "model.layers.28.mlp.experts.58.up_proj", "model.layers.28.mlp.experts.59.up_proj", "model.layers.28.mlp.experts.60.up_proj", "model.layers.28.mlp.experts.61.up_proj", "model.layers.28.mlp.experts.62.up_proj", "model.layers.28.mlp.experts.63.up_proj", "model.layers.28.mlp.experts.64.up_proj", "model.layers.28.mlp.experts.65.up_proj", "model.layers.28.mlp.experts.66.up_proj", "model.layers.28.mlp.experts.67.up_proj", "model.layers.28.mlp.experts.68.up_proj", "model.layers.28.mlp.experts.69.up_proj", "model.layers.28.mlp.experts.70.up_proj", "model.layers.28.mlp.experts.71.up_proj", "model.layers.28.mlp.experts.72.up_proj", "model.layers.28.mlp.experts.73.up_proj", "model.layers.28.mlp.experts.74.up_proj", "model.layers.28.mlp.experts.75.up_proj", "model.layers.28.mlp.experts.76.up_proj", "model.layers.28.mlp.experts.77.up_proj", "model.layers.28.mlp.experts.78.up_proj", "model.layers.28.mlp.experts.79.up_proj", "model.layers.28.mlp.experts.80.up_proj", "model.layers.28.mlp.experts.81.up_proj", "model.layers.28.mlp.experts.82.up_proj", "model.layers.28.mlp.experts.83.up_proj", "model.layers.28.mlp.experts.84.up_proj", "model.layers.28.mlp.experts.85.up_proj", "model.layers.28.mlp.experts.86.up_proj", "model.layers.28.mlp.experts.87.up_proj", "model.layers.28.mlp.experts.88.up_proj", "model.layers.28.mlp.experts.89.up_proj", "model.layers.28.mlp.experts.90.up_proj", "model.layers.28.mlp.experts.91.up_proj", "model.layers.28.mlp.experts.92.up_proj", "model.layers.28.mlp.experts.93.up_proj", "model.layers.28.mlp.experts.94.up_proj", "model.layers.28.mlp.experts.95.up_proj", "model.layers.28.mlp.experts.96.up_proj", "model.layers.28.mlp.experts.97.up_proj", "model.layers.28.mlp.experts.98.up_proj", "model.layers.28.mlp.experts.99.up_proj", "model.layers.28.mlp.experts.100.up_proj", "model.layers.28.mlp.experts.101.up_proj", "model.layers.28.mlp.experts.102.up_proj", "model.layers.28.mlp.experts.103.up_proj", "model.layers.28.mlp.experts.104.up_proj", "model.layers.28.mlp.experts.105.up_proj", "model.layers.28.mlp.experts.106.up_proj", "model.layers.28.mlp.experts.107.up_proj", "model.layers.28.mlp.experts.108.up_proj", "model.layers.28.mlp.experts.109.up_proj", "model.layers.28.mlp.experts.110.up_proj", "model.layers.28.mlp.experts.111.up_proj", "model.layers.28.mlp.experts.112.up_proj", "model.layers.28.mlp.experts.113.up_proj", "model.layers.28.mlp.experts.114.up_proj", "model.layers.28.mlp.experts.115.up_proj", "model.layers.28.mlp.experts.116.up_proj", "model.layers.28.mlp.experts.117.up_proj", "model.layers.28.mlp.experts.118.up_proj", "model.layers.28.mlp.experts.119.up_proj", "model.layers.28.mlp.experts.120.up_proj", "model.layers.28.mlp.experts.121.up_proj", "model.layers.28.mlp.experts.122.up_proj", "model.layers.28.mlp.experts.123.up_proj", "model.layers.28.mlp.experts.124.up_proj", "model.layers.28.mlp.experts.125.up_proj", "model.layers.28.mlp.experts.126.up_proj", "model.layers.28.mlp.experts.127.up_proj", "model.layers.28.mlp.experts.128.up_proj", "model.layers.28.mlp.experts.129.up_proj", "model.layers.28.mlp.experts.130.up_proj", "model.layers.28.mlp.experts.131.up_proj", "model.layers.28.mlp.experts.132.up_proj", "model.layers.28.mlp.experts.133.up_proj", "model.layers.28.mlp.experts.134.up_proj", "model.layers.28.mlp.experts.135.up_proj", "model.layers.28.mlp.experts.136.up_proj", "model.layers.28.mlp.experts.137.up_proj", "model.layers.28.mlp.experts.138.up_proj", "model.layers.28.mlp.experts.139.up_proj", "model.layers.28.mlp.experts.140.up_proj", "model.layers.28.mlp.experts.141.up_proj", "model.layers.28.mlp.experts.142.up_proj", "model.layers.28.mlp.experts.143.up_proj", "model.layers.28.mlp.experts.144.up_proj", "model.layers.28.mlp.experts.145.up_proj", "model.layers.28.mlp.experts.146.up_proj", "model.layers.28.mlp.experts.147.up_proj", "model.layers.28.mlp.experts.148.up_proj", "model.layers.28.mlp.experts.149.up_proj", "model.layers.28.mlp.experts.150.up_proj", "model.layers.28.mlp.experts.151.up_proj", "model.layers.28.mlp.experts.152.up_proj", "model.layers.28.mlp.experts.153.up_proj", "model.layers.28.mlp.experts.154.up_proj", "model.layers.28.mlp.experts.155.up_proj", "model.layers.28.mlp.experts.156.up_proj", "model.layers.28.mlp.experts.157.up_proj", "model.layers.28.mlp.experts.158.up_proj", "model.layers.28.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0029201097786426544, "dbits": 2516582400 } ] }, { "idx": 170, "layers": [ "model.layers.28.mlp.experts.0.down_proj", "model.layers.28.mlp.experts.1.down_proj", "model.layers.28.mlp.experts.2.down_proj", "model.layers.28.mlp.experts.3.down_proj", "model.layers.28.mlp.experts.4.down_proj", "model.layers.28.mlp.experts.5.down_proj", "model.layers.28.mlp.experts.6.down_proj", "model.layers.28.mlp.experts.7.down_proj", "model.layers.28.mlp.experts.8.down_proj", "model.layers.28.mlp.experts.9.down_proj", "model.layers.28.mlp.experts.10.down_proj", "model.layers.28.mlp.experts.11.down_proj", "model.layers.28.mlp.experts.12.down_proj", "model.layers.28.mlp.experts.13.down_proj", "model.layers.28.mlp.experts.14.down_proj", "model.layers.28.mlp.experts.15.down_proj", "model.layers.28.mlp.experts.16.down_proj", "model.layers.28.mlp.experts.17.down_proj", "model.layers.28.mlp.experts.18.down_proj", "model.layers.28.mlp.experts.19.down_proj", "model.layers.28.mlp.experts.20.down_proj", "model.layers.28.mlp.experts.21.down_proj", "model.layers.28.mlp.experts.22.down_proj", "model.layers.28.mlp.experts.23.down_proj", "model.layers.28.mlp.experts.24.down_proj", "model.layers.28.mlp.experts.25.down_proj", "model.layers.28.mlp.experts.26.down_proj", "model.layers.28.mlp.experts.27.down_proj", "model.layers.28.mlp.experts.28.down_proj", "model.layers.28.mlp.experts.29.down_proj", "model.layers.28.mlp.experts.30.down_proj", "model.layers.28.mlp.experts.31.down_proj", "model.layers.28.mlp.experts.32.down_proj", "model.layers.28.mlp.experts.33.down_proj", "model.layers.28.mlp.experts.34.down_proj", "model.layers.28.mlp.experts.35.down_proj", "model.layers.28.mlp.experts.36.down_proj", "model.layers.28.mlp.experts.37.down_proj", "model.layers.28.mlp.experts.38.down_proj", "model.layers.28.mlp.experts.39.down_proj", "model.layers.28.mlp.experts.40.down_proj", "model.layers.28.mlp.experts.41.down_proj", "model.layers.28.mlp.experts.42.down_proj", "model.layers.28.mlp.experts.43.down_proj", "model.layers.28.mlp.experts.44.down_proj", "model.layers.28.mlp.experts.45.down_proj", "model.layers.28.mlp.experts.46.down_proj", "model.layers.28.mlp.experts.47.down_proj", "model.layers.28.mlp.experts.48.down_proj", "model.layers.28.mlp.experts.49.down_proj", "model.layers.28.mlp.experts.50.down_proj", "model.layers.28.mlp.experts.51.down_proj", "model.layers.28.mlp.experts.52.down_proj", "model.layers.28.mlp.experts.53.down_proj", "model.layers.28.mlp.experts.54.down_proj", "model.layers.28.mlp.experts.55.down_proj", "model.layers.28.mlp.experts.56.down_proj", "model.layers.28.mlp.experts.57.down_proj", "model.layers.28.mlp.experts.58.down_proj", "model.layers.28.mlp.experts.59.down_proj", "model.layers.28.mlp.experts.60.down_proj", "model.layers.28.mlp.experts.61.down_proj", "model.layers.28.mlp.experts.62.down_proj", "model.layers.28.mlp.experts.63.down_proj", "model.layers.28.mlp.experts.64.down_proj", "model.layers.28.mlp.experts.65.down_proj", "model.layers.28.mlp.experts.66.down_proj", "model.layers.28.mlp.experts.67.down_proj", "model.layers.28.mlp.experts.68.down_proj", "model.layers.28.mlp.experts.69.down_proj", "model.layers.28.mlp.experts.70.down_proj", "model.layers.28.mlp.experts.71.down_proj", "model.layers.28.mlp.experts.72.down_proj", "model.layers.28.mlp.experts.73.down_proj", "model.layers.28.mlp.experts.74.down_proj", "model.layers.28.mlp.experts.75.down_proj", "model.layers.28.mlp.experts.76.down_proj", "model.layers.28.mlp.experts.77.down_proj", "model.layers.28.mlp.experts.78.down_proj", "model.layers.28.mlp.experts.79.down_proj", "model.layers.28.mlp.experts.80.down_proj", "model.layers.28.mlp.experts.81.down_proj", "model.layers.28.mlp.experts.82.down_proj", "model.layers.28.mlp.experts.83.down_proj", "model.layers.28.mlp.experts.84.down_proj", "model.layers.28.mlp.experts.85.down_proj", "model.layers.28.mlp.experts.86.down_proj", "model.layers.28.mlp.experts.87.down_proj", "model.layers.28.mlp.experts.88.down_proj", "model.layers.28.mlp.experts.89.down_proj", "model.layers.28.mlp.experts.90.down_proj", "model.layers.28.mlp.experts.91.down_proj", "model.layers.28.mlp.experts.92.down_proj", "model.layers.28.mlp.experts.93.down_proj", "model.layers.28.mlp.experts.94.down_proj", "model.layers.28.mlp.experts.95.down_proj", "model.layers.28.mlp.experts.96.down_proj", "model.layers.28.mlp.experts.97.down_proj", "model.layers.28.mlp.experts.98.down_proj", "model.layers.28.mlp.experts.99.down_proj", "model.layers.28.mlp.experts.100.down_proj", "model.layers.28.mlp.experts.101.down_proj", "model.layers.28.mlp.experts.102.down_proj", "model.layers.28.mlp.experts.103.down_proj", "model.layers.28.mlp.experts.104.down_proj", "model.layers.28.mlp.experts.105.down_proj", "model.layers.28.mlp.experts.106.down_proj", "model.layers.28.mlp.experts.107.down_proj", "model.layers.28.mlp.experts.108.down_proj", "model.layers.28.mlp.experts.109.down_proj", "model.layers.28.mlp.experts.110.down_proj", "model.layers.28.mlp.experts.111.down_proj", "model.layers.28.mlp.experts.112.down_proj", "model.layers.28.mlp.experts.113.down_proj", "model.layers.28.mlp.experts.114.down_proj", "model.layers.28.mlp.experts.115.down_proj", "model.layers.28.mlp.experts.116.down_proj", "model.layers.28.mlp.experts.117.down_proj", "model.layers.28.mlp.experts.118.down_proj", "model.layers.28.mlp.experts.119.down_proj", "model.layers.28.mlp.experts.120.down_proj", "model.layers.28.mlp.experts.121.down_proj", "model.layers.28.mlp.experts.122.down_proj", "model.layers.28.mlp.experts.123.down_proj", "model.layers.28.mlp.experts.124.down_proj", "model.layers.28.mlp.experts.125.down_proj", "model.layers.28.mlp.experts.126.down_proj", "model.layers.28.mlp.experts.127.down_proj", "model.layers.28.mlp.experts.128.down_proj", "model.layers.28.mlp.experts.129.down_proj", "model.layers.28.mlp.experts.130.down_proj", "model.layers.28.mlp.experts.131.down_proj", "model.layers.28.mlp.experts.132.down_proj", "model.layers.28.mlp.experts.133.down_proj", "model.layers.28.mlp.experts.134.down_proj", "model.layers.28.mlp.experts.135.down_proj", "model.layers.28.mlp.experts.136.down_proj", "model.layers.28.mlp.experts.137.down_proj", "model.layers.28.mlp.experts.138.down_proj", "model.layers.28.mlp.experts.139.down_proj", "model.layers.28.mlp.experts.140.down_proj", "model.layers.28.mlp.experts.141.down_proj", "model.layers.28.mlp.experts.142.down_proj", "model.layers.28.mlp.experts.143.down_proj", "model.layers.28.mlp.experts.144.down_proj", "model.layers.28.mlp.experts.145.down_proj", "model.layers.28.mlp.experts.146.down_proj", "model.layers.28.mlp.experts.147.down_proj", "model.layers.28.mlp.experts.148.down_proj", "model.layers.28.mlp.experts.149.down_proj", "model.layers.28.mlp.experts.150.down_proj", "model.layers.28.mlp.experts.151.down_proj", "model.layers.28.mlp.experts.152.down_proj", "model.layers.28.mlp.experts.153.down_proj", "model.layers.28.mlp.experts.154.down_proj", "model.layers.28.mlp.experts.155.down_proj", "model.layers.28.mlp.experts.156.down_proj", "model.layers.28.mlp.experts.157.down_proj", "model.layers.28.mlp.experts.158.down_proj", "model.layers.28.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0020892456173897123, "dbits": 1258291200 } ] }, { "idx": 171, "layers": [ "model.layers.29.self_attn.q_proj" ], "candidates": [ { "dkld": -0.001578619331121467, "dbits": 62914560 } ] }, { "idx": 172, "layers": [ "model.layers.29.self_attn.k_proj", "model.layers.29.self_attn.v_proj" ], "candidates": [ { "dkld": -0.007322622835636172, "dbits": 10485760 } ] }, { "idx": 173, "layers": [ "model.layers.29.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0022080816328525765, "dbits": 62914560 } ] }, { "idx": 174, "layers": [ "model.layers.29.mlp.shared_experts.gate_proj", "model.layers.29.mlp.shared_experts.up_proj", "model.layers.29.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.002457425743341457, "dbits": 23592960 } ] }, { "idx": 175, "layers": [ "model.layers.29.mlp.experts.0.gate_proj", "model.layers.29.mlp.experts.1.gate_proj", "model.layers.29.mlp.experts.2.gate_proj", "model.layers.29.mlp.experts.3.gate_proj", "model.layers.29.mlp.experts.4.gate_proj", "model.layers.29.mlp.experts.5.gate_proj", "model.layers.29.mlp.experts.6.gate_proj", "model.layers.29.mlp.experts.7.gate_proj", "model.layers.29.mlp.experts.8.gate_proj", "model.layers.29.mlp.experts.9.gate_proj", "model.layers.29.mlp.experts.10.gate_proj", "model.layers.29.mlp.experts.11.gate_proj", "model.layers.29.mlp.experts.12.gate_proj", "model.layers.29.mlp.experts.13.gate_proj", "model.layers.29.mlp.experts.14.gate_proj", "model.layers.29.mlp.experts.15.gate_proj", "model.layers.29.mlp.experts.16.gate_proj", "model.layers.29.mlp.experts.17.gate_proj", "model.layers.29.mlp.experts.18.gate_proj", "model.layers.29.mlp.experts.19.gate_proj", "model.layers.29.mlp.experts.20.gate_proj", "model.layers.29.mlp.experts.21.gate_proj", "model.layers.29.mlp.experts.22.gate_proj", "model.layers.29.mlp.experts.23.gate_proj", "model.layers.29.mlp.experts.24.gate_proj", "model.layers.29.mlp.experts.25.gate_proj", "model.layers.29.mlp.experts.26.gate_proj", "model.layers.29.mlp.experts.27.gate_proj", "model.layers.29.mlp.experts.28.gate_proj", "model.layers.29.mlp.experts.29.gate_proj", "model.layers.29.mlp.experts.30.gate_proj", "model.layers.29.mlp.experts.31.gate_proj", "model.layers.29.mlp.experts.32.gate_proj", "model.layers.29.mlp.experts.33.gate_proj", "model.layers.29.mlp.experts.34.gate_proj", "model.layers.29.mlp.experts.35.gate_proj", "model.layers.29.mlp.experts.36.gate_proj", "model.layers.29.mlp.experts.37.gate_proj", "model.layers.29.mlp.experts.38.gate_proj", "model.layers.29.mlp.experts.39.gate_proj", "model.layers.29.mlp.experts.40.gate_proj", "model.layers.29.mlp.experts.41.gate_proj", "model.layers.29.mlp.experts.42.gate_proj", "model.layers.29.mlp.experts.43.gate_proj", "model.layers.29.mlp.experts.44.gate_proj", "model.layers.29.mlp.experts.45.gate_proj", "model.layers.29.mlp.experts.46.gate_proj", "model.layers.29.mlp.experts.47.gate_proj", "model.layers.29.mlp.experts.48.gate_proj", "model.layers.29.mlp.experts.49.gate_proj", "model.layers.29.mlp.experts.50.gate_proj", "model.layers.29.mlp.experts.51.gate_proj", "model.layers.29.mlp.experts.52.gate_proj", "model.layers.29.mlp.experts.53.gate_proj", "model.layers.29.mlp.experts.54.gate_proj", "model.layers.29.mlp.experts.55.gate_proj", "model.layers.29.mlp.experts.56.gate_proj", "model.layers.29.mlp.experts.57.gate_proj", "model.layers.29.mlp.experts.58.gate_proj", "model.layers.29.mlp.experts.59.gate_proj", "model.layers.29.mlp.experts.60.gate_proj", "model.layers.29.mlp.experts.61.gate_proj", "model.layers.29.mlp.experts.62.gate_proj", "model.layers.29.mlp.experts.63.gate_proj", "model.layers.29.mlp.experts.64.gate_proj", "model.layers.29.mlp.experts.65.gate_proj", "model.layers.29.mlp.experts.66.gate_proj", "model.layers.29.mlp.experts.67.gate_proj", "model.layers.29.mlp.experts.68.gate_proj", "model.layers.29.mlp.experts.69.gate_proj", "model.layers.29.mlp.experts.70.gate_proj", "model.layers.29.mlp.experts.71.gate_proj", "model.layers.29.mlp.experts.72.gate_proj", "model.layers.29.mlp.experts.73.gate_proj", "model.layers.29.mlp.experts.74.gate_proj", "model.layers.29.mlp.experts.75.gate_proj", "model.layers.29.mlp.experts.76.gate_proj", "model.layers.29.mlp.experts.77.gate_proj", "model.layers.29.mlp.experts.78.gate_proj", "model.layers.29.mlp.experts.79.gate_proj", "model.layers.29.mlp.experts.80.gate_proj", "model.layers.29.mlp.experts.81.gate_proj", "model.layers.29.mlp.experts.82.gate_proj", "model.layers.29.mlp.experts.83.gate_proj", "model.layers.29.mlp.experts.84.gate_proj", "model.layers.29.mlp.experts.85.gate_proj", "model.layers.29.mlp.experts.86.gate_proj", "model.layers.29.mlp.experts.87.gate_proj", "model.layers.29.mlp.experts.88.gate_proj", "model.layers.29.mlp.experts.89.gate_proj", "model.layers.29.mlp.experts.90.gate_proj", "model.layers.29.mlp.experts.91.gate_proj", "model.layers.29.mlp.experts.92.gate_proj", "model.layers.29.mlp.experts.93.gate_proj", "model.layers.29.mlp.experts.94.gate_proj", "model.layers.29.mlp.experts.95.gate_proj", "model.layers.29.mlp.experts.96.gate_proj", "model.layers.29.mlp.experts.97.gate_proj", "model.layers.29.mlp.experts.98.gate_proj", "model.layers.29.mlp.experts.99.gate_proj", "model.layers.29.mlp.experts.100.gate_proj", "model.layers.29.mlp.experts.101.gate_proj", "model.layers.29.mlp.experts.102.gate_proj", "model.layers.29.mlp.experts.103.gate_proj", "model.layers.29.mlp.experts.104.gate_proj", "model.layers.29.mlp.experts.105.gate_proj", "model.layers.29.mlp.experts.106.gate_proj", "model.layers.29.mlp.experts.107.gate_proj", "model.layers.29.mlp.experts.108.gate_proj", "model.layers.29.mlp.experts.109.gate_proj", "model.layers.29.mlp.experts.110.gate_proj", "model.layers.29.mlp.experts.111.gate_proj", "model.layers.29.mlp.experts.112.gate_proj", "model.layers.29.mlp.experts.113.gate_proj", "model.layers.29.mlp.experts.114.gate_proj", "model.layers.29.mlp.experts.115.gate_proj", "model.layers.29.mlp.experts.116.gate_proj", "model.layers.29.mlp.experts.117.gate_proj", "model.layers.29.mlp.experts.118.gate_proj", "model.layers.29.mlp.experts.119.gate_proj", "model.layers.29.mlp.experts.120.gate_proj", "model.layers.29.mlp.experts.121.gate_proj", "model.layers.29.mlp.experts.122.gate_proj", "model.layers.29.mlp.experts.123.gate_proj", "model.layers.29.mlp.experts.124.gate_proj", "model.layers.29.mlp.experts.125.gate_proj", "model.layers.29.mlp.experts.126.gate_proj", "model.layers.29.mlp.experts.127.gate_proj", "model.layers.29.mlp.experts.128.gate_proj", "model.layers.29.mlp.experts.129.gate_proj", "model.layers.29.mlp.experts.130.gate_proj", "model.layers.29.mlp.experts.131.gate_proj", "model.layers.29.mlp.experts.132.gate_proj", "model.layers.29.mlp.experts.133.gate_proj", "model.layers.29.mlp.experts.134.gate_proj", "model.layers.29.mlp.experts.135.gate_proj", "model.layers.29.mlp.experts.136.gate_proj", "model.layers.29.mlp.experts.137.gate_proj", "model.layers.29.mlp.experts.138.gate_proj", "model.layers.29.mlp.experts.139.gate_proj", "model.layers.29.mlp.experts.140.gate_proj", "model.layers.29.mlp.experts.141.gate_proj", "model.layers.29.mlp.experts.142.gate_proj", "model.layers.29.mlp.experts.143.gate_proj", "model.layers.29.mlp.experts.144.gate_proj", "model.layers.29.mlp.experts.145.gate_proj", "model.layers.29.mlp.experts.146.gate_proj", "model.layers.29.mlp.experts.147.gate_proj", "model.layers.29.mlp.experts.148.gate_proj", "model.layers.29.mlp.experts.149.gate_proj", "model.layers.29.mlp.experts.150.gate_proj", "model.layers.29.mlp.experts.151.gate_proj", "model.layers.29.mlp.experts.152.gate_proj", "model.layers.29.mlp.experts.153.gate_proj", "model.layers.29.mlp.experts.154.gate_proj", "model.layers.29.mlp.experts.155.gate_proj", "model.layers.29.mlp.experts.156.gate_proj", "model.layers.29.mlp.experts.157.gate_proj", "model.layers.29.mlp.experts.158.gate_proj", "model.layers.29.mlp.experts.159.gate_proj", "model.layers.29.mlp.experts.0.up_proj", "model.layers.29.mlp.experts.1.up_proj", "model.layers.29.mlp.experts.2.up_proj", "model.layers.29.mlp.experts.3.up_proj", "model.layers.29.mlp.experts.4.up_proj", "model.layers.29.mlp.experts.5.up_proj", "model.layers.29.mlp.experts.6.up_proj", "model.layers.29.mlp.experts.7.up_proj", "model.layers.29.mlp.experts.8.up_proj", "model.layers.29.mlp.experts.9.up_proj", "model.layers.29.mlp.experts.10.up_proj", "model.layers.29.mlp.experts.11.up_proj", "model.layers.29.mlp.experts.12.up_proj", "model.layers.29.mlp.experts.13.up_proj", "model.layers.29.mlp.experts.14.up_proj", "model.layers.29.mlp.experts.15.up_proj", "model.layers.29.mlp.experts.16.up_proj", "model.layers.29.mlp.experts.17.up_proj", "model.layers.29.mlp.experts.18.up_proj", "model.layers.29.mlp.experts.19.up_proj", "model.layers.29.mlp.experts.20.up_proj", "model.layers.29.mlp.experts.21.up_proj", "model.layers.29.mlp.experts.22.up_proj", "model.layers.29.mlp.experts.23.up_proj", "model.layers.29.mlp.experts.24.up_proj", "model.layers.29.mlp.experts.25.up_proj", "model.layers.29.mlp.experts.26.up_proj", "model.layers.29.mlp.experts.27.up_proj", "model.layers.29.mlp.experts.28.up_proj", "model.layers.29.mlp.experts.29.up_proj", "model.layers.29.mlp.experts.30.up_proj", "model.layers.29.mlp.experts.31.up_proj", "model.layers.29.mlp.experts.32.up_proj", "model.layers.29.mlp.experts.33.up_proj", "model.layers.29.mlp.experts.34.up_proj", "model.layers.29.mlp.experts.35.up_proj", "model.layers.29.mlp.experts.36.up_proj", "model.layers.29.mlp.experts.37.up_proj", "model.layers.29.mlp.experts.38.up_proj", "model.layers.29.mlp.experts.39.up_proj", "model.layers.29.mlp.experts.40.up_proj", "model.layers.29.mlp.experts.41.up_proj", "model.layers.29.mlp.experts.42.up_proj", "model.layers.29.mlp.experts.43.up_proj", "model.layers.29.mlp.experts.44.up_proj", "model.layers.29.mlp.experts.45.up_proj", "model.layers.29.mlp.experts.46.up_proj", "model.layers.29.mlp.experts.47.up_proj", "model.layers.29.mlp.experts.48.up_proj", "model.layers.29.mlp.experts.49.up_proj", "model.layers.29.mlp.experts.50.up_proj", "model.layers.29.mlp.experts.51.up_proj", "model.layers.29.mlp.experts.52.up_proj", "model.layers.29.mlp.experts.53.up_proj", "model.layers.29.mlp.experts.54.up_proj", "model.layers.29.mlp.experts.55.up_proj", "model.layers.29.mlp.experts.56.up_proj", "model.layers.29.mlp.experts.57.up_proj", "model.layers.29.mlp.experts.58.up_proj", "model.layers.29.mlp.experts.59.up_proj", "model.layers.29.mlp.experts.60.up_proj", "model.layers.29.mlp.experts.61.up_proj", "model.layers.29.mlp.experts.62.up_proj", "model.layers.29.mlp.experts.63.up_proj", "model.layers.29.mlp.experts.64.up_proj", "model.layers.29.mlp.experts.65.up_proj", "model.layers.29.mlp.experts.66.up_proj", "model.layers.29.mlp.experts.67.up_proj", "model.layers.29.mlp.experts.68.up_proj", "model.layers.29.mlp.experts.69.up_proj", "model.layers.29.mlp.experts.70.up_proj", "model.layers.29.mlp.experts.71.up_proj", "model.layers.29.mlp.experts.72.up_proj", "model.layers.29.mlp.experts.73.up_proj", "model.layers.29.mlp.experts.74.up_proj", "model.layers.29.mlp.experts.75.up_proj", "model.layers.29.mlp.experts.76.up_proj", "model.layers.29.mlp.experts.77.up_proj", "model.layers.29.mlp.experts.78.up_proj", "model.layers.29.mlp.experts.79.up_proj", "model.layers.29.mlp.experts.80.up_proj", "model.layers.29.mlp.experts.81.up_proj", "model.layers.29.mlp.experts.82.up_proj", "model.layers.29.mlp.experts.83.up_proj", "model.layers.29.mlp.experts.84.up_proj", "model.layers.29.mlp.experts.85.up_proj", "model.layers.29.mlp.experts.86.up_proj", "model.layers.29.mlp.experts.87.up_proj", "model.layers.29.mlp.experts.88.up_proj", "model.layers.29.mlp.experts.89.up_proj", "model.layers.29.mlp.experts.90.up_proj", "model.layers.29.mlp.experts.91.up_proj", "model.layers.29.mlp.experts.92.up_proj", "model.layers.29.mlp.experts.93.up_proj", "model.layers.29.mlp.experts.94.up_proj", "model.layers.29.mlp.experts.95.up_proj", "model.layers.29.mlp.experts.96.up_proj", "model.layers.29.mlp.experts.97.up_proj", "model.layers.29.mlp.experts.98.up_proj", "model.layers.29.mlp.experts.99.up_proj", "model.layers.29.mlp.experts.100.up_proj", "model.layers.29.mlp.experts.101.up_proj", "model.layers.29.mlp.experts.102.up_proj", "model.layers.29.mlp.experts.103.up_proj", "model.layers.29.mlp.experts.104.up_proj", "model.layers.29.mlp.experts.105.up_proj", "model.layers.29.mlp.experts.106.up_proj", "model.layers.29.mlp.experts.107.up_proj", "model.layers.29.mlp.experts.108.up_proj", "model.layers.29.mlp.experts.109.up_proj", "model.layers.29.mlp.experts.110.up_proj", "model.layers.29.mlp.experts.111.up_proj", "model.layers.29.mlp.experts.112.up_proj", "model.layers.29.mlp.experts.113.up_proj", "model.layers.29.mlp.experts.114.up_proj", "model.layers.29.mlp.experts.115.up_proj", "model.layers.29.mlp.experts.116.up_proj", "model.layers.29.mlp.experts.117.up_proj", "model.layers.29.mlp.experts.118.up_proj", "model.layers.29.mlp.experts.119.up_proj", "model.layers.29.mlp.experts.120.up_proj", "model.layers.29.mlp.experts.121.up_proj", "model.layers.29.mlp.experts.122.up_proj", "model.layers.29.mlp.experts.123.up_proj", "model.layers.29.mlp.experts.124.up_proj", "model.layers.29.mlp.experts.125.up_proj", "model.layers.29.mlp.experts.126.up_proj", "model.layers.29.mlp.experts.127.up_proj", "model.layers.29.mlp.experts.128.up_proj", "model.layers.29.mlp.experts.129.up_proj", "model.layers.29.mlp.experts.130.up_proj", "model.layers.29.mlp.experts.131.up_proj", "model.layers.29.mlp.experts.132.up_proj", "model.layers.29.mlp.experts.133.up_proj", "model.layers.29.mlp.experts.134.up_proj", "model.layers.29.mlp.experts.135.up_proj", "model.layers.29.mlp.experts.136.up_proj", "model.layers.29.mlp.experts.137.up_proj", "model.layers.29.mlp.experts.138.up_proj", "model.layers.29.mlp.experts.139.up_proj", "model.layers.29.mlp.experts.140.up_proj", "model.layers.29.mlp.experts.141.up_proj", "model.layers.29.mlp.experts.142.up_proj", "model.layers.29.mlp.experts.143.up_proj", "model.layers.29.mlp.experts.144.up_proj", "model.layers.29.mlp.experts.145.up_proj", "model.layers.29.mlp.experts.146.up_proj", "model.layers.29.mlp.experts.147.up_proj", "model.layers.29.mlp.experts.148.up_proj", "model.layers.29.mlp.experts.149.up_proj", "model.layers.29.mlp.experts.150.up_proj", "model.layers.29.mlp.experts.151.up_proj", "model.layers.29.mlp.experts.152.up_proj", "model.layers.29.mlp.experts.153.up_proj", "model.layers.29.mlp.experts.154.up_proj", "model.layers.29.mlp.experts.155.up_proj", "model.layers.29.mlp.experts.156.up_proj", "model.layers.29.mlp.experts.157.up_proj", "model.layers.29.mlp.experts.158.up_proj", "model.layers.29.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00325673967599871, "dbits": 2516582400 } ] }, { "idx": 176, "layers": [ "model.layers.29.mlp.experts.0.down_proj", "model.layers.29.mlp.experts.1.down_proj", "model.layers.29.mlp.experts.2.down_proj", "model.layers.29.mlp.experts.3.down_proj", "model.layers.29.mlp.experts.4.down_proj", "model.layers.29.mlp.experts.5.down_proj", "model.layers.29.mlp.experts.6.down_proj", "model.layers.29.mlp.experts.7.down_proj", "model.layers.29.mlp.experts.8.down_proj", "model.layers.29.mlp.experts.9.down_proj", "model.layers.29.mlp.experts.10.down_proj", "model.layers.29.mlp.experts.11.down_proj", "model.layers.29.mlp.experts.12.down_proj", "model.layers.29.mlp.experts.13.down_proj", "model.layers.29.mlp.experts.14.down_proj", "model.layers.29.mlp.experts.15.down_proj", "model.layers.29.mlp.experts.16.down_proj", "model.layers.29.mlp.experts.17.down_proj", "model.layers.29.mlp.experts.18.down_proj", "model.layers.29.mlp.experts.19.down_proj", "model.layers.29.mlp.experts.20.down_proj", "model.layers.29.mlp.experts.21.down_proj", "model.layers.29.mlp.experts.22.down_proj", "model.layers.29.mlp.experts.23.down_proj", "model.layers.29.mlp.experts.24.down_proj", "model.layers.29.mlp.experts.25.down_proj", "model.layers.29.mlp.experts.26.down_proj", "model.layers.29.mlp.experts.27.down_proj", "model.layers.29.mlp.experts.28.down_proj", "model.layers.29.mlp.experts.29.down_proj", "model.layers.29.mlp.experts.30.down_proj", "model.layers.29.mlp.experts.31.down_proj", "model.layers.29.mlp.experts.32.down_proj", "model.layers.29.mlp.experts.33.down_proj", "model.layers.29.mlp.experts.34.down_proj", "model.layers.29.mlp.experts.35.down_proj", "model.layers.29.mlp.experts.36.down_proj", "model.layers.29.mlp.experts.37.down_proj", "model.layers.29.mlp.experts.38.down_proj", "model.layers.29.mlp.experts.39.down_proj", "model.layers.29.mlp.experts.40.down_proj", "model.layers.29.mlp.experts.41.down_proj", "model.layers.29.mlp.experts.42.down_proj", "model.layers.29.mlp.experts.43.down_proj", "model.layers.29.mlp.experts.44.down_proj", "model.layers.29.mlp.experts.45.down_proj", "model.layers.29.mlp.experts.46.down_proj", "model.layers.29.mlp.experts.47.down_proj", "model.layers.29.mlp.experts.48.down_proj", "model.layers.29.mlp.experts.49.down_proj", "model.layers.29.mlp.experts.50.down_proj", "model.layers.29.mlp.experts.51.down_proj", "model.layers.29.mlp.experts.52.down_proj", "model.layers.29.mlp.experts.53.down_proj", "model.layers.29.mlp.experts.54.down_proj", "model.layers.29.mlp.experts.55.down_proj", "model.layers.29.mlp.experts.56.down_proj", "model.layers.29.mlp.experts.57.down_proj", "model.layers.29.mlp.experts.58.down_proj", "model.layers.29.mlp.experts.59.down_proj", "model.layers.29.mlp.experts.60.down_proj", "model.layers.29.mlp.experts.61.down_proj", "model.layers.29.mlp.experts.62.down_proj", "model.layers.29.mlp.experts.63.down_proj", "model.layers.29.mlp.experts.64.down_proj", "model.layers.29.mlp.experts.65.down_proj", "model.layers.29.mlp.experts.66.down_proj", "model.layers.29.mlp.experts.67.down_proj", "model.layers.29.mlp.experts.68.down_proj", "model.layers.29.mlp.experts.69.down_proj", "model.layers.29.mlp.experts.70.down_proj", "model.layers.29.mlp.experts.71.down_proj", "model.layers.29.mlp.experts.72.down_proj", "model.layers.29.mlp.experts.73.down_proj", "model.layers.29.mlp.experts.74.down_proj", "model.layers.29.mlp.experts.75.down_proj", "model.layers.29.mlp.experts.76.down_proj", "model.layers.29.mlp.experts.77.down_proj", "model.layers.29.mlp.experts.78.down_proj", "model.layers.29.mlp.experts.79.down_proj", "model.layers.29.mlp.experts.80.down_proj", "model.layers.29.mlp.experts.81.down_proj", "model.layers.29.mlp.experts.82.down_proj", "model.layers.29.mlp.experts.83.down_proj", "model.layers.29.mlp.experts.84.down_proj", "model.layers.29.mlp.experts.85.down_proj", "model.layers.29.mlp.experts.86.down_proj", "model.layers.29.mlp.experts.87.down_proj", "model.layers.29.mlp.experts.88.down_proj", "model.layers.29.mlp.experts.89.down_proj", "model.layers.29.mlp.experts.90.down_proj", "model.layers.29.mlp.experts.91.down_proj", "model.layers.29.mlp.experts.92.down_proj", "model.layers.29.mlp.experts.93.down_proj", "model.layers.29.mlp.experts.94.down_proj", "model.layers.29.mlp.experts.95.down_proj", "model.layers.29.mlp.experts.96.down_proj", "model.layers.29.mlp.experts.97.down_proj", "model.layers.29.mlp.experts.98.down_proj", "model.layers.29.mlp.experts.99.down_proj", "model.layers.29.mlp.experts.100.down_proj", "model.layers.29.mlp.experts.101.down_proj", "model.layers.29.mlp.experts.102.down_proj", "model.layers.29.mlp.experts.103.down_proj", "model.layers.29.mlp.experts.104.down_proj", "model.layers.29.mlp.experts.105.down_proj", "model.layers.29.mlp.experts.106.down_proj", "model.layers.29.mlp.experts.107.down_proj", "model.layers.29.mlp.experts.108.down_proj", "model.layers.29.mlp.experts.109.down_proj", "model.layers.29.mlp.experts.110.down_proj", "model.layers.29.mlp.experts.111.down_proj", "model.layers.29.mlp.experts.112.down_proj", "model.layers.29.mlp.experts.113.down_proj", "model.layers.29.mlp.experts.114.down_proj", "model.layers.29.mlp.experts.115.down_proj", "model.layers.29.mlp.experts.116.down_proj", "model.layers.29.mlp.experts.117.down_proj", "model.layers.29.mlp.experts.118.down_proj", "model.layers.29.mlp.experts.119.down_proj", "model.layers.29.mlp.experts.120.down_proj", "model.layers.29.mlp.experts.121.down_proj", "model.layers.29.mlp.experts.122.down_proj", "model.layers.29.mlp.experts.123.down_proj", "model.layers.29.mlp.experts.124.down_proj", "model.layers.29.mlp.experts.125.down_proj", "model.layers.29.mlp.experts.126.down_proj", "model.layers.29.mlp.experts.127.down_proj", "model.layers.29.mlp.experts.128.down_proj", "model.layers.29.mlp.experts.129.down_proj", "model.layers.29.mlp.experts.130.down_proj", "model.layers.29.mlp.experts.131.down_proj", "model.layers.29.mlp.experts.132.down_proj", "model.layers.29.mlp.experts.133.down_proj", "model.layers.29.mlp.experts.134.down_proj", "model.layers.29.mlp.experts.135.down_proj", "model.layers.29.mlp.experts.136.down_proj", "model.layers.29.mlp.experts.137.down_proj", "model.layers.29.mlp.experts.138.down_proj", "model.layers.29.mlp.experts.139.down_proj", "model.layers.29.mlp.experts.140.down_proj", "model.layers.29.mlp.experts.141.down_proj", "model.layers.29.mlp.experts.142.down_proj", "model.layers.29.mlp.experts.143.down_proj", "model.layers.29.mlp.experts.144.down_proj", "model.layers.29.mlp.experts.145.down_proj", "model.layers.29.mlp.experts.146.down_proj", "model.layers.29.mlp.experts.147.down_proj", "model.layers.29.mlp.experts.148.down_proj", "model.layers.29.mlp.experts.149.down_proj", "model.layers.29.mlp.experts.150.down_proj", "model.layers.29.mlp.experts.151.down_proj", "model.layers.29.mlp.experts.152.down_proj", "model.layers.29.mlp.experts.153.down_proj", "model.layers.29.mlp.experts.154.down_proj", "model.layers.29.mlp.experts.155.down_proj", "model.layers.29.mlp.experts.156.down_proj", "model.layers.29.mlp.experts.157.down_proj", "model.layers.29.mlp.experts.158.down_proj", "model.layers.29.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0019296169281006192, "dbits": 1258291200 } ] }, { "idx": 177, "layers": [ "model.layers.30.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0008313171565532684, "dbits": 62914560 } ] }, { "idx": 178, "layers": [ "model.layers.30.self_attn.k_proj", "model.layers.30.self_attn.v_proj" ], "candidates": [ { "dkld": -0.005953115224838268, "dbits": 10485760 } ] }, { "idx": 179, "layers": [ "model.layers.30.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00748574882745745, "dbits": 62914560 } ] }, { "idx": 180, "layers": [ "model.layers.30.mlp.shared_experts.gate_proj", "model.layers.30.mlp.shared_experts.up_proj", "model.layers.30.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00390627682209016, "dbits": 23592960 } ] }, { "idx": 181, "layers": [ "model.layers.30.mlp.experts.0.gate_proj", "model.layers.30.mlp.experts.1.gate_proj", "model.layers.30.mlp.experts.2.gate_proj", "model.layers.30.mlp.experts.3.gate_proj", "model.layers.30.mlp.experts.4.gate_proj", "model.layers.30.mlp.experts.5.gate_proj", "model.layers.30.mlp.experts.6.gate_proj", "model.layers.30.mlp.experts.7.gate_proj", "model.layers.30.mlp.experts.8.gate_proj", "model.layers.30.mlp.experts.9.gate_proj", "model.layers.30.mlp.experts.10.gate_proj", "model.layers.30.mlp.experts.11.gate_proj", "model.layers.30.mlp.experts.12.gate_proj", "model.layers.30.mlp.experts.13.gate_proj", "model.layers.30.mlp.experts.14.gate_proj", "model.layers.30.mlp.experts.15.gate_proj", "model.layers.30.mlp.experts.16.gate_proj", "model.layers.30.mlp.experts.17.gate_proj", "model.layers.30.mlp.experts.18.gate_proj", "model.layers.30.mlp.experts.19.gate_proj", "model.layers.30.mlp.experts.20.gate_proj", "model.layers.30.mlp.experts.21.gate_proj", "model.layers.30.mlp.experts.22.gate_proj", "model.layers.30.mlp.experts.23.gate_proj", "model.layers.30.mlp.experts.24.gate_proj", "model.layers.30.mlp.experts.25.gate_proj", "model.layers.30.mlp.experts.26.gate_proj", "model.layers.30.mlp.experts.27.gate_proj", "model.layers.30.mlp.experts.28.gate_proj", "model.layers.30.mlp.experts.29.gate_proj", "model.layers.30.mlp.experts.30.gate_proj", "model.layers.30.mlp.experts.31.gate_proj", "model.layers.30.mlp.experts.32.gate_proj", "model.layers.30.mlp.experts.33.gate_proj", "model.layers.30.mlp.experts.34.gate_proj", "model.layers.30.mlp.experts.35.gate_proj", "model.layers.30.mlp.experts.36.gate_proj", "model.layers.30.mlp.experts.37.gate_proj", "model.layers.30.mlp.experts.38.gate_proj", "model.layers.30.mlp.experts.39.gate_proj", "model.layers.30.mlp.experts.40.gate_proj", "model.layers.30.mlp.experts.41.gate_proj", "model.layers.30.mlp.experts.42.gate_proj", "model.layers.30.mlp.experts.43.gate_proj", "model.layers.30.mlp.experts.44.gate_proj", "model.layers.30.mlp.experts.45.gate_proj", "model.layers.30.mlp.experts.46.gate_proj", "model.layers.30.mlp.experts.47.gate_proj", "model.layers.30.mlp.experts.48.gate_proj", "model.layers.30.mlp.experts.49.gate_proj", "model.layers.30.mlp.experts.50.gate_proj", "model.layers.30.mlp.experts.51.gate_proj", "model.layers.30.mlp.experts.52.gate_proj", "model.layers.30.mlp.experts.53.gate_proj", "model.layers.30.mlp.experts.54.gate_proj", "model.layers.30.mlp.experts.55.gate_proj", "model.layers.30.mlp.experts.56.gate_proj", "model.layers.30.mlp.experts.57.gate_proj", "model.layers.30.mlp.experts.58.gate_proj", "model.layers.30.mlp.experts.59.gate_proj", "model.layers.30.mlp.experts.60.gate_proj", "model.layers.30.mlp.experts.61.gate_proj", "model.layers.30.mlp.experts.62.gate_proj", "model.layers.30.mlp.experts.63.gate_proj", "model.layers.30.mlp.experts.64.gate_proj", "model.layers.30.mlp.experts.65.gate_proj", "model.layers.30.mlp.experts.66.gate_proj", "model.layers.30.mlp.experts.67.gate_proj", "model.layers.30.mlp.experts.68.gate_proj", "model.layers.30.mlp.experts.69.gate_proj", "model.layers.30.mlp.experts.70.gate_proj", "model.layers.30.mlp.experts.71.gate_proj", "model.layers.30.mlp.experts.72.gate_proj", "model.layers.30.mlp.experts.73.gate_proj", "model.layers.30.mlp.experts.74.gate_proj", "model.layers.30.mlp.experts.75.gate_proj", "model.layers.30.mlp.experts.76.gate_proj", "model.layers.30.mlp.experts.77.gate_proj", "model.layers.30.mlp.experts.78.gate_proj", "model.layers.30.mlp.experts.79.gate_proj", "model.layers.30.mlp.experts.80.gate_proj", "model.layers.30.mlp.experts.81.gate_proj", "model.layers.30.mlp.experts.82.gate_proj", "model.layers.30.mlp.experts.83.gate_proj", "model.layers.30.mlp.experts.84.gate_proj", "model.layers.30.mlp.experts.85.gate_proj", "model.layers.30.mlp.experts.86.gate_proj", "model.layers.30.mlp.experts.87.gate_proj", "model.layers.30.mlp.experts.88.gate_proj", "model.layers.30.mlp.experts.89.gate_proj", "model.layers.30.mlp.experts.90.gate_proj", "model.layers.30.mlp.experts.91.gate_proj", "model.layers.30.mlp.experts.92.gate_proj", "model.layers.30.mlp.experts.93.gate_proj", "model.layers.30.mlp.experts.94.gate_proj", "model.layers.30.mlp.experts.95.gate_proj", "model.layers.30.mlp.experts.96.gate_proj", "model.layers.30.mlp.experts.97.gate_proj", "model.layers.30.mlp.experts.98.gate_proj", "model.layers.30.mlp.experts.99.gate_proj", "model.layers.30.mlp.experts.100.gate_proj", "model.layers.30.mlp.experts.101.gate_proj", "model.layers.30.mlp.experts.102.gate_proj", "model.layers.30.mlp.experts.103.gate_proj", "model.layers.30.mlp.experts.104.gate_proj", "model.layers.30.mlp.experts.105.gate_proj", "model.layers.30.mlp.experts.106.gate_proj", "model.layers.30.mlp.experts.107.gate_proj", "model.layers.30.mlp.experts.108.gate_proj", "model.layers.30.mlp.experts.109.gate_proj", "model.layers.30.mlp.experts.110.gate_proj", "model.layers.30.mlp.experts.111.gate_proj", "model.layers.30.mlp.experts.112.gate_proj", "model.layers.30.mlp.experts.113.gate_proj", "model.layers.30.mlp.experts.114.gate_proj", "model.layers.30.mlp.experts.115.gate_proj", "model.layers.30.mlp.experts.116.gate_proj", "model.layers.30.mlp.experts.117.gate_proj", "model.layers.30.mlp.experts.118.gate_proj", "model.layers.30.mlp.experts.119.gate_proj", "model.layers.30.mlp.experts.120.gate_proj", "model.layers.30.mlp.experts.121.gate_proj", "model.layers.30.mlp.experts.122.gate_proj", "model.layers.30.mlp.experts.123.gate_proj", "model.layers.30.mlp.experts.124.gate_proj", "model.layers.30.mlp.experts.125.gate_proj", "model.layers.30.mlp.experts.126.gate_proj", "model.layers.30.mlp.experts.127.gate_proj", "model.layers.30.mlp.experts.128.gate_proj", "model.layers.30.mlp.experts.129.gate_proj", "model.layers.30.mlp.experts.130.gate_proj", "model.layers.30.mlp.experts.131.gate_proj", "model.layers.30.mlp.experts.132.gate_proj", "model.layers.30.mlp.experts.133.gate_proj", "model.layers.30.mlp.experts.134.gate_proj", "model.layers.30.mlp.experts.135.gate_proj", "model.layers.30.mlp.experts.136.gate_proj", "model.layers.30.mlp.experts.137.gate_proj", "model.layers.30.mlp.experts.138.gate_proj", "model.layers.30.mlp.experts.139.gate_proj", "model.layers.30.mlp.experts.140.gate_proj", "model.layers.30.mlp.experts.141.gate_proj", "model.layers.30.mlp.experts.142.gate_proj", "model.layers.30.mlp.experts.143.gate_proj", "model.layers.30.mlp.experts.144.gate_proj", "model.layers.30.mlp.experts.145.gate_proj", "model.layers.30.mlp.experts.146.gate_proj", "model.layers.30.mlp.experts.147.gate_proj", "model.layers.30.mlp.experts.148.gate_proj", "model.layers.30.mlp.experts.149.gate_proj", "model.layers.30.mlp.experts.150.gate_proj", "model.layers.30.mlp.experts.151.gate_proj", "model.layers.30.mlp.experts.152.gate_proj", "model.layers.30.mlp.experts.153.gate_proj", "model.layers.30.mlp.experts.154.gate_proj", "model.layers.30.mlp.experts.155.gate_proj", "model.layers.30.mlp.experts.156.gate_proj", "model.layers.30.mlp.experts.157.gate_proj", "model.layers.30.mlp.experts.158.gate_proj", "model.layers.30.mlp.experts.159.gate_proj", "model.layers.30.mlp.experts.0.up_proj", "model.layers.30.mlp.experts.1.up_proj", "model.layers.30.mlp.experts.2.up_proj", "model.layers.30.mlp.experts.3.up_proj", "model.layers.30.mlp.experts.4.up_proj", "model.layers.30.mlp.experts.5.up_proj", "model.layers.30.mlp.experts.6.up_proj", "model.layers.30.mlp.experts.7.up_proj", "model.layers.30.mlp.experts.8.up_proj", "model.layers.30.mlp.experts.9.up_proj", "model.layers.30.mlp.experts.10.up_proj", "model.layers.30.mlp.experts.11.up_proj", "model.layers.30.mlp.experts.12.up_proj", "model.layers.30.mlp.experts.13.up_proj", "model.layers.30.mlp.experts.14.up_proj", "model.layers.30.mlp.experts.15.up_proj", "model.layers.30.mlp.experts.16.up_proj", "model.layers.30.mlp.experts.17.up_proj", "model.layers.30.mlp.experts.18.up_proj", "model.layers.30.mlp.experts.19.up_proj", "model.layers.30.mlp.experts.20.up_proj", "model.layers.30.mlp.experts.21.up_proj", "model.layers.30.mlp.experts.22.up_proj", "model.layers.30.mlp.experts.23.up_proj", "model.layers.30.mlp.experts.24.up_proj", "model.layers.30.mlp.experts.25.up_proj", "model.layers.30.mlp.experts.26.up_proj", "model.layers.30.mlp.experts.27.up_proj", "model.layers.30.mlp.experts.28.up_proj", "model.layers.30.mlp.experts.29.up_proj", "model.layers.30.mlp.experts.30.up_proj", "model.layers.30.mlp.experts.31.up_proj", "model.layers.30.mlp.experts.32.up_proj", "model.layers.30.mlp.experts.33.up_proj", "model.layers.30.mlp.experts.34.up_proj", "model.layers.30.mlp.experts.35.up_proj", "model.layers.30.mlp.experts.36.up_proj", "model.layers.30.mlp.experts.37.up_proj", "model.layers.30.mlp.experts.38.up_proj", "model.layers.30.mlp.experts.39.up_proj", "model.layers.30.mlp.experts.40.up_proj", "model.layers.30.mlp.experts.41.up_proj", "model.layers.30.mlp.experts.42.up_proj", "model.layers.30.mlp.experts.43.up_proj", "model.layers.30.mlp.experts.44.up_proj", "model.layers.30.mlp.experts.45.up_proj", "model.layers.30.mlp.experts.46.up_proj", "model.layers.30.mlp.experts.47.up_proj", "model.layers.30.mlp.experts.48.up_proj", "model.layers.30.mlp.experts.49.up_proj", "model.layers.30.mlp.experts.50.up_proj", "model.layers.30.mlp.experts.51.up_proj", "model.layers.30.mlp.experts.52.up_proj", "model.layers.30.mlp.experts.53.up_proj", "model.layers.30.mlp.experts.54.up_proj", "model.layers.30.mlp.experts.55.up_proj", "model.layers.30.mlp.experts.56.up_proj", "model.layers.30.mlp.experts.57.up_proj", "model.layers.30.mlp.experts.58.up_proj", "model.layers.30.mlp.experts.59.up_proj", "model.layers.30.mlp.experts.60.up_proj", "model.layers.30.mlp.experts.61.up_proj", "model.layers.30.mlp.experts.62.up_proj", "model.layers.30.mlp.experts.63.up_proj", "model.layers.30.mlp.experts.64.up_proj", "model.layers.30.mlp.experts.65.up_proj", "model.layers.30.mlp.experts.66.up_proj", "model.layers.30.mlp.experts.67.up_proj", "model.layers.30.mlp.experts.68.up_proj", "model.layers.30.mlp.experts.69.up_proj", "model.layers.30.mlp.experts.70.up_proj", "model.layers.30.mlp.experts.71.up_proj", "model.layers.30.mlp.experts.72.up_proj", "model.layers.30.mlp.experts.73.up_proj", "model.layers.30.mlp.experts.74.up_proj", "model.layers.30.mlp.experts.75.up_proj", "model.layers.30.mlp.experts.76.up_proj", "model.layers.30.mlp.experts.77.up_proj", "model.layers.30.mlp.experts.78.up_proj", "model.layers.30.mlp.experts.79.up_proj", "model.layers.30.mlp.experts.80.up_proj", "model.layers.30.mlp.experts.81.up_proj", "model.layers.30.mlp.experts.82.up_proj", "model.layers.30.mlp.experts.83.up_proj", "model.layers.30.mlp.experts.84.up_proj", "model.layers.30.mlp.experts.85.up_proj", "model.layers.30.mlp.experts.86.up_proj", "model.layers.30.mlp.experts.87.up_proj", "model.layers.30.mlp.experts.88.up_proj", "model.layers.30.mlp.experts.89.up_proj", "model.layers.30.mlp.experts.90.up_proj", "model.layers.30.mlp.experts.91.up_proj", "model.layers.30.mlp.experts.92.up_proj", "model.layers.30.mlp.experts.93.up_proj", "model.layers.30.mlp.experts.94.up_proj", "model.layers.30.mlp.experts.95.up_proj", "model.layers.30.mlp.experts.96.up_proj", "model.layers.30.mlp.experts.97.up_proj", "model.layers.30.mlp.experts.98.up_proj", "model.layers.30.mlp.experts.99.up_proj", "model.layers.30.mlp.experts.100.up_proj", "model.layers.30.mlp.experts.101.up_proj", "model.layers.30.mlp.experts.102.up_proj", "model.layers.30.mlp.experts.103.up_proj", "model.layers.30.mlp.experts.104.up_proj", "model.layers.30.mlp.experts.105.up_proj", "model.layers.30.mlp.experts.106.up_proj", "model.layers.30.mlp.experts.107.up_proj", "model.layers.30.mlp.experts.108.up_proj", "model.layers.30.mlp.experts.109.up_proj", "model.layers.30.mlp.experts.110.up_proj", "model.layers.30.mlp.experts.111.up_proj", "model.layers.30.mlp.experts.112.up_proj", "model.layers.30.mlp.experts.113.up_proj", "model.layers.30.mlp.experts.114.up_proj", "model.layers.30.mlp.experts.115.up_proj", "model.layers.30.mlp.experts.116.up_proj", "model.layers.30.mlp.experts.117.up_proj", "model.layers.30.mlp.experts.118.up_proj", "model.layers.30.mlp.experts.119.up_proj", "model.layers.30.mlp.experts.120.up_proj", "model.layers.30.mlp.experts.121.up_proj", "model.layers.30.mlp.experts.122.up_proj", "model.layers.30.mlp.experts.123.up_proj", "model.layers.30.mlp.experts.124.up_proj", "model.layers.30.mlp.experts.125.up_proj", "model.layers.30.mlp.experts.126.up_proj", "model.layers.30.mlp.experts.127.up_proj", "model.layers.30.mlp.experts.128.up_proj", "model.layers.30.mlp.experts.129.up_proj", "model.layers.30.mlp.experts.130.up_proj", "model.layers.30.mlp.experts.131.up_proj", "model.layers.30.mlp.experts.132.up_proj", "model.layers.30.mlp.experts.133.up_proj", "model.layers.30.mlp.experts.134.up_proj", "model.layers.30.mlp.experts.135.up_proj", "model.layers.30.mlp.experts.136.up_proj", "model.layers.30.mlp.experts.137.up_proj", "model.layers.30.mlp.experts.138.up_proj", "model.layers.30.mlp.experts.139.up_proj", "model.layers.30.mlp.experts.140.up_proj", "model.layers.30.mlp.experts.141.up_proj", "model.layers.30.mlp.experts.142.up_proj", "model.layers.30.mlp.experts.143.up_proj", "model.layers.30.mlp.experts.144.up_proj", "model.layers.30.mlp.experts.145.up_proj", "model.layers.30.mlp.experts.146.up_proj", "model.layers.30.mlp.experts.147.up_proj", "model.layers.30.mlp.experts.148.up_proj", "model.layers.30.mlp.experts.149.up_proj", "model.layers.30.mlp.experts.150.up_proj", "model.layers.30.mlp.experts.151.up_proj", "model.layers.30.mlp.experts.152.up_proj", "model.layers.30.mlp.experts.153.up_proj", "model.layers.30.mlp.experts.154.up_proj", "model.layers.30.mlp.experts.155.up_proj", "model.layers.30.mlp.experts.156.up_proj", "model.layers.30.mlp.experts.157.up_proj", "model.layers.30.mlp.experts.158.up_proj", "model.layers.30.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.004492390155792225, "dbits": 2516582400 } ] }, { "idx": 182, "layers": [ "model.layers.30.mlp.experts.0.down_proj", "model.layers.30.mlp.experts.1.down_proj", "model.layers.30.mlp.experts.2.down_proj", "model.layers.30.mlp.experts.3.down_proj", "model.layers.30.mlp.experts.4.down_proj", "model.layers.30.mlp.experts.5.down_proj", "model.layers.30.mlp.experts.6.down_proj", "model.layers.30.mlp.experts.7.down_proj", "model.layers.30.mlp.experts.8.down_proj", "model.layers.30.mlp.experts.9.down_proj", "model.layers.30.mlp.experts.10.down_proj", "model.layers.30.mlp.experts.11.down_proj", "model.layers.30.mlp.experts.12.down_proj", "model.layers.30.mlp.experts.13.down_proj", "model.layers.30.mlp.experts.14.down_proj", "model.layers.30.mlp.experts.15.down_proj", "model.layers.30.mlp.experts.16.down_proj", "model.layers.30.mlp.experts.17.down_proj", "model.layers.30.mlp.experts.18.down_proj", "model.layers.30.mlp.experts.19.down_proj", "model.layers.30.mlp.experts.20.down_proj", "model.layers.30.mlp.experts.21.down_proj", "model.layers.30.mlp.experts.22.down_proj", "model.layers.30.mlp.experts.23.down_proj", "model.layers.30.mlp.experts.24.down_proj", "model.layers.30.mlp.experts.25.down_proj", "model.layers.30.mlp.experts.26.down_proj", "model.layers.30.mlp.experts.27.down_proj", "model.layers.30.mlp.experts.28.down_proj", "model.layers.30.mlp.experts.29.down_proj", "model.layers.30.mlp.experts.30.down_proj", "model.layers.30.mlp.experts.31.down_proj", "model.layers.30.mlp.experts.32.down_proj", "model.layers.30.mlp.experts.33.down_proj", "model.layers.30.mlp.experts.34.down_proj", "model.layers.30.mlp.experts.35.down_proj", "model.layers.30.mlp.experts.36.down_proj", "model.layers.30.mlp.experts.37.down_proj", "model.layers.30.mlp.experts.38.down_proj", "model.layers.30.mlp.experts.39.down_proj", "model.layers.30.mlp.experts.40.down_proj", "model.layers.30.mlp.experts.41.down_proj", "model.layers.30.mlp.experts.42.down_proj", "model.layers.30.mlp.experts.43.down_proj", "model.layers.30.mlp.experts.44.down_proj", "model.layers.30.mlp.experts.45.down_proj", "model.layers.30.mlp.experts.46.down_proj", "model.layers.30.mlp.experts.47.down_proj", "model.layers.30.mlp.experts.48.down_proj", "model.layers.30.mlp.experts.49.down_proj", "model.layers.30.mlp.experts.50.down_proj", "model.layers.30.mlp.experts.51.down_proj", "model.layers.30.mlp.experts.52.down_proj", "model.layers.30.mlp.experts.53.down_proj", "model.layers.30.mlp.experts.54.down_proj", "model.layers.30.mlp.experts.55.down_proj", "model.layers.30.mlp.experts.56.down_proj", "model.layers.30.mlp.experts.57.down_proj", "model.layers.30.mlp.experts.58.down_proj", "model.layers.30.mlp.experts.59.down_proj", "model.layers.30.mlp.experts.60.down_proj", "model.layers.30.mlp.experts.61.down_proj", "model.layers.30.mlp.experts.62.down_proj", "model.layers.30.mlp.experts.63.down_proj", "model.layers.30.mlp.experts.64.down_proj", "model.layers.30.mlp.experts.65.down_proj", "model.layers.30.mlp.experts.66.down_proj", "model.layers.30.mlp.experts.67.down_proj", "model.layers.30.mlp.experts.68.down_proj", "model.layers.30.mlp.experts.69.down_proj", "model.layers.30.mlp.experts.70.down_proj", "model.layers.30.mlp.experts.71.down_proj", "model.layers.30.mlp.experts.72.down_proj", "model.layers.30.mlp.experts.73.down_proj", "model.layers.30.mlp.experts.74.down_proj", "model.layers.30.mlp.experts.75.down_proj", "model.layers.30.mlp.experts.76.down_proj", "model.layers.30.mlp.experts.77.down_proj", "model.layers.30.mlp.experts.78.down_proj", "model.layers.30.mlp.experts.79.down_proj", "model.layers.30.mlp.experts.80.down_proj", "model.layers.30.mlp.experts.81.down_proj", "model.layers.30.mlp.experts.82.down_proj", "model.layers.30.mlp.experts.83.down_proj", "model.layers.30.mlp.experts.84.down_proj", "model.layers.30.mlp.experts.85.down_proj", "model.layers.30.mlp.experts.86.down_proj", "model.layers.30.mlp.experts.87.down_proj", "model.layers.30.mlp.experts.88.down_proj", "model.layers.30.mlp.experts.89.down_proj", "model.layers.30.mlp.experts.90.down_proj", "model.layers.30.mlp.experts.91.down_proj", "model.layers.30.mlp.experts.92.down_proj", "model.layers.30.mlp.experts.93.down_proj", "model.layers.30.mlp.experts.94.down_proj", "model.layers.30.mlp.experts.95.down_proj", "model.layers.30.mlp.experts.96.down_proj", "model.layers.30.mlp.experts.97.down_proj", "model.layers.30.mlp.experts.98.down_proj", "model.layers.30.mlp.experts.99.down_proj", "model.layers.30.mlp.experts.100.down_proj", "model.layers.30.mlp.experts.101.down_proj", "model.layers.30.mlp.experts.102.down_proj", "model.layers.30.mlp.experts.103.down_proj", "model.layers.30.mlp.experts.104.down_proj", "model.layers.30.mlp.experts.105.down_proj", "model.layers.30.mlp.experts.106.down_proj", "model.layers.30.mlp.experts.107.down_proj", "model.layers.30.mlp.experts.108.down_proj", "model.layers.30.mlp.experts.109.down_proj", "model.layers.30.mlp.experts.110.down_proj", "model.layers.30.mlp.experts.111.down_proj", "model.layers.30.mlp.experts.112.down_proj", "model.layers.30.mlp.experts.113.down_proj", "model.layers.30.mlp.experts.114.down_proj", "model.layers.30.mlp.experts.115.down_proj", "model.layers.30.mlp.experts.116.down_proj", "model.layers.30.mlp.experts.117.down_proj", "model.layers.30.mlp.experts.118.down_proj", "model.layers.30.mlp.experts.119.down_proj", "model.layers.30.mlp.experts.120.down_proj", "model.layers.30.mlp.experts.121.down_proj", "model.layers.30.mlp.experts.122.down_proj", "model.layers.30.mlp.experts.123.down_proj", "model.layers.30.mlp.experts.124.down_proj", "model.layers.30.mlp.experts.125.down_proj", "model.layers.30.mlp.experts.126.down_proj", "model.layers.30.mlp.experts.127.down_proj", "model.layers.30.mlp.experts.128.down_proj", "model.layers.30.mlp.experts.129.down_proj", "model.layers.30.mlp.experts.130.down_proj", "model.layers.30.mlp.experts.131.down_proj", "model.layers.30.mlp.experts.132.down_proj", "model.layers.30.mlp.experts.133.down_proj", "model.layers.30.mlp.experts.134.down_proj", "model.layers.30.mlp.experts.135.down_proj", "model.layers.30.mlp.experts.136.down_proj", "model.layers.30.mlp.experts.137.down_proj", "model.layers.30.mlp.experts.138.down_proj", "model.layers.30.mlp.experts.139.down_proj", "model.layers.30.mlp.experts.140.down_proj", "model.layers.30.mlp.experts.141.down_proj", "model.layers.30.mlp.experts.142.down_proj", "model.layers.30.mlp.experts.143.down_proj", "model.layers.30.mlp.experts.144.down_proj", "model.layers.30.mlp.experts.145.down_proj", "model.layers.30.mlp.experts.146.down_proj", "model.layers.30.mlp.experts.147.down_proj", "model.layers.30.mlp.experts.148.down_proj", "model.layers.30.mlp.experts.149.down_proj", "model.layers.30.mlp.experts.150.down_proj", "model.layers.30.mlp.experts.151.down_proj", "model.layers.30.mlp.experts.152.down_proj", "model.layers.30.mlp.experts.153.down_proj", "model.layers.30.mlp.experts.154.down_proj", "model.layers.30.mlp.experts.155.down_proj", "model.layers.30.mlp.experts.156.down_proj", "model.layers.30.mlp.experts.157.down_proj", "model.layers.30.mlp.experts.158.down_proj", "model.layers.30.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0023116253316402546, "dbits": 1258291200 } ] }, { "idx": 183, "layers": [ "model.layers.31.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0018058449029922374, "dbits": 62914560 } ] }, { "idx": 184, "layers": [ "model.layers.31.self_attn.k_proj", "model.layers.31.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0013874545693397522, "dbits": 10485760 } ] }, { "idx": 185, "layers": [ "model.layers.31.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0005826711654662864, "dbits": 62914560 } ] }, { "idx": 186, "layers": [ "model.layers.31.mlp.shared_experts.gate_proj", "model.layers.31.mlp.shared_experts.up_proj", "model.layers.31.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.004131151735782601, "dbits": 23592960 } ] }, { "idx": 187, "layers": [ "model.layers.31.mlp.experts.0.gate_proj", "model.layers.31.mlp.experts.1.gate_proj", "model.layers.31.mlp.experts.2.gate_proj", "model.layers.31.mlp.experts.3.gate_proj", "model.layers.31.mlp.experts.4.gate_proj", "model.layers.31.mlp.experts.5.gate_proj", "model.layers.31.mlp.experts.6.gate_proj", "model.layers.31.mlp.experts.7.gate_proj", "model.layers.31.mlp.experts.8.gate_proj", "model.layers.31.mlp.experts.9.gate_proj", "model.layers.31.mlp.experts.10.gate_proj", "model.layers.31.mlp.experts.11.gate_proj", "model.layers.31.mlp.experts.12.gate_proj", "model.layers.31.mlp.experts.13.gate_proj", "model.layers.31.mlp.experts.14.gate_proj", "model.layers.31.mlp.experts.15.gate_proj", "model.layers.31.mlp.experts.16.gate_proj", "model.layers.31.mlp.experts.17.gate_proj", "model.layers.31.mlp.experts.18.gate_proj", "model.layers.31.mlp.experts.19.gate_proj", "model.layers.31.mlp.experts.20.gate_proj", "model.layers.31.mlp.experts.21.gate_proj", "model.layers.31.mlp.experts.22.gate_proj", "model.layers.31.mlp.experts.23.gate_proj", "model.layers.31.mlp.experts.24.gate_proj", "model.layers.31.mlp.experts.25.gate_proj", "model.layers.31.mlp.experts.26.gate_proj", "model.layers.31.mlp.experts.27.gate_proj", "model.layers.31.mlp.experts.28.gate_proj", "model.layers.31.mlp.experts.29.gate_proj", "model.layers.31.mlp.experts.30.gate_proj", "model.layers.31.mlp.experts.31.gate_proj", "model.layers.31.mlp.experts.32.gate_proj", "model.layers.31.mlp.experts.33.gate_proj", "model.layers.31.mlp.experts.34.gate_proj", "model.layers.31.mlp.experts.35.gate_proj", "model.layers.31.mlp.experts.36.gate_proj", "model.layers.31.mlp.experts.37.gate_proj", "model.layers.31.mlp.experts.38.gate_proj", "model.layers.31.mlp.experts.39.gate_proj", "model.layers.31.mlp.experts.40.gate_proj", "model.layers.31.mlp.experts.41.gate_proj", "model.layers.31.mlp.experts.42.gate_proj", "model.layers.31.mlp.experts.43.gate_proj", "model.layers.31.mlp.experts.44.gate_proj", "model.layers.31.mlp.experts.45.gate_proj", "model.layers.31.mlp.experts.46.gate_proj", "model.layers.31.mlp.experts.47.gate_proj", "model.layers.31.mlp.experts.48.gate_proj", "model.layers.31.mlp.experts.49.gate_proj", "model.layers.31.mlp.experts.50.gate_proj", "model.layers.31.mlp.experts.51.gate_proj", "model.layers.31.mlp.experts.52.gate_proj", "model.layers.31.mlp.experts.53.gate_proj", "model.layers.31.mlp.experts.54.gate_proj", "model.layers.31.mlp.experts.55.gate_proj", "model.layers.31.mlp.experts.56.gate_proj", "model.layers.31.mlp.experts.57.gate_proj", "model.layers.31.mlp.experts.58.gate_proj", "model.layers.31.mlp.experts.59.gate_proj", "model.layers.31.mlp.experts.60.gate_proj", "model.layers.31.mlp.experts.61.gate_proj", "model.layers.31.mlp.experts.62.gate_proj", "model.layers.31.mlp.experts.63.gate_proj", "model.layers.31.mlp.experts.64.gate_proj", "model.layers.31.mlp.experts.65.gate_proj", "model.layers.31.mlp.experts.66.gate_proj", "model.layers.31.mlp.experts.67.gate_proj", "model.layers.31.mlp.experts.68.gate_proj", "model.layers.31.mlp.experts.69.gate_proj", "model.layers.31.mlp.experts.70.gate_proj", "model.layers.31.mlp.experts.71.gate_proj", "model.layers.31.mlp.experts.72.gate_proj", "model.layers.31.mlp.experts.73.gate_proj", "model.layers.31.mlp.experts.74.gate_proj", "model.layers.31.mlp.experts.75.gate_proj", "model.layers.31.mlp.experts.76.gate_proj", "model.layers.31.mlp.experts.77.gate_proj", "model.layers.31.mlp.experts.78.gate_proj", "model.layers.31.mlp.experts.79.gate_proj", "model.layers.31.mlp.experts.80.gate_proj", "model.layers.31.mlp.experts.81.gate_proj", "model.layers.31.mlp.experts.82.gate_proj", "model.layers.31.mlp.experts.83.gate_proj", "model.layers.31.mlp.experts.84.gate_proj", "model.layers.31.mlp.experts.85.gate_proj", "model.layers.31.mlp.experts.86.gate_proj", "model.layers.31.mlp.experts.87.gate_proj", "model.layers.31.mlp.experts.88.gate_proj", "model.layers.31.mlp.experts.89.gate_proj", "model.layers.31.mlp.experts.90.gate_proj", "model.layers.31.mlp.experts.91.gate_proj", "model.layers.31.mlp.experts.92.gate_proj", "model.layers.31.mlp.experts.93.gate_proj", "model.layers.31.mlp.experts.94.gate_proj", "model.layers.31.mlp.experts.95.gate_proj", "model.layers.31.mlp.experts.96.gate_proj", "model.layers.31.mlp.experts.97.gate_proj", "model.layers.31.mlp.experts.98.gate_proj", "model.layers.31.mlp.experts.99.gate_proj", "model.layers.31.mlp.experts.100.gate_proj", "model.layers.31.mlp.experts.101.gate_proj", "model.layers.31.mlp.experts.102.gate_proj", "model.layers.31.mlp.experts.103.gate_proj", "model.layers.31.mlp.experts.104.gate_proj", "model.layers.31.mlp.experts.105.gate_proj", "model.layers.31.mlp.experts.106.gate_proj", "model.layers.31.mlp.experts.107.gate_proj", "model.layers.31.mlp.experts.108.gate_proj", "model.layers.31.mlp.experts.109.gate_proj", "model.layers.31.mlp.experts.110.gate_proj", "model.layers.31.mlp.experts.111.gate_proj", "model.layers.31.mlp.experts.112.gate_proj", "model.layers.31.mlp.experts.113.gate_proj", "model.layers.31.mlp.experts.114.gate_proj", "model.layers.31.mlp.experts.115.gate_proj", "model.layers.31.mlp.experts.116.gate_proj", "model.layers.31.mlp.experts.117.gate_proj", "model.layers.31.mlp.experts.118.gate_proj", "model.layers.31.mlp.experts.119.gate_proj", "model.layers.31.mlp.experts.120.gate_proj", "model.layers.31.mlp.experts.121.gate_proj", "model.layers.31.mlp.experts.122.gate_proj", "model.layers.31.mlp.experts.123.gate_proj", "model.layers.31.mlp.experts.124.gate_proj", "model.layers.31.mlp.experts.125.gate_proj", "model.layers.31.mlp.experts.126.gate_proj", "model.layers.31.mlp.experts.127.gate_proj", "model.layers.31.mlp.experts.128.gate_proj", "model.layers.31.mlp.experts.129.gate_proj", "model.layers.31.mlp.experts.130.gate_proj", "model.layers.31.mlp.experts.131.gate_proj", "model.layers.31.mlp.experts.132.gate_proj", "model.layers.31.mlp.experts.133.gate_proj", "model.layers.31.mlp.experts.134.gate_proj", "model.layers.31.mlp.experts.135.gate_proj", "model.layers.31.mlp.experts.136.gate_proj", "model.layers.31.mlp.experts.137.gate_proj", "model.layers.31.mlp.experts.138.gate_proj", "model.layers.31.mlp.experts.139.gate_proj", "model.layers.31.mlp.experts.140.gate_proj", "model.layers.31.mlp.experts.141.gate_proj", "model.layers.31.mlp.experts.142.gate_proj", "model.layers.31.mlp.experts.143.gate_proj", "model.layers.31.mlp.experts.144.gate_proj", "model.layers.31.mlp.experts.145.gate_proj", "model.layers.31.mlp.experts.146.gate_proj", "model.layers.31.mlp.experts.147.gate_proj", "model.layers.31.mlp.experts.148.gate_proj", "model.layers.31.mlp.experts.149.gate_proj", "model.layers.31.mlp.experts.150.gate_proj", "model.layers.31.mlp.experts.151.gate_proj", "model.layers.31.mlp.experts.152.gate_proj", "model.layers.31.mlp.experts.153.gate_proj", "model.layers.31.mlp.experts.154.gate_proj", "model.layers.31.mlp.experts.155.gate_proj", "model.layers.31.mlp.experts.156.gate_proj", "model.layers.31.mlp.experts.157.gate_proj", "model.layers.31.mlp.experts.158.gate_proj", "model.layers.31.mlp.experts.159.gate_proj", "model.layers.31.mlp.experts.0.up_proj", "model.layers.31.mlp.experts.1.up_proj", "model.layers.31.mlp.experts.2.up_proj", "model.layers.31.mlp.experts.3.up_proj", "model.layers.31.mlp.experts.4.up_proj", "model.layers.31.mlp.experts.5.up_proj", "model.layers.31.mlp.experts.6.up_proj", "model.layers.31.mlp.experts.7.up_proj", "model.layers.31.mlp.experts.8.up_proj", "model.layers.31.mlp.experts.9.up_proj", "model.layers.31.mlp.experts.10.up_proj", "model.layers.31.mlp.experts.11.up_proj", "model.layers.31.mlp.experts.12.up_proj", "model.layers.31.mlp.experts.13.up_proj", "model.layers.31.mlp.experts.14.up_proj", "model.layers.31.mlp.experts.15.up_proj", "model.layers.31.mlp.experts.16.up_proj", "model.layers.31.mlp.experts.17.up_proj", "model.layers.31.mlp.experts.18.up_proj", "model.layers.31.mlp.experts.19.up_proj", "model.layers.31.mlp.experts.20.up_proj", "model.layers.31.mlp.experts.21.up_proj", "model.layers.31.mlp.experts.22.up_proj", "model.layers.31.mlp.experts.23.up_proj", "model.layers.31.mlp.experts.24.up_proj", "model.layers.31.mlp.experts.25.up_proj", "model.layers.31.mlp.experts.26.up_proj", "model.layers.31.mlp.experts.27.up_proj", "model.layers.31.mlp.experts.28.up_proj", "model.layers.31.mlp.experts.29.up_proj", "model.layers.31.mlp.experts.30.up_proj", "model.layers.31.mlp.experts.31.up_proj", "model.layers.31.mlp.experts.32.up_proj", "model.layers.31.mlp.experts.33.up_proj", "model.layers.31.mlp.experts.34.up_proj", "model.layers.31.mlp.experts.35.up_proj", "model.layers.31.mlp.experts.36.up_proj", "model.layers.31.mlp.experts.37.up_proj", "model.layers.31.mlp.experts.38.up_proj", "model.layers.31.mlp.experts.39.up_proj", "model.layers.31.mlp.experts.40.up_proj", "model.layers.31.mlp.experts.41.up_proj", "model.layers.31.mlp.experts.42.up_proj", "model.layers.31.mlp.experts.43.up_proj", "model.layers.31.mlp.experts.44.up_proj", "model.layers.31.mlp.experts.45.up_proj", "model.layers.31.mlp.experts.46.up_proj", "model.layers.31.mlp.experts.47.up_proj", "model.layers.31.mlp.experts.48.up_proj", "model.layers.31.mlp.experts.49.up_proj", "model.layers.31.mlp.experts.50.up_proj", "model.layers.31.mlp.experts.51.up_proj", "model.layers.31.mlp.experts.52.up_proj", "model.layers.31.mlp.experts.53.up_proj", "model.layers.31.mlp.experts.54.up_proj", "model.layers.31.mlp.experts.55.up_proj", "model.layers.31.mlp.experts.56.up_proj", "model.layers.31.mlp.experts.57.up_proj", "model.layers.31.mlp.experts.58.up_proj", "model.layers.31.mlp.experts.59.up_proj", "model.layers.31.mlp.experts.60.up_proj", "model.layers.31.mlp.experts.61.up_proj", "model.layers.31.mlp.experts.62.up_proj", "model.layers.31.mlp.experts.63.up_proj", "model.layers.31.mlp.experts.64.up_proj", "model.layers.31.mlp.experts.65.up_proj", "model.layers.31.mlp.experts.66.up_proj", "model.layers.31.mlp.experts.67.up_proj", "model.layers.31.mlp.experts.68.up_proj", "model.layers.31.mlp.experts.69.up_proj", "model.layers.31.mlp.experts.70.up_proj", "model.layers.31.mlp.experts.71.up_proj", "model.layers.31.mlp.experts.72.up_proj", "model.layers.31.mlp.experts.73.up_proj", "model.layers.31.mlp.experts.74.up_proj", "model.layers.31.mlp.experts.75.up_proj", "model.layers.31.mlp.experts.76.up_proj", "model.layers.31.mlp.experts.77.up_proj", "model.layers.31.mlp.experts.78.up_proj", "model.layers.31.mlp.experts.79.up_proj", "model.layers.31.mlp.experts.80.up_proj", "model.layers.31.mlp.experts.81.up_proj", "model.layers.31.mlp.experts.82.up_proj", "model.layers.31.mlp.experts.83.up_proj", "model.layers.31.mlp.experts.84.up_proj", "model.layers.31.mlp.experts.85.up_proj", "model.layers.31.mlp.experts.86.up_proj", "model.layers.31.mlp.experts.87.up_proj", "model.layers.31.mlp.experts.88.up_proj", "model.layers.31.mlp.experts.89.up_proj", "model.layers.31.mlp.experts.90.up_proj", "model.layers.31.mlp.experts.91.up_proj", "model.layers.31.mlp.experts.92.up_proj", "model.layers.31.mlp.experts.93.up_proj", "model.layers.31.mlp.experts.94.up_proj", "model.layers.31.mlp.experts.95.up_proj", "model.layers.31.mlp.experts.96.up_proj", "model.layers.31.mlp.experts.97.up_proj", "model.layers.31.mlp.experts.98.up_proj", "model.layers.31.mlp.experts.99.up_proj", "model.layers.31.mlp.experts.100.up_proj", "model.layers.31.mlp.experts.101.up_proj", "model.layers.31.mlp.experts.102.up_proj", "model.layers.31.mlp.experts.103.up_proj", "model.layers.31.mlp.experts.104.up_proj", "model.layers.31.mlp.experts.105.up_proj", "model.layers.31.mlp.experts.106.up_proj", "model.layers.31.mlp.experts.107.up_proj", "model.layers.31.mlp.experts.108.up_proj", "model.layers.31.mlp.experts.109.up_proj", "model.layers.31.mlp.experts.110.up_proj", "model.layers.31.mlp.experts.111.up_proj", "model.layers.31.mlp.experts.112.up_proj", "model.layers.31.mlp.experts.113.up_proj", "model.layers.31.mlp.experts.114.up_proj", "model.layers.31.mlp.experts.115.up_proj", "model.layers.31.mlp.experts.116.up_proj", "model.layers.31.mlp.experts.117.up_proj", "model.layers.31.mlp.experts.118.up_proj", "model.layers.31.mlp.experts.119.up_proj", "model.layers.31.mlp.experts.120.up_proj", "model.layers.31.mlp.experts.121.up_proj", "model.layers.31.mlp.experts.122.up_proj", "model.layers.31.mlp.experts.123.up_proj", "model.layers.31.mlp.experts.124.up_proj", "model.layers.31.mlp.experts.125.up_proj", "model.layers.31.mlp.experts.126.up_proj", "model.layers.31.mlp.experts.127.up_proj", "model.layers.31.mlp.experts.128.up_proj", "model.layers.31.mlp.experts.129.up_proj", "model.layers.31.mlp.experts.130.up_proj", "model.layers.31.mlp.experts.131.up_proj", "model.layers.31.mlp.experts.132.up_proj", "model.layers.31.mlp.experts.133.up_proj", "model.layers.31.mlp.experts.134.up_proj", "model.layers.31.mlp.experts.135.up_proj", "model.layers.31.mlp.experts.136.up_proj", "model.layers.31.mlp.experts.137.up_proj", "model.layers.31.mlp.experts.138.up_proj", "model.layers.31.mlp.experts.139.up_proj", "model.layers.31.mlp.experts.140.up_proj", "model.layers.31.mlp.experts.141.up_proj", "model.layers.31.mlp.experts.142.up_proj", "model.layers.31.mlp.experts.143.up_proj", "model.layers.31.mlp.experts.144.up_proj", "model.layers.31.mlp.experts.145.up_proj", "model.layers.31.mlp.experts.146.up_proj", "model.layers.31.mlp.experts.147.up_proj", "model.layers.31.mlp.experts.148.up_proj", "model.layers.31.mlp.experts.149.up_proj", "model.layers.31.mlp.experts.150.up_proj", "model.layers.31.mlp.experts.151.up_proj", "model.layers.31.mlp.experts.152.up_proj", "model.layers.31.mlp.experts.153.up_proj", "model.layers.31.mlp.experts.154.up_proj", "model.layers.31.mlp.experts.155.up_proj", "model.layers.31.mlp.experts.156.up_proj", "model.layers.31.mlp.experts.157.up_proj", "model.layers.31.mlp.experts.158.up_proj", "model.layers.31.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0037307411432266346, "dbits": 2516582400 } ] }, { "idx": 188, "layers": [ "model.layers.31.mlp.experts.0.down_proj", "model.layers.31.mlp.experts.1.down_proj", "model.layers.31.mlp.experts.2.down_proj", "model.layers.31.mlp.experts.3.down_proj", "model.layers.31.mlp.experts.4.down_proj", "model.layers.31.mlp.experts.5.down_proj", "model.layers.31.mlp.experts.6.down_proj", "model.layers.31.mlp.experts.7.down_proj", "model.layers.31.mlp.experts.8.down_proj", "model.layers.31.mlp.experts.9.down_proj", "model.layers.31.mlp.experts.10.down_proj", "model.layers.31.mlp.experts.11.down_proj", "model.layers.31.mlp.experts.12.down_proj", "model.layers.31.mlp.experts.13.down_proj", "model.layers.31.mlp.experts.14.down_proj", "model.layers.31.mlp.experts.15.down_proj", "model.layers.31.mlp.experts.16.down_proj", "model.layers.31.mlp.experts.17.down_proj", "model.layers.31.mlp.experts.18.down_proj", "model.layers.31.mlp.experts.19.down_proj", "model.layers.31.mlp.experts.20.down_proj", "model.layers.31.mlp.experts.21.down_proj", "model.layers.31.mlp.experts.22.down_proj", "model.layers.31.mlp.experts.23.down_proj", "model.layers.31.mlp.experts.24.down_proj", "model.layers.31.mlp.experts.25.down_proj", "model.layers.31.mlp.experts.26.down_proj", "model.layers.31.mlp.experts.27.down_proj", "model.layers.31.mlp.experts.28.down_proj", "model.layers.31.mlp.experts.29.down_proj", "model.layers.31.mlp.experts.30.down_proj", "model.layers.31.mlp.experts.31.down_proj", "model.layers.31.mlp.experts.32.down_proj", "model.layers.31.mlp.experts.33.down_proj", "model.layers.31.mlp.experts.34.down_proj", "model.layers.31.mlp.experts.35.down_proj", "model.layers.31.mlp.experts.36.down_proj", "model.layers.31.mlp.experts.37.down_proj", "model.layers.31.mlp.experts.38.down_proj", "model.layers.31.mlp.experts.39.down_proj", "model.layers.31.mlp.experts.40.down_proj", "model.layers.31.mlp.experts.41.down_proj", "model.layers.31.mlp.experts.42.down_proj", "model.layers.31.mlp.experts.43.down_proj", "model.layers.31.mlp.experts.44.down_proj", "model.layers.31.mlp.experts.45.down_proj", "model.layers.31.mlp.experts.46.down_proj", "model.layers.31.mlp.experts.47.down_proj", "model.layers.31.mlp.experts.48.down_proj", "model.layers.31.mlp.experts.49.down_proj", "model.layers.31.mlp.experts.50.down_proj", "model.layers.31.mlp.experts.51.down_proj", "model.layers.31.mlp.experts.52.down_proj", "model.layers.31.mlp.experts.53.down_proj", "model.layers.31.mlp.experts.54.down_proj", "model.layers.31.mlp.experts.55.down_proj", "model.layers.31.mlp.experts.56.down_proj", "model.layers.31.mlp.experts.57.down_proj", "model.layers.31.mlp.experts.58.down_proj", "model.layers.31.mlp.experts.59.down_proj", "model.layers.31.mlp.experts.60.down_proj", "model.layers.31.mlp.experts.61.down_proj", "model.layers.31.mlp.experts.62.down_proj", "model.layers.31.mlp.experts.63.down_proj", "model.layers.31.mlp.experts.64.down_proj", "model.layers.31.mlp.experts.65.down_proj", "model.layers.31.mlp.experts.66.down_proj", "model.layers.31.mlp.experts.67.down_proj", "model.layers.31.mlp.experts.68.down_proj", "model.layers.31.mlp.experts.69.down_proj", "model.layers.31.mlp.experts.70.down_proj", "model.layers.31.mlp.experts.71.down_proj", "model.layers.31.mlp.experts.72.down_proj", "model.layers.31.mlp.experts.73.down_proj", "model.layers.31.mlp.experts.74.down_proj", "model.layers.31.mlp.experts.75.down_proj", "model.layers.31.mlp.experts.76.down_proj", "model.layers.31.mlp.experts.77.down_proj", "model.layers.31.mlp.experts.78.down_proj", "model.layers.31.mlp.experts.79.down_proj", "model.layers.31.mlp.experts.80.down_proj", "model.layers.31.mlp.experts.81.down_proj", "model.layers.31.mlp.experts.82.down_proj", "model.layers.31.mlp.experts.83.down_proj", "model.layers.31.mlp.experts.84.down_proj", "model.layers.31.mlp.experts.85.down_proj", "model.layers.31.mlp.experts.86.down_proj", "model.layers.31.mlp.experts.87.down_proj", "model.layers.31.mlp.experts.88.down_proj", "model.layers.31.mlp.experts.89.down_proj", "model.layers.31.mlp.experts.90.down_proj", "model.layers.31.mlp.experts.91.down_proj", "model.layers.31.mlp.experts.92.down_proj", "model.layers.31.mlp.experts.93.down_proj", "model.layers.31.mlp.experts.94.down_proj", "model.layers.31.mlp.experts.95.down_proj", "model.layers.31.mlp.experts.96.down_proj", "model.layers.31.mlp.experts.97.down_proj", "model.layers.31.mlp.experts.98.down_proj", "model.layers.31.mlp.experts.99.down_proj", "model.layers.31.mlp.experts.100.down_proj", "model.layers.31.mlp.experts.101.down_proj", "model.layers.31.mlp.experts.102.down_proj", "model.layers.31.mlp.experts.103.down_proj", "model.layers.31.mlp.experts.104.down_proj", "model.layers.31.mlp.experts.105.down_proj", "model.layers.31.mlp.experts.106.down_proj", "model.layers.31.mlp.experts.107.down_proj", "model.layers.31.mlp.experts.108.down_proj", "model.layers.31.mlp.experts.109.down_proj", "model.layers.31.mlp.experts.110.down_proj", "model.layers.31.mlp.experts.111.down_proj", "model.layers.31.mlp.experts.112.down_proj", "model.layers.31.mlp.experts.113.down_proj", "model.layers.31.mlp.experts.114.down_proj", "model.layers.31.mlp.experts.115.down_proj", "model.layers.31.mlp.experts.116.down_proj", "model.layers.31.mlp.experts.117.down_proj", "model.layers.31.mlp.experts.118.down_proj", "model.layers.31.mlp.experts.119.down_proj", "model.layers.31.mlp.experts.120.down_proj", "model.layers.31.mlp.experts.121.down_proj", "model.layers.31.mlp.experts.122.down_proj", "model.layers.31.mlp.experts.123.down_proj", "model.layers.31.mlp.experts.124.down_proj", "model.layers.31.mlp.experts.125.down_proj", "model.layers.31.mlp.experts.126.down_proj", "model.layers.31.mlp.experts.127.down_proj", "model.layers.31.mlp.experts.128.down_proj", "model.layers.31.mlp.experts.129.down_proj", "model.layers.31.mlp.experts.130.down_proj", "model.layers.31.mlp.experts.131.down_proj", "model.layers.31.mlp.experts.132.down_proj", "model.layers.31.mlp.experts.133.down_proj", "model.layers.31.mlp.experts.134.down_proj", "model.layers.31.mlp.experts.135.down_proj", "model.layers.31.mlp.experts.136.down_proj", "model.layers.31.mlp.experts.137.down_proj", "model.layers.31.mlp.experts.138.down_proj", "model.layers.31.mlp.experts.139.down_proj", "model.layers.31.mlp.experts.140.down_proj", "model.layers.31.mlp.experts.141.down_proj", "model.layers.31.mlp.experts.142.down_proj", "model.layers.31.mlp.experts.143.down_proj", "model.layers.31.mlp.experts.144.down_proj", "model.layers.31.mlp.experts.145.down_proj", "model.layers.31.mlp.experts.146.down_proj", "model.layers.31.mlp.experts.147.down_proj", "model.layers.31.mlp.experts.148.down_proj", "model.layers.31.mlp.experts.149.down_proj", "model.layers.31.mlp.experts.150.down_proj", "model.layers.31.mlp.experts.151.down_proj", "model.layers.31.mlp.experts.152.down_proj", "model.layers.31.mlp.experts.153.down_proj", "model.layers.31.mlp.experts.154.down_proj", "model.layers.31.mlp.experts.155.down_proj", "model.layers.31.mlp.experts.156.down_proj", "model.layers.31.mlp.experts.157.down_proj", "model.layers.31.mlp.experts.158.down_proj", "model.layers.31.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0028920121490955686, "dbits": 1258291200 } ] }, { "idx": 189, "layers": [ "model.layers.32.self_attn.q_proj" ], "candidates": [ { "dkld": -0.002857087552547488, "dbits": 62914560 } ] }, { "idx": 190, "layers": [ "model.layers.32.self_attn.k_proj", "model.layers.32.self_attn.v_proj" ], "candidates": [ { "dkld": -0.003029436618089687, "dbits": 10485760 } ] }, { "idx": 191, "layers": [ "model.layers.32.self_attn.o_proj" ], "candidates": [ { "dkld": -0.006624695658683788, "dbits": 62914560 } ] }, { "idx": 192, "layers": [ "model.layers.32.mlp.shared_experts.gate_proj", "model.layers.32.mlp.shared_experts.up_proj", "model.layers.32.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.001437731832265865, "dbits": 23592960 } ] }, { "idx": 193, "layers": [ "model.layers.32.mlp.experts.0.gate_proj", "model.layers.32.mlp.experts.1.gate_proj", "model.layers.32.mlp.experts.2.gate_proj", "model.layers.32.mlp.experts.3.gate_proj", "model.layers.32.mlp.experts.4.gate_proj", "model.layers.32.mlp.experts.5.gate_proj", "model.layers.32.mlp.experts.6.gate_proj", "model.layers.32.mlp.experts.7.gate_proj", "model.layers.32.mlp.experts.8.gate_proj", "model.layers.32.mlp.experts.9.gate_proj", "model.layers.32.mlp.experts.10.gate_proj", "model.layers.32.mlp.experts.11.gate_proj", "model.layers.32.mlp.experts.12.gate_proj", "model.layers.32.mlp.experts.13.gate_proj", "model.layers.32.mlp.experts.14.gate_proj", "model.layers.32.mlp.experts.15.gate_proj", "model.layers.32.mlp.experts.16.gate_proj", "model.layers.32.mlp.experts.17.gate_proj", "model.layers.32.mlp.experts.18.gate_proj", "model.layers.32.mlp.experts.19.gate_proj", "model.layers.32.mlp.experts.20.gate_proj", "model.layers.32.mlp.experts.21.gate_proj", "model.layers.32.mlp.experts.22.gate_proj", "model.layers.32.mlp.experts.23.gate_proj", "model.layers.32.mlp.experts.24.gate_proj", "model.layers.32.mlp.experts.25.gate_proj", "model.layers.32.mlp.experts.26.gate_proj", "model.layers.32.mlp.experts.27.gate_proj", "model.layers.32.mlp.experts.28.gate_proj", "model.layers.32.mlp.experts.29.gate_proj", "model.layers.32.mlp.experts.30.gate_proj", "model.layers.32.mlp.experts.31.gate_proj", "model.layers.32.mlp.experts.32.gate_proj", "model.layers.32.mlp.experts.33.gate_proj", "model.layers.32.mlp.experts.34.gate_proj", "model.layers.32.mlp.experts.35.gate_proj", "model.layers.32.mlp.experts.36.gate_proj", "model.layers.32.mlp.experts.37.gate_proj", "model.layers.32.mlp.experts.38.gate_proj", "model.layers.32.mlp.experts.39.gate_proj", "model.layers.32.mlp.experts.40.gate_proj", "model.layers.32.mlp.experts.41.gate_proj", "model.layers.32.mlp.experts.42.gate_proj", "model.layers.32.mlp.experts.43.gate_proj", "model.layers.32.mlp.experts.44.gate_proj", "model.layers.32.mlp.experts.45.gate_proj", "model.layers.32.mlp.experts.46.gate_proj", "model.layers.32.mlp.experts.47.gate_proj", "model.layers.32.mlp.experts.48.gate_proj", "model.layers.32.mlp.experts.49.gate_proj", "model.layers.32.mlp.experts.50.gate_proj", "model.layers.32.mlp.experts.51.gate_proj", "model.layers.32.mlp.experts.52.gate_proj", "model.layers.32.mlp.experts.53.gate_proj", "model.layers.32.mlp.experts.54.gate_proj", "model.layers.32.mlp.experts.55.gate_proj", "model.layers.32.mlp.experts.56.gate_proj", "model.layers.32.mlp.experts.57.gate_proj", "model.layers.32.mlp.experts.58.gate_proj", "model.layers.32.mlp.experts.59.gate_proj", "model.layers.32.mlp.experts.60.gate_proj", "model.layers.32.mlp.experts.61.gate_proj", "model.layers.32.mlp.experts.62.gate_proj", "model.layers.32.mlp.experts.63.gate_proj", "model.layers.32.mlp.experts.64.gate_proj", "model.layers.32.mlp.experts.65.gate_proj", "model.layers.32.mlp.experts.66.gate_proj", "model.layers.32.mlp.experts.67.gate_proj", "model.layers.32.mlp.experts.68.gate_proj", "model.layers.32.mlp.experts.69.gate_proj", "model.layers.32.mlp.experts.70.gate_proj", "model.layers.32.mlp.experts.71.gate_proj", "model.layers.32.mlp.experts.72.gate_proj", "model.layers.32.mlp.experts.73.gate_proj", "model.layers.32.mlp.experts.74.gate_proj", "model.layers.32.mlp.experts.75.gate_proj", "model.layers.32.mlp.experts.76.gate_proj", "model.layers.32.mlp.experts.77.gate_proj", "model.layers.32.mlp.experts.78.gate_proj", "model.layers.32.mlp.experts.79.gate_proj", "model.layers.32.mlp.experts.80.gate_proj", "model.layers.32.mlp.experts.81.gate_proj", "model.layers.32.mlp.experts.82.gate_proj", "model.layers.32.mlp.experts.83.gate_proj", "model.layers.32.mlp.experts.84.gate_proj", "model.layers.32.mlp.experts.85.gate_proj", "model.layers.32.mlp.experts.86.gate_proj", "model.layers.32.mlp.experts.87.gate_proj", "model.layers.32.mlp.experts.88.gate_proj", "model.layers.32.mlp.experts.89.gate_proj", "model.layers.32.mlp.experts.90.gate_proj", "model.layers.32.mlp.experts.91.gate_proj", "model.layers.32.mlp.experts.92.gate_proj", "model.layers.32.mlp.experts.93.gate_proj", "model.layers.32.mlp.experts.94.gate_proj", "model.layers.32.mlp.experts.95.gate_proj", "model.layers.32.mlp.experts.96.gate_proj", "model.layers.32.mlp.experts.97.gate_proj", "model.layers.32.mlp.experts.98.gate_proj", "model.layers.32.mlp.experts.99.gate_proj", "model.layers.32.mlp.experts.100.gate_proj", "model.layers.32.mlp.experts.101.gate_proj", "model.layers.32.mlp.experts.102.gate_proj", "model.layers.32.mlp.experts.103.gate_proj", "model.layers.32.mlp.experts.104.gate_proj", "model.layers.32.mlp.experts.105.gate_proj", "model.layers.32.mlp.experts.106.gate_proj", "model.layers.32.mlp.experts.107.gate_proj", "model.layers.32.mlp.experts.108.gate_proj", "model.layers.32.mlp.experts.109.gate_proj", "model.layers.32.mlp.experts.110.gate_proj", "model.layers.32.mlp.experts.111.gate_proj", "model.layers.32.mlp.experts.112.gate_proj", "model.layers.32.mlp.experts.113.gate_proj", "model.layers.32.mlp.experts.114.gate_proj", "model.layers.32.mlp.experts.115.gate_proj", "model.layers.32.mlp.experts.116.gate_proj", "model.layers.32.mlp.experts.117.gate_proj", "model.layers.32.mlp.experts.118.gate_proj", "model.layers.32.mlp.experts.119.gate_proj", "model.layers.32.mlp.experts.120.gate_proj", "model.layers.32.mlp.experts.121.gate_proj", "model.layers.32.mlp.experts.122.gate_proj", "model.layers.32.mlp.experts.123.gate_proj", "model.layers.32.mlp.experts.124.gate_proj", "model.layers.32.mlp.experts.125.gate_proj", "model.layers.32.mlp.experts.126.gate_proj", "model.layers.32.mlp.experts.127.gate_proj", "model.layers.32.mlp.experts.128.gate_proj", "model.layers.32.mlp.experts.129.gate_proj", "model.layers.32.mlp.experts.130.gate_proj", "model.layers.32.mlp.experts.131.gate_proj", "model.layers.32.mlp.experts.132.gate_proj", "model.layers.32.mlp.experts.133.gate_proj", "model.layers.32.mlp.experts.134.gate_proj", "model.layers.32.mlp.experts.135.gate_proj", "model.layers.32.mlp.experts.136.gate_proj", "model.layers.32.mlp.experts.137.gate_proj", "model.layers.32.mlp.experts.138.gate_proj", "model.layers.32.mlp.experts.139.gate_proj", "model.layers.32.mlp.experts.140.gate_proj", "model.layers.32.mlp.experts.141.gate_proj", "model.layers.32.mlp.experts.142.gate_proj", "model.layers.32.mlp.experts.143.gate_proj", "model.layers.32.mlp.experts.144.gate_proj", "model.layers.32.mlp.experts.145.gate_proj", "model.layers.32.mlp.experts.146.gate_proj", "model.layers.32.mlp.experts.147.gate_proj", "model.layers.32.mlp.experts.148.gate_proj", "model.layers.32.mlp.experts.149.gate_proj", "model.layers.32.mlp.experts.150.gate_proj", "model.layers.32.mlp.experts.151.gate_proj", "model.layers.32.mlp.experts.152.gate_proj", "model.layers.32.mlp.experts.153.gate_proj", "model.layers.32.mlp.experts.154.gate_proj", "model.layers.32.mlp.experts.155.gate_proj", "model.layers.32.mlp.experts.156.gate_proj", "model.layers.32.mlp.experts.157.gate_proj", "model.layers.32.mlp.experts.158.gate_proj", "model.layers.32.mlp.experts.159.gate_proj", "model.layers.32.mlp.experts.0.up_proj", "model.layers.32.mlp.experts.1.up_proj", "model.layers.32.mlp.experts.2.up_proj", "model.layers.32.mlp.experts.3.up_proj", "model.layers.32.mlp.experts.4.up_proj", "model.layers.32.mlp.experts.5.up_proj", "model.layers.32.mlp.experts.6.up_proj", "model.layers.32.mlp.experts.7.up_proj", "model.layers.32.mlp.experts.8.up_proj", "model.layers.32.mlp.experts.9.up_proj", "model.layers.32.mlp.experts.10.up_proj", "model.layers.32.mlp.experts.11.up_proj", "model.layers.32.mlp.experts.12.up_proj", "model.layers.32.mlp.experts.13.up_proj", "model.layers.32.mlp.experts.14.up_proj", "model.layers.32.mlp.experts.15.up_proj", "model.layers.32.mlp.experts.16.up_proj", "model.layers.32.mlp.experts.17.up_proj", "model.layers.32.mlp.experts.18.up_proj", "model.layers.32.mlp.experts.19.up_proj", "model.layers.32.mlp.experts.20.up_proj", "model.layers.32.mlp.experts.21.up_proj", "model.layers.32.mlp.experts.22.up_proj", "model.layers.32.mlp.experts.23.up_proj", "model.layers.32.mlp.experts.24.up_proj", "model.layers.32.mlp.experts.25.up_proj", "model.layers.32.mlp.experts.26.up_proj", "model.layers.32.mlp.experts.27.up_proj", "model.layers.32.mlp.experts.28.up_proj", "model.layers.32.mlp.experts.29.up_proj", "model.layers.32.mlp.experts.30.up_proj", "model.layers.32.mlp.experts.31.up_proj", "model.layers.32.mlp.experts.32.up_proj", "model.layers.32.mlp.experts.33.up_proj", "model.layers.32.mlp.experts.34.up_proj", "model.layers.32.mlp.experts.35.up_proj", "model.layers.32.mlp.experts.36.up_proj", "model.layers.32.mlp.experts.37.up_proj", "model.layers.32.mlp.experts.38.up_proj", "model.layers.32.mlp.experts.39.up_proj", "model.layers.32.mlp.experts.40.up_proj", "model.layers.32.mlp.experts.41.up_proj", "model.layers.32.mlp.experts.42.up_proj", "model.layers.32.mlp.experts.43.up_proj", "model.layers.32.mlp.experts.44.up_proj", "model.layers.32.mlp.experts.45.up_proj", "model.layers.32.mlp.experts.46.up_proj", "model.layers.32.mlp.experts.47.up_proj", "model.layers.32.mlp.experts.48.up_proj", "model.layers.32.mlp.experts.49.up_proj", "model.layers.32.mlp.experts.50.up_proj", "model.layers.32.mlp.experts.51.up_proj", "model.layers.32.mlp.experts.52.up_proj", "model.layers.32.mlp.experts.53.up_proj", "model.layers.32.mlp.experts.54.up_proj", "model.layers.32.mlp.experts.55.up_proj", "model.layers.32.mlp.experts.56.up_proj", "model.layers.32.mlp.experts.57.up_proj", "model.layers.32.mlp.experts.58.up_proj", "model.layers.32.mlp.experts.59.up_proj", "model.layers.32.mlp.experts.60.up_proj", "model.layers.32.mlp.experts.61.up_proj", "model.layers.32.mlp.experts.62.up_proj", "model.layers.32.mlp.experts.63.up_proj", "model.layers.32.mlp.experts.64.up_proj", "model.layers.32.mlp.experts.65.up_proj", "model.layers.32.mlp.experts.66.up_proj", "model.layers.32.mlp.experts.67.up_proj", "model.layers.32.mlp.experts.68.up_proj", "model.layers.32.mlp.experts.69.up_proj", "model.layers.32.mlp.experts.70.up_proj", "model.layers.32.mlp.experts.71.up_proj", "model.layers.32.mlp.experts.72.up_proj", "model.layers.32.mlp.experts.73.up_proj", "model.layers.32.mlp.experts.74.up_proj", "model.layers.32.mlp.experts.75.up_proj", "model.layers.32.mlp.experts.76.up_proj", "model.layers.32.mlp.experts.77.up_proj", "model.layers.32.mlp.experts.78.up_proj", "model.layers.32.mlp.experts.79.up_proj", "model.layers.32.mlp.experts.80.up_proj", "model.layers.32.mlp.experts.81.up_proj", "model.layers.32.mlp.experts.82.up_proj", "model.layers.32.mlp.experts.83.up_proj", "model.layers.32.mlp.experts.84.up_proj", "model.layers.32.mlp.experts.85.up_proj", "model.layers.32.mlp.experts.86.up_proj", "model.layers.32.mlp.experts.87.up_proj", "model.layers.32.mlp.experts.88.up_proj", "model.layers.32.mlp.experts.89.up_proj", "model.layers.32.mlp.experts.90.up_proj", "model.layers.32.mlp.experts.91.up_proj", "model.layers.32.mlp.experts.92.up_proj", "model.layers.32.mlp.experts.93.up_proj", "model.layers.32.mlp.experts.94.up_proj", "model.layers.32.mlp.experts.95.up_proj", "model.layers.32.mlp.experts.96.up_proj", "model.layers.32.mlp.experts.97.up_proj", "model.layers.32.mlp.experts.98.up_proj", "model.layers.32.mlp.experts.99.up_proj", "model.layers.32.mlp.experts.100.up_proj", "model.layers.32.mlp.experts.101.up_proj", "model.layers.32.mlp.experts.102.up_proj", "model.layers.32.mlp.experts.103.up_proj", "model.layers.32.mlp.experts.104.up_proj", "model.layers.32.mlp.experts.105.up_proj", "model.layers.32.mlp.experts.106.up_proj", "model.layers.32.mlp.experts.107.up_proj", "model.layers.32.mlp.experts.108.up_proj", "model.layers.32.mlp.experts.109.up_proj", "model.layers.32.mlp.experts.110.up_proj", "model.layers.32.mlp.experts.111.up_proj", "model.layers.32.mlp.experts.112.up_proj", "model.layers.32.mlp.experts.113.up_proj", "model.layers.32.mlp.experts.114.up_proj", "model.layers.32.mlp.experts.115.up_proj", "model.layers.32.mlp.experts.116.up_proj", "model.layers.32.mlp.experts.117.up_proj", "model.layers.32.mlp.experts.118.up_proj", "model.layers.32.mlp.experts.119.up_proj", "model.layers.32.mlp.experts.120.up_proj", "model.layers.32.mlp.experts.121.up_proj", "model.layers.32.mlp.experts.122.up_proj", "model.layers.32.mlp.experts.123.up_proj", "model.layers.32.mlp.experts.124.up_proj", "model.layers.32.mlp.experts.125.up_proj", "model.layers.32.mlp.experts.126.up_proj", "model.layers.32.mlp.experts.127.up_proj", "model.layers.32.mlp.experts.128.up_proj", "model.layers.32.mlp.experts.129.up_proj", "model.layers.32.mlp.experts.130.up_proj", "model.layers.32.mlp.experts.131.up_proj", "model.layers.32.mlp.experts.132.up_proj", "model.layers.32.mlp.experts.133.up_proj", "model.layers.32.mlp.experts.134.up_proj", "model.layers.32.mlp.experts.135.up_proj", "model.layers.32.mlp.experts.136.up_proj", "model.layers.32.mlp.experts.137.up_proj", "model.layers.32.mlp.experts.138.up_proj", "model.layers.32.mlp.experts.139.up_proj", "model.layers.32.mlp.experts.140.up_proj", "model.layers.32.mlp.experts.141.up_proj", "model.layers.32.mlp.experts.142.up_proj", "model.layers.32.mlp.experts.143.up_proj", "model.layers.32.mlp.experts.144.up_proj", "model.layers.32.mlp.experts.145.up_proj", "model.layers.32.mlp.experts.146.up_proj", "model.layers.32.mlp.experts.147.up_proj", "model.layers.32.mlp.experts.148.up_proj", "model.layers.32.mlp.experts.149.up_proj", "model.layers.32.mlp.experts.150.up_proj", "model.layers.32.mlp.experts.151.up_proj", "model.layers.32.mlp.experts.152.up_proj", "model.layers.32.mlp.experts.153.up_proj", "model.layers.32.mlp.experts.154.up_proj", "model.layers.32.mlp.experts.155.up_proj", "model.layers.32.mlp.experts.156.up_proj", "model.layers.32.mlp.experts.157.up_proj", "model.layers.32.mlp.experts.158.up_proj", "model.layers.32.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0011544272303581349, "dbits": 2516582400 } ] }, { "idx": 194, "layers": [ "model.layers.32.mlp.experts.0.down_proj", "model.layers.32.mlp.experts.1.down_proj", "model.layers.32.mlp.experts.2.down_proj", "model.layers.32.mlp.experts.3.down_proj", "model.layers.32.mlp.experts.4.down_proj", "model.layers.32.mlp.experts.5.down_proj", "model.layers.32.mlp.experts.6.down_proj", "model.layers.32.mlp.experts.7.down_proj", "model.layers.32.mlp.experts.8.down_proj", "model.layers.32.mlp.experts.9.down_proj", "model.layers.32.mlp.experts.10.down_proj", "model.layers.32.mlp.experts.11.down_proj", "model.layers.32.mlp.experts.12.down_proj", "model.layers.32.mlp.experts.13.down_proj", "model.layers.32.mlp.experts.14.down_proj", "model.layers.32.mlp.experts.15.down_proj", "model.layers.32.mlp.experts.16.down_proj", "model.layers.32.mlp.experts.17.down_proj", "model.layers.32.mlp.experts.18.down_proj", "model.layers.32.mlp.experts.19.down_proj", "model.layers.32.mlp.experts.20.down_proj", "model.layers.32.mlp.experts.21.down_proj", "model.layers.32.mlp.experts.22.down_proj", "model.layers.32.mlp.experts.23.down_proj", "model.layers.32.mlp.experts.24.down_proj", "model.layers.32.mlp.experts.25.down_proj", "model.layers.32.mlp.experts.26.down_proj", "model.layers.32.mlp.experts.27.down_proj", "model.layers.32.mlp.experts.28.down_proj", "model.layers.32.mlp.experts.29.down_proj", "model.layers.32.mlp.experts.30.down_proj", "model.layers.32.mlp.experts.31.down_proj", "model.layers.32.mlp.experts.32.down_proj", "model.layers.32.mlp.experts.33.down_proj", "model.layers.32.mlp.experts.34.down_proj", "model.layers.32.mlp.experts.35.down_proj", "model.layers.32.mlp.experts.36.down_proj", "model.layers.32.mlp.experts.37.down_proj", "model.layers.32.mlp.experts.38.down_proj", "model.layers.32.mlp.experts.39.down_proj", "model.layers.32.mlp.experts.40.down_proj", "model.layers.32.mlp.experts.41.down_proj", "model.layers.32.mlp.experts.42.down_proj", "model.layers.32.mlp.experts.43.down_proj", "model.layers.32.mlp.experts.44.down_proj", "model.layers.32.mlp.experts.45.down_proj", "model.layers.32.mlp.experts.46.down_proj", "model.layers.32.mlp.experts.47.down_proj", "model.layers.32.mlp.experts.48.down_proj", "model.layers.32.mlp.experts.49.down_proj", "model.layers.32.mlp.experts.50.down_proj", "model.layers.32.mlp.experts.51.down_proj", "model.layers.32.mlp.experts.52.down_proj", "model.layers.32.mlp.experts.53.down_proj", "model.layers.32.mlp.experts.54.down_proj", "model.layers.32.mlp.experts.55.down_proj", "model.layers.32.mlp.experts.56.down_proj", "model.layers.32.mlp.experts.57.down_proj", "model.layers.32.mlp.experts.58.down_proj", "model.layers.32.mlp.experts.59.down_proj", "model.layers.32.mlp.experts.60.down_proj", "model.layers.32.mlp.experts.61.down_proj", "model.layers.32.mlp.experts.62.down_proj", "model.layers.32.mlp.experts.63.down_proj", "model.layers.32.mlp.experts.64.down_proj", "model.layers.32.mlp.experts.65.down_proj", "model.layers.32.mlp.experts.66.down_proj", "model.layers.32.mlp.experts.67.down_proj", "model.layers.32.mlp.experts.68.down_proj", "model.layers.32.mlp.experts.69.down_proj", "model.layers.32.mlp.experts.70.down_proj", "model.layers.32.mlp.experts.71.down_proj", "model.layers.32.mlp.experts.72.down_proj", "model.layers.32.mlp.experts.73.down_proj", "model.layers.32.mlp.experts.74.down_proj", "model.layers.32.mlp.experts.75.down_proj", "model.layers.32.mlp.experts.76.down_proj", "model.layers.32.mlp.experts.77.down_proj", "model.layers.32.mlp.experts.78.down_proj", "model.layers.32.mlp.experts.79.down_proj", "model.layers.32.mlp.experts.80.down_proj", "model.layers.32.mlp.experts.81.down_proj", "model.layers.32.mlp.experts.82.down_proj", "model.layers.32.mlp.experts.83.down_proj", "model.layers.32.mlp.experts.84.down_proj", "model.layers.32.mlp.experts.85.down_proj", "model.layers.32.mlp.experts.86.down_proj", "model.layers.32.mlp.experts.87.down_proj", "model.layers.32.mlp.experts.88.down_proj", "model.layers.32.mlp.experts.89.down_proj", "model.layers.32.mlp.experts.90.down_proj", "model.layers.32.mlp.experts.91.down_proj", "model.layers.32.mlp.experts.92.down_proj", "model.layers.32.mlp.experts.93.down_proj", "model.layers.32.mlp.experts.94.down_proj", "model.layers.32.mlp.experts.95.down_proj", "model.layers.32.mlp.experts.96.down_proj", "model.layers.32.mlp.experts.97.down_proj", "model.layers.32.mlp.experts.98.down_proj", "model.layers.32.mlp.experts.99.down_proj", "model.layers.32.mlp.experts.100.down_proj", "model.layers.32.mlp.experts.101.down_proj", "model.layers.32.mlp.experts.102.down_proj", "model.layers.32.mlp.experts.103.down_proj", "model.layers.32.mlp.experts.104.down_proj", "model.layers.32.mlp.experts.105.down_proj", "model.layers.32.mlp.experts.106.down_proj", "model.layers.32.mlp.experts.107.down_proj", "model.layers.32.mlp.experts.108.down_proj", "model.layers.32.mlp.experts.109.down_proj", "model.layers.32.mlp.experts.110.down_proj", "model.layers.32.mlp.experts.111.down_proj", "model.layers.32.mlp.experts.112.down_proj", "model.layers.32.mlp.experts.113.down_proj", "model.layers.32.mlp.experts.114.down_proj", "model.layers.32.mlp.experts.115.down_proj", "model.layers.32.mlp.experts.116.down_proj", "model.layers.32.mlp.experts.117.down_proj", "model.layers.32.mlp.experts.118.down_proj", "model.layers.32.mlp.experts.119.down_proj", "model.layers.32.mlp.experts.120.down_proj", "model.layers.32.mlp.experts.121.down_proj", "model.layers.32.mlp.experts.122.down_proj", "model.layers.32.mlp.experts.123.down_proj", "model.layers.32.mlp.experts.124.down_proj", "model.layers.32.mlp.experts.125.down_proj", "model.layers.32.mlp.experts.126.down_proj", "model.layers.32.mlp.experts.127.down_proj", "model.layers.32.mlp.experts.128.down_proj", "model.layers.32.mlp.experts.129.down_proj", "model.layers.32.mlp.experts.130.down_proj", "model.layers.32.mlp.experts.131.down_proj", "model.layers.32.mlp.experts.132.down_proj", "model.layers.32.mlp.experts.133.down_proj", "model.layers.32.mlp.experts.134.down_proj", "model.layers.32.mlp.experts.135.down_proj", "model.layers.32.mlp.experts.136.down_proj", "model.layers.32.mlp.experts.137.down_proj", "model.layers.32.mlp.experts.138.down_proj", "model.layers.32.mlp.experts.139.down_proj", "model.layers.32.mlp.experts.140.down_proj", "model.layers.32.mlp.experts.141.down_proj", "model.layers.32.mlp.experts.142.down_proj", "model.layers.32.mlp.experts.143.down_proj", "model.layers.32.mlp.experts.144.down_proj", "model.layers.32.mlp.experts.145.down_proj", "model.layers.32.mlp.experts.146.down_proj", "model.layers.32.mlp.experts.147.down_proj", "model.layers.32.mlp.experts.148.down_proj", "model.layers.32.mlp.experts.149.down_proj", "model.layers.32.mlp.experts.150.down_proj", "model.layers.32.mlp.experts.151.down_proj", "model.layers.32.mlp.experts.152.down_proj", "model.layers.32.mlp.experts.153.down_proj", "model.layers.32.mlp.experts.154.down_proj", "model.layers.32.mlp.experts.155.down_proj", "model.layers.32.mlp.experts.156.down_proj", "model.layers.32.mlp.experts.157.down_proj", "model.layers.32.mlp.experts.158.down_proj", "model.layers.32.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002017778903245948, "dbits": 1258291200 } ] }, { "idx": 195, "layers": [ "model.layers.33.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0027065210044384114, "dbits": 62914560 } ] }, { "idx": 196, "layers": [ "model.layers.33.self_attn.k_proj", "model.layers.33.self_attn.v_proj" ], "candidates": [ { "dkld": -0.001738421618938446, "dbits": 10485760 } ] }, { "idx": 197, "layers": [ "model.layers.33.self_attn.o_proj" ], "candidates": [ { "dkld": -0.005802110582590114, "dbits": 62914560 } ] }, { "idx": 198, "layers": [ "model.layers.33.mlp.shared_experts.gate_proj", "model.layers.33.mlp.shared_experts.up_proj", "model.layers.33.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.002579060196876537, "dbits": 23592960 } ] }, { "idx": 199, "layers": [ "model.layers.33.mlp.experts.0.gate_proj", "model.layers.33.mlp.experts.1.gate_proj", "model.layers.33.mlp.experts.2.gate_proj", "model.layers.33.mlp.experts.3.gate_proj", "model.layers.33.mlp.experts.4.gate_proj", "model.layers.33.mlp.experts.5.gate_proj", "model.layers.33.mlp.experts.6.gate_proj", "model.layers.33.mlp.experts.7.gate_proj", "model.layers.33.mlp.experts.8.gate_proj", "model.layers.33.mlp.experts.9.gate_proj", "model.layers.33.mlp.experts.10.gate_proj", "model.layers.33.mlp.experts.11.gate_proj", "model.layers.33.mlp.experts.12.gate_proj", "model.layers.33.mlp.experts.13.gate_proj", "model.layers.33.mlp.experts.14.gate_proj", "model.layers.33.mlp.experts.15.gate_proj", "model.layers.33.mlp.experts.16.gate_proj", "model.layers.33.mlp.experts.17.gate_proj", "model.layers.33.mlp.experts.18.gate_proj", "model.layers.33.mlp.experts.19.gate_proj", "model.layers.33.mlp.experts.20.gate_proj", "model.layers.33.mlp.experts.21.gate_proj", "model.layers.33.mlp.experts.22.gate_proj", "model.layers.33.mlp.experts.23.gate_proj", "model.layers.33.mlp.experts.24.gate_proj", "model.layers.33.mlp.experts.25.gate_proj", "model.layers.33.mlp.experts.26.gate_proj", "model.layers.33.mlp.experts.27.gate_proj", "model.layers.33.mlp.experts.28.gate_proj", "model.layers.33.mlp.experts.29.gate_proj", "model.layers.33.mlp.experts.30.gate_proj", "model.layers.33.mlp.experts.31.gate_proj", "model.layers.33.mlp.experts.32.gate_proj", "model.layers.33.mlp.experts.33.gate_proj", "model.layers.33.mlp.experts.34.gate_proj", "model.layers.33.mlp.experts.35.gate_proj", "model.layers.33.mlp.experts.36.gate_proj", "model.layers.33.mlp.experts.37.gate_proj", "model.layers.33.mlp.experts.38.gate_proj", "model.layers.33.mlp.experts.39.gate_proj", "model.layers.33.mlp.experts.40.gate_proj", "model.layers.33.mlp.experts.41.gate_proj", "model.layers.33.mlp.experts.42.gate_proj", "model.layers.33.mlp.experts.43.gate_proj", "model.layers.33.mlp.experts.44.gate_proj", "model.layers.33.mlp.experts.45.gate_proj", "model.layers.33.mlp.experts.46.gate_proj", "model.layers.33.mlp.experts.47.gate_proj", "model.layers.33.mlp.experts.48.gate_proj", "model.layers.33.mlp.experts.49.gate_proj", "model.layers.33.mlp.experts.50.gate_proj", "model.layers.33.mlp.experts.51.gate_proj", "model.layers.33.mlp.experts.52.gate_proj", "model.layers.33.mlp.experts.53.gate_proj", "model.layers.33.mlp.experts.54.gate_proj", "model.layers.33.mlp.experts.55.gate_proj", "model.layers.33.mlp.experts.56.gate_proj", "model.layers.33.mlp.experts.57.gate_proj", "model.layers.33.mlp.experts.58.gate_proj", "model.layers.33.mlp.experts.59.gate_proj", "model.layers.33.mlp.experts.60.gate_proj", "model.layers.33.mlp.experts.61.gate_proj", "model.layers.33.mlp.experts.62.gate_proj", "model.layers.33.mlp.experts.63.gate_proj", "model.layers.33.mlp.experts.64.gate_proj", "model.layers.33.mlp.experts.65.gate_proj", "model.layers.33.mlp.experts.66.gate_proj", "model.layers.33.mlp.experts.67.gate_proj", "model.layers.33.mlp.experts.68.gate_proj", "model.layers.33.mlp.experts.69.gate_proj", "model.layers.33.mlp.experts.70.gate_proj", "model.layers.33.mlp.experts.71.gate_proj", "model.layers.33.mlp.experts.72.gate_proj", "model.layers.33.mlp.experts.73.gate_proj", "model.layers.33.mlp.experts.74.gate_proj", "model.layers.33.mlp.experts.75.gate_proj", "model.layers.33.mlp.experts.76.gate_proj", "model.layers.33.mlp.experts.77.gate_proj", "model.layers.33.mlp.experts.78.gate_proj", "model.layers.33.mlp.experts.79.gate_proj", "model.layers.33.mlp.experts.80.gate_proj", "model.layers.33.mlp.experts.81.gate_proj", "model.layers.33.mlp.experts.82.gate_proj", "model.layers.33.mlp.experts.83.gate_proj", "model.layers.33.mlp.experts.84.gate_proj", "model.layers.33.mlp.experts.85.gate_proj", "model.layers.33.mlp.experts.86.gate_proj", "model.layers.33.mlp.experts.87.gate_proj", "model.layers.33.mlp.experts.88.gate_proj", "model.layers.33.mlp.experts.89.gate_proj", "model.layers.33.mlp.experts.90.gate_proj", "model.layers.33.mlp.experts.91.gate_proj", "model.layers.33.mlp.experts.92.gate_proj", "model.layers.33.mlp.experts.93.gate_proj", "model.layers.33.mlp.experts.94.gate_proj", "model.layers.33.mlp.experts.95.gate_proj", "model.layers.33.mlp.experts.96.gate_proj", "model.layers.33.mlp.experts.97.gate_proj", "model.layers.33.mlp.experts.98.gate_proj", "model.layers.33.mlp.experts.99.gate_proj", "model.layers.33.mlp.experts.100.gate_proj", "model.layers.33.mlp.experts.101.gate_proj", "model.layers.33.mlp.experts.102.gate_proj", "model.layers.33.mlp.experts.103.gate_proj", "model.layers.33.mlp.experts.104.gate_proj", "model.layers.33.mlp.experts.105.gate_proj", "model.layers.33.mlp.experts.106.gate_proj", "model.layers.33.mlp.experts.107.gate_proj", "model.layers.33.mlp.experts.108.gate_proj", "model.layers.33.mlp.experts.109.gate_proj", "model.layers.33.mlp.experts.110.gate_proj", "model.layers.33.mlp.experts.111.gate_proj", "model.layers.33.mlp.experts.112.gate_proj", "model.layers.33.mlp.experts.113.gate_proj", "model.layers.33.mlp.experts.114.gate_proj", "model.layers.33.mlp.experts.115.gate_proj", "model.layers.33.mlp.experts.116.gate_proj", "model.layers.33.mlp.experts.117.gate_proj", "model.layers.33.mlp.experts.118.gate_proj", "model.layers.33.mlp.experts.119.gate_proj", "model.layers.33.mlp.experts.120.gate_proj", "model.layers.33.mlp.experts.121.gate_proj", "model.layers.33.mlp.experts.122.gate_proj", "model.layers.33.mlp.experts.123.gate_proj", "model.layers.33.mlp.experts.124.gate_proj", "model.layers.33.mlp.experts.125.gate_proj", "model.layers.33.mlp.experts.126.gate_proj", "model.layers.33.mlp.experts.127.gate_proj", "model.layers.33.mlp.experts.128.gate_proj", "model.layers.33.mlp.experts.129.gate_proj", "model.layers.33.mlp.experts.130.gate_proj", "model.layers.33.mlp.experts.131.gate_proj", "model.layers.33.mlp.experts.132.gate_proj", "model.layers.33.mlp.experts.133.gate_proj", "model.layers.33.mlp.experts.134.gate_proj", "model.layers.33.mlp.experts.135.gate_proj", "model.layers.33.mlp.experts.136.gate_proj", "model.layers.33.mlp.experts.137.gate_proj", "model.layers.33.mlp.experts.138.gate_proj", "model.layers.33.mlp.experts.139.gate_proj", "model.layers.33.mlp.experts.140.gate_proj", "model.layers.33.mlp.experts.141.gate_proj", "model.layers.33.mlp.experts.142.gate_proj", "model.layers.33.mlp.experts.143.gate_proj", "model.layers.33.mlp.experts.144.gate_proj", "model.layers.33.mlp.experts.145.gate_proj", "model.layers.33.mlp.experts.146.gate_proj", "model.layers.33.mlp.experts.147.gate_proj", "model.layers.33.mlp.experts.148.gate_proj", "model.layers.33.mlp.experts.149.gate_proj", "model.layers.33.mlp.experts.150.gate_proj", "model.layers.33.mlp.experts.151.gate_proj", "model.layers.33.mlp.experts.152.gate_proj", "model.layers.33.mlp.experts.153.gate_proj", "model.layers.33.mlp.experts.154.gate_proj", "model.layers.33.mlp.experts.155.gate_proj", "model.layers.33.mlp.experts.156.gate_proj", "model.layers.33.mlp.experts.157.gate_proj", "model.layers.33.mlp.experts.158.gate_proj", "model.layers.33.mlp.experts.159.gate_proj", "model.layers.33.mlp.experts.0.up_proj", "model.layers.33.mlp.experts.1.up_proj", "model.layers.33.mlp.experts.2.up_proj", "model.layers.33.mlp.experts.3.up_proj", "model.layers.33.mlp.experts.4.up_proj", "model.layers.33.mlp.experts.5.up_proj", "model.layers.33.mlp.experts.6.up_proj", "model.layers.33.mlp.experts.7.up_proj", "model.layers.33.mlp.experts.8.up_proj", "model.layers.33.mlp.experts.9.up_proj", "model.layers.33.mlp.experts.10.up_proj", "model.layers.33.mlp.experts.11.up_proj", "model.layers.33.mlp.experts.12.up_proj", "model.layers.33.mlp.experts.13.up_proj", "model.layers.33.mlp.experts.14.up_proj", "model.layers.33.mlp.experts.15.up_proj", "model.layers.33.mlp.experts.16.up_proj", "model.layers.33.mlp.experts.17.up_proj", "model.layers.33.mlp.experts.18.up_proj", "model.layers.33.mlp.experts.19.up_proj", "model.layers.33.mlp.experts.20.up_proj", "model.layers.33.mlp.experts.21.up_proj", "model.layers.33.mlp.experts.22.up_proj", "model.layers.33.mlp.experts.23.up_proj", "model.layers.33.mlp.experts.24.up_proj", "model.layers.33.mlp.experts.25.up_proj", "model.layers.33.mlp.experts.26.up_proj", "model.layers.33.mlp.experts.27.up_proj", "model.layers.33.mlp.experts.28.up_proj", "model.layers.33.mlp.experts.29.up_proj", "model.layers.33.mlp.experts.30.up_proj", "model.layers.33.mlp.experts.31.up_proj", "model.layers.33.mlp.experts.32.up_proj", "model.layers.33.mlp.experts.33.up_proj", "model.layers.33.mlp.experts.34.up_proj", "model.layers.33.mlp.experts.35.up_proj", "model.layers.33.mlp.experts.36.up_proj", "model.layers.33.mlp.experts.37.up_proj", "model.layers.33.mlp.experts.38.up_proj", "model.layers.33.mlp.experts.39.up_proj", "model.layers.33.mlp.experts.40.up_proj", "model.layers.33.mlp.experts.41.up_proj", "model.layers.33.mlp.experts.42.up_proj", "model.layers.33.mlp.experts.43.up_proj", "model.layers.33.mlp.experts.44.up_proj", "model.layers.33.mlp.experts.45.up_proj", "model.layers.33.mlp.experts.46.up_proj", "model.layers.33.mlp.experts.47.up_proj", "model.layers.33.mlp.experts.48.up_proj", "model.layers.33.mlp.experts.49.up_proj", "model.layers.33.mlp.experts.50.up_proj", "model.layers.33.mlp.experts.51.up_proj", "model.layers.33.mlp.experts.52.up_proj", "model.layers.33.mlp.experts.53.up_proj", "model.layers.33.mlp.experts.54.up_proj", "model.layers.33.mlp.experts.55.up_proj", "model.layers.33.mlp.experts.56.up_proj", "model.layers.33.mlp.experts.57.up_proj", "model.layers.33.mlp.experts.58.up_proj", "model.layers.33.mlp.experts.59.up_proj", "model.layers.33.mlp.experts.60.up_proj", "model.layers.33.mlp.experts.61.up_proj", "model.layers.33.mlp.experts.62.up_proj", "model.layers.33.mlp.experts.63.up_proj", "model.layers.33.mlp.experts.64.up_proj", "model.layers.33.mlp.experts.65.up_proj", "model.layers.33.mlp.experts.66.up_proj", "model.layers.33.mlp.experts.67.up_proj", "model.layers.33.mlp.experts.68.up_proj", "model.layers.33.mlp.experts.69.up_proj", "model.layers.33.mlp.experts.70.up_proj", "model.layers.33.mlp.experts.71.up_proj", "model.layers.33.mlp.experts.72.up_proj", "model.layers.33.mlp.experts.73.up_proj", "model.layers.33.mlp.experts.74.up_proj", "model.layers.33.mlp.experts.75.up_proj", "model.layers.33.mlp.experts.76.up_proj", "model.layers.33.mlp.experts.77.up_proj", "model.layers.33.mlp.experts.78.up_proj", "model.layers.33.mlp.experts.79.up_proj", "model.layers.33.mlp.experts.80.up_proj", "model.layers.33.mlp.experts.81.up_proj", "model.layers.33.mlp.experts.82.up_proj", "model.layers.33.mlp.experts.83.up_proj", "model.layers.33.mlp.experts.84.up_proj", "model.layers.33.mlp.experts.85.up_proj", "model.layers.33.mlp.experts.86.up_proj", "model.layers.33.mlp.experts.87.up_proj", "model.layers.33.mlp.experts.88.up_proj", "model.layers.33.mlp.experts.89.up_proj", "model.layers.33.mlp.experts.90.up_proj", "model.layers.33.mlp.experts.91.up_proj", "model.layers.33.mlp.experts.92.up_proj", "model.layers.33.mlp.experts.93.up_proj", "model.layers.33.mlp.experts.94.up_proj", "model.layers.33.mlp.experts.95.up_proj", "model.layers.33.mlp.experts.96.up_proj", "model.layers.33.mlp.experts.97.up_proj", "model.layers.33.mlp.experts.98.up_proj", "model.layers.33.mlp.experts.99.up_proj", "model.layers.33.mlp.experts.100.up_proj", "model.layers.33.mlp.experts.101.up_proj", "model.layers.33.mlp.experts.102.up_proj", "model.layers.33.mlp.experts.103.up_proj", "model.layers.33.mlp.experts.104.up_proj", "model.layers.33.mlp.experts.105.up_proj", "model.layers.33.mlp.experts.106.up_proj", "model.layers.33.mlp.experts.107.up_proj", "model.layers.33.mlp.experts.108.up_proj", "model.layers.33.mlp.experts.109.up_proj", "model.layers.33.mlp.experts.110.up_proj", "model.layers.33.mlp.experts.111.up_proj", "model.layers.33.mlp.experts.112.up_proj", "model.layers.33.mlp.experts.113.up_proj", "model.layers.33.mlp.experts.114.up_proj", "model.layers.33.mlp.experts.115.up_proj", "model.layers.33.mlp.experts.116.up_proj", "model.layers.33.mlp.experts.117.up_proj", "model.layers.33.mlp.experts.118.up_proj", "model.layers.33.mlp.experts.119.up_proj", "model.layers.33.mlp.experts.120.up_proj", "model.layers.33.mlp.experts.121.up_proj", "model.layers.33.mlp.experts.122.up_proj", "model.layers.33.mlp.experts.123.up_proj", "model.layers.33.mlp.experts.124.up_proj", "model.layers.33.mlp.experts.125.up_proj", "model.layers.33.mlp.experts.126.up_proj", "model.layers.33.mlp.experts.127.up_proj", "model.layers.33.mlp.experts.128.up_proj", "model.layers.33.mlp.experts.129.up_proj", "model.layers.33.mlp.experts.130.up_proj", "model.layers.33.mlp.experts.131.up_proj", "model.layers.33.mlp.experts.132.up_proj", "model.layers.33.mlp.experts.133.up_proj", "model.layers.33.mlp.experts.134.up_proj", "model.layers.33.mlp.experts.135.up_proj", "model.layers.33.mlp.experts.136.up_proj", "model.layers.33.mlp.experts.137.up_proj", "model.layers.33.mlp.experts.138.up_proj", "model.layers.33.mlp.experts.139.up_proj", "model.layers.33.mlp.experts.140.up_proj", "model.layers.33.mlp.experts.141.up_proj", "model.layers.33.mlp.experts.142.up_proj", "model.layers.33.mlp.experts.143.up_proj", "model.layers.33.mlp.experts.144.up_proj", "model.layers.33.mlp.experts.145.up_proj", "model.layers.33.mlp.experts.146.up_proj", "model.layers.33.mlp.experts.147.up_proj", "model.layers.33.mlp.experts.148.up_proj", "model.layers.33.mlp.experts.149.up_proj", "model.layers.33.mlp.experts.150.up_proj", "model.layers.33.mlp.experts.151.up_proj", "model.layers.33.mlp.experts.152.up_proj", "model.layers.33.mlp.experts.153.up_proj", "model.layers.33.mlp.experts.154.up_proj", "model.layers.33.mlp.experts.155.up_proj", "model.layers.33.mlp.experts.156.up_proj", "model.layers.33.mlp.experts.157.up_proj", "model.layers.33.mlp.experts.158.up_proj", "model.layers.33.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0017708942294120678, "dbits": 2516582400 } ] }, { "idx": 200, "layers": [ "model.layers.33.mlp.experts.0.down_proj", "model.layers.33.mlp.experts.1.down_proj", "model.layers.33.mlp.experts.2.down_proj", "model.layers.33.mlp.experts.3.down_proj", "model.layers.33.mlp.experts.4.down_proj", "model.layers.33.mlp.experts.5.down_proj", "model.layers.33.mlp.experts.6.down_proj", "model.layers.33.mlp.experts.7.down_proj", "model.layers.33.mlp.experts.8.down_proj", "model.layers.33.mlp.experts.9.down_proj", "model.layers.33.mlp.experts.10.down_proj", "model.layers.33.mlp.experts.11.down_proj", "model.layers.33.mlp.experts.12.down_proj", "model.layers.33.mlp.experts.13.down_proj", "model.layers.33.mlp.experts.14.down_proj", "model.layers.33.mlp.experts.15.down_proj", "model.layers.33.mlp.experts.16.down_proj", "model.layers.33.mlp.experts.17.down_proj", "model.layers.33.mlp.experts.18.down_proj", "model.layers.33.mlp.experts.19.down_proj", "model.layers.33.mlp.experts.20.down_proj", "model.layers.33.mlp.experts.21.down_proj", "model.layers.33.mlp.experts.22.down_proj", "model.layers.33.mlp.experts.23.down_proj", "model.layers.33.mlp.experts.24.down_proj", "model.layers.33.mlp.experts.25.down_proj", "model.layers.33.mlp.experts.26.down_proj", "model.layers.33.mlp.experts.27.down_proj", "model.layers.33.mlp.experts.28.down_proj", "model.layers.33.mlp.experts.29.down_proj", "model.layers.33.mlp.experts.30.down_proj", "model.layers.33.mlp.experts.31.down_proj", "model.layers.33.mlp.experts.32.down_proj", "model.layers.33.mlp.experts.33.down_proj", "model.layers.33.mlp.experts.34.down_proj", "model.layers.33.mlp.experts.35.down_proj", "model.layers.33.mlp.experts.36.down_proj", "model.layers.33.mlp.experts.37.down_proj", "model.layers.33.mlp.experts.38.down_proj", "model.layers.33.mlp.experts.39.down_proj", "model.layers.33.mlp.experts.40.down_proj", "model.layers.33.mlp.experts.41.down_proj", "model.layers.33.mlp.experts.42.down_proj", "model.layers.33.mlp.experts.43.down_proj", "model.layers.33.mlp.experts.44.down_proj", "model.layers.33.mlp.experts.45.down_proj", "model.layers.33.mlp.experts.46.down_proj", "model.layers.33.mlp.experts.47.down_proj", "model.layers.33.mlp.experts.48.down_proj", "model.layers.33.mlp.experts.49.down_proj", "model.layers.33.mlp.experts.50.down_proj", "model.layers.33.mlp.experts.51.down_proj", "model.layers.33.mlp.experts.52.down_proj", "model.layers.33.mlp.experts.53.down_proj", "model.layers.33.mlp.experts.54.down_proj", "model.layers.33.mlp.experts.55.down_proj", "model.layers.33.mlp.experts.56.down_proj", "model.layers.33.mlp.experts.57.down_proj", "model.layers.33.mlp.experts.58.down_proj", "model.layers.33.mlp.experts.59.down_proj", "model.layers.33.mlp.experts.60.down_proj", "model.layers.33.mlp.experts.61.down_proj", "model.layers.33.mlp.experts.62.down_proj", "model.layers.33.mlp.experts.63.down_proj", "model.layers.33.mlp.experts.64.down_proj", "model.layers.33.mlp.experts.65.down_proj", "model.layers.33.mlp.experts.66.down_proj", "model.layers.33.mlp.experts.67.down_proj", "model.layers.33.mlp.experts.68.down_proj", "model.layers.33.mlp.experts.69.down_proj", "model.layers.33.mlp.experts.70.down_proj", "model.layers.33.mlp.experts.71.down_proj", "model.layers.33.mlp.experts.72.down_proj", "model.layers.33.mlp.experts.73.down_proj", "model.layers.33.mlp.experts.74.down_proj", "model.layers.33.mlp.experts.75.down_proj", "model.layers.33.mlp.experts.76.down_proj", "model.layers.33.mlp.experts.77.down_proj", "model.layers.33.mlp.experts.78.down_proj", "model.layers.33.mlp.experts.79.down_proj", "model.layers.33.mlp.experts.80.down_proj", "model.layers.33.mlp.experts.81.down_proj", "model.layers.33.mlp.experts.82.down_proj", "model.layers.33.mlp.experts.83.down_proj", "model.layers.33.mlp.experts.84.down_proj", "model.layers.33.mlp.experts.85.down_proj", "model.layers.33.mlp.experts.86.down_proj", "model.layers.33.mlp.experts.87.down_proj", "model.layers.33.mlp.experts.88.down_proj", "model.layers.33.mlp.experts.89.down_proj", "model.layers.33.mlp.experts.90.down_proj", "model.layers.33.mlp.experts.91.down_proj", "model.layers.33.mlp.experts.92.down_proj", "model.layers.33.mlp.experts.93.down_proj", "model.layers.33.mlp.experts.94.down_proj", "model.layers.33.mlp.experts.95.down_proj", "model.layers.33.mlp.experts.96.down_proj", "model.layers.33.mlp.experts.97.down_proj", "model.layers.33.mlp.experts.98.down_proj", "model.layers.33.mlp.experts.99.down_proj", "model.layers.33.mlp.experts.100.down_proj", "model.layers.33.mlp.experts.101.down_proj", "model.layers.33.mlp.experts.102.down_proj", "model.layers.33.mlp.experts.103.down_proj", "model.layers.33.mlp.experts.104.down_proj", "model.layers.33.mlp.experts.105.down_proj", "model.layers.33.mlp.experts.106.down_proj", "model.layers.33.mlp.experts.107.down_proj", "model.layers.33.mlp.experts.108.down_proj", "model.layers.33.mlp.experts.109.down_proj", "model.layers.33.mlp.experts.110.down_proj", "model.layers.33.mlp.experts.111.down_proj", "model.layers.33.mlp.experts.112.down_proj", "model.layers.33.mlp.experts.113.down_proj", "model.layers.33.mlp.experts.114.down_proj", "model.layers.33.mlp.experts.115.down_proj", "model.layers.33.mlp.experts.116.down_proj", "model.layers.33.mlp.experts.117.down_proj", "model.layers.33.mlp.experts.118.down_proj", "model.layers.33.mlp.experts.119.down_proj", "model.layers.33.mlp.experts.120.down_proj", "model.layers.33.mlp.experts.121.down_proj", "model.layers.33.mlp.experts.122.down_proj", "model.layers.33.mlp.experts.123.down_proj", "model.layers.33.mlp.experts.124.down_proj", "model.layers.33.mlp.experts.125.down_proj", "model.layers.33.mlp.experts.126.down_proj", "model.layers.33.mlp.experts.127.down_proj", "model.layers.33.mlp.experts.128.down_proj", "model.layers.33.mlp.experts.129.down_proj", "model.layers.33.mlp.experts.130.down_proj", "model.layers.33.mlp.experts.131.down_proj", "model.layers.33.mlp.experts.132.down_proj", "model.layers.33.mlp.experts.133.down_proj", "model.layers.33.mlp.experts.134.down_proj", "model.layers.33.mlp.experts.135.down_proj", "model.layers.33.mlp.experts.136.down_proj", "model.layers.33.mlp.experts.137.down_proj", "model.layers.33.mlp.experts.138.down_proj", "model.layers.33.mlp.experts.139.down_proj", "model.layers.33.mlp.experts.140.down_proj", "model.layers.33.mlp.experts.141.down_proj", "model.layers.33.mlp.experts.142.down_proj", "model.layers.33.mlp.experts.143.down_proj", "model.layers.33.mlp.experts.144.down_proj", "model.layers.33.mlp.experts.145.down_proj", "model.layers.33.mlp.experts.146.down_proj", "model.layers.33.mlp.experts.147.down_proj", "model.layers.33.mlp.experts.148.down_proj", "model.layers.33.mlp.experts.149.down_proj", "model.layers.33.mlp.experts.150.down_proj", "model.layers.33.mlp.experts.151.down_proj", "model.layers.33.mlp.experts.152.down_proj", "model.layers.33.mlp.experts.153.down_proj", "model.layers.33.mlp.experts.154.down_proj", "model.layers.33.mlp.experts.155.down_proj", "model.layers.33.mlp.experts.156.down_proj", "model.layers.33.mlp.experts.157.down_proj", "model.layers.33.mlp.experts.158.down_proj", "model.layers.33.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00176244378089907, "dbits": 1258291200 } ] }, { "idx": 201, "layers": [ "model.layers.34.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0007120512425899395, "dbits": 62914560 } ] }, { "idx": 202, "layers": [ "model.layers.34.self_attn.k_proj", "model.layers.34.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0015227861702442058, "dbits": 10485760 } ] }, { "idx": 203, "layers": [ "model.layers.34.self_attn.o_proj" ], "candidates": [ { "dkld": 0.001294954866170872, "dbits": 62914560 } ] }, { "idx": 204, "layers": [ "model.layers.34.mlp.shared_experts.gate_proj", "model.layers.34.mlp.shared_experts.up_proj", "model.layers.34.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -3.236755728724949e-05, "dbits": 23592960 } ] }, { "idx": 205, "layers": [ "model.layers.34.mlp.experts.0.gate_proj", "model.layers.34.mlp.experts.1.gate_proj", "model.layers.34.mlp.experts.2.gate_proj", "model.layers.34.mlp.experts.3.gate_proj", "model.layers.34.mlp.experts.4.gate_proj", "model.layers.34.mlp.experts.5.gate_proj", "model.layers.34.mlp.experts.6.gate_proj", "model.layers.34.mlp.experts.7.gate_proj", "model.layers.34.mlp.experts.8.gate_proj", "model.layers.34.mlp.experts.9.gate_proj", "model.layers.34.mlp.experts.10.gate_proj", "model.layers.34.mlp.experts.11.gate_proj", "model.layers.34.mlp.experts.12.gate_proj", "model.layers.34.mlp.experts.13.gate_proj", "model.layers.34.mlp.experts.14.gate_proj", "model.layers.34.mlp.experts.15.gate_proj", "model.layers.34.mlp.experts.16.gate_proj", "model.layers.34.mlp.experts.17.gate_proj", "model.layers.34.mlp.experts.18.gate_proj", "model.layers.34.mlp.experts.19.gate_proj", "model.layers.34.mlp.experts.20.gate_proj", "model.layers.34.mlp.experts.21.gate_proj", "model.layers.34.mlp.experts.22.gate_proj", "model.layers.34.mlp.experts.23.gate_proj", "model.layers.34.mlp.experts.24.gate_proj", "model.layers.34.mlp.experts.25.gate_proj", "model.layers.34.mlp.experts.26.gate_proj", "model.layers.34.mlp.experts.27.gate_proj", "model.layers.34.mlp.experts.28.gate_proj", "model.layers.34.mlp.experts.29.gate_proj", "model.layers.34.mlp.experts.30.gate_proj", "model.layers.34.mlp.experts.31.gate_proj", "model.layers.34.mlp.experts.32.gate_proj", "model.layers.34.mlp.experts.33.gate_proj", "model.layers.34.mlp.experts.34.gate_proj", "model.layers.34.mlp.experts.35.gate_proj", "model.layers.34.mlp.experts.36.gate_proj", "model.layers.34.mlp.experts.37.gate_proj", "model.layers.34.mlp.experts.38.gate_proj", "model.layers.34.mlp.experts.39.gate_proj", "model.layers.34.mlp.experts.40.gate_proj", "model.layers.34.mlp.experts.41.gate_proj", "model.layers.34.mlp.experts.42.gate_proj", "model.layers.34.mlp.experts.43.gate_proj", "model.layers.34.mlp.experts.44.gate_proj", "model.layers.34.mlp.experts.45.gate_proj", "model.layers.34.mlp.experts.46.gate_proj", "model.layers.34.mlp.experts.47.gate_proj", "model.layers.34.mlp.experts.48.gate_proj", "model.layers.34.mlp.experts.49.gate_proj", "model.layers.34.mlp.experts.50.gate_proj", "model.layers.34.mlp.experts.51.gate_proj", "model.layers.34.mlp.experts.52.gate_proj", "model.layers.34.mlp.experts.53.gate_proj", "model.layers.34.mlp.experts.54.gate_proj", "model.layers.34.mlp.experts.55.gate_proj", "model.layers.34.mlp.experts.56.gate_proj", "model.layers.34.mlp.experts.57.gate_proj", "model.layers.34.mlp.experts.58.gate_proj", "model.layers.34.mlp.experts.59.gate_proj", "model.layers.34.mlp.experts.60.gate_proj", "model.layers.34.mlp.experts.61.gate_proj", "model.layers.34.mlp.experts.62.gate_proj", "model.layers.34.mlp.experts.63.gate_proj", "model.layers.34.mlp.experts.64.gate_proj", "model.layers.34.mlp.experts.65.gate_proj", "model.layers.34.mlp.experts.66.gate_proj", "model.layers.34.mlp.experts.67.gate_proj", "model.layers.34.mlp.experts.68.gate_proj", "model.layers.34.mlp.experts.69.gate_proj", "model.layers.34.mlp.experts.70.gate_proj", "model.layers.34.mlp.experts.71.gate_proj", "model.layers.34.mlp.experts.72.gate_proj", "model.layers.34.mlp.experts.73.gate_proj", "model.layers.34.mlp.experts.74.gate_proj", "model.layers.34.mlp.experts.75.gate_proj", "model.layers.34.mlp.experts.76.gate_proj", "model.layers.34.mlp.experts.77.gate_proj", "model.layers.34.mlp.experts.78.gate_proj", "model.layers.34.mlp.experts.79.gate_proj", "model.layers.34.mlp.experts.80.gate_proj", "model.layers.34.mlp.experts.81.gate_proj", "model.layers.34.mlp.experts.82.gate_proj", "model.layers.34.mlp.experts.83.gate_proj", "model.layers.34.mlp.experts.84.gate_proj", "model.layers.34.mlp.experts.85.gate_proj", "model.layers.34.mlp.experts.86.gate_proj", "model.layers.34.mlp.experts.87.gate_proj", "model.layers.34.mlp.experts.88.gate_proj", "model.layers.34.mlp.experts.89.gate_proj", "model.layers.34.mlp.experts.90.gate_proj", "model.layers.34.mlp.experts.91.gate_proj", "model.layers.34.mlp.experts.92.gate_proj", "model.layers.34.mlp.experts.93.gate_proj", "model.layers.34.mlp.experts.94.gate_proj", "model.layers.34.mlp.experts.95.gate_proj", "model.layers.34.mlp.experts.96.gate_proj", "model.layers.34.mlp.experts.97.gate_proj", "model.layers.34.mlp.experts.98.gate_proj", "model.layers.34.mlp.experts.99.gate_proj", "model.layers.34.mlp.experts.100.gate_proj", "model.layers.34.mlp.experts.101.gate_proj", "model.layers.34.mlp.experts.102.gate_proj", "model.layers.34.mlp.experts.103.gate_proj", "model.layers.34.mlp.experts.104.gate_proj", "model.layers.34.mlp.experts.105.gate_proj", "model.layers.34.mlp.experts.106.gate_proj", "model.layers.34.mlp.experts.107.gate_proj", "model.layers.34.mlp.experts.108.gate_proj", "model.layers.34.mlp.experts.109.gate_proj", "model.layers.34.mlp.experts.110.gate_proj", "model.layers.34.mlp.experts.111.gate_proj", "model.layers.34.mlp.experts.112.gate_proj", "model.layers.34.mlp.experts.113.gate_proj", "model.layers.34.mlp.experts.114.gate_proj", "model.layers.34.mlp.experts.115.gate_proj", "model.layers.34.mlp.experts.116.gate_proj", "model.layers.34.mlp.experts.117.gate_proj", "model.layers.34.mlp.experts.118.gate_proj", "model.layers.34.mlp.experts.119.gate_proj", "model.layers.34.mlp.experts.120.gate_proj", "model.layers.34.mlp.experts.121.gate_proj", "model.layers.34.mlp.experts.122.gate_proj", "model.layers.34.mlp.experts.123.gate_proj", "model.layers.34.mlp.experts.124.gate_proj", "model.layers.34.mlp.experts.125.gate_proj", "model.layers.34.mlp.experts.126.gate_proj", "model.layers.34.mlp.experts.127.gate_proj", "model.layers.34.mlp.experts.128.gate_proj", "model.layers.34.mlp.experts.129.gate_proj", "model.layers.34.mlp.experts.130.gate_proj", "model.layers.34.mlp.experts.131.gate_proj", "model.layers.34.mlp.experts.132.gate_proj", "model.layers.34.mlp.experts.133.gate_proj", "model.layers.34.mlp.experts.134.gate_proj", "model.layers.34.mlp.experts.135.gate_proj", "model.layers.34.mlp.experts.136.gate_proj", "model.layers.34.mlp.experts.137.gate_proj", "model.layers.34.mlp.experts.138.gate_proj", "model.layers.34.mlp.experts.139.gate_proj", "model.layers.34.mlp.experts.140.gate_proj", "model.layers.34.mlp.experts.141.gate_proj", "model.layers.34.mlp.experts.142.gate_proj", "model.layers.34.mlp.experts.143.gate_proj", "model.layers.34.mlp.experts.144.gate_proj", "model.layers.34.mlp.experts.145.gate_proj", "model.layers.34.mlp.experts.146.gate_proj", "model.layers.34.mlp.experts.147.gate_proj", "model.layers.34.mlp.experts.148.gate_proj", "model.layers.34.mlp.experts.149.gate_proj", "model.layers.34.mlp.experts.150.gate_proj", "model.layers.34.mlp.experts.151.gate_proj", "model.layers.34.mlp.experts.152.gate_proj", "model.layers.34.mlp.experts.153.gate_proj", "model.layers.34.mlp.experts.154.gate_proj", "model.layers.34.mlp.experts.155.gate_proj", "model.layers.34.mlp.experts.156.gate_proj", "model.layers.34.mlp.experts.157.gate_proj", "model.layers.34.mlp.experts.158.gate_proj", "model.layers.34.mlp.experts.159.gate_proj", "model.layers.34.mlp.experts.0.up_proj", "model.layers.34.mlp.experts.1.up_proj", "model.layers.34.mlp.experts.2.up_proj", "model.layers.34.mlp.experts.3.up_proj", "model.layers.34.mlp.experts.4.up_proj", "model.layers.34.mlp.experts.5.up_proj", "model.layers.34.mlp.experts.6.up_proj", "model.layers.34.mlp.experts.7.up_proj", "model.layers.34.mlp.experts.8.up_proj", "model.layers.34.mlp.experts.9.up_proj", "model.layers.34.mlp.experts.10.up_proj", "model.layers.34.mlp.experts.11.up_proj", "model.layers.34.mlp.experts.12.up_proj", "model.layers.34.mlp.experts.13.up_proj", "model.layers.34.mlp.experts.14.up_proj", "model.layers.34.mlp.experts.15.up_proj", "model.layers.34.mlp.experts.16.up_proj", "model.layers.34.mlp.experts.17.up_proj", "model.layers.34.mlp.experts.18.up_proj", "model.layers.34.mlp.experts.19.up_proj", "model.layers.34.mlp.experts.20.up_proj", "model.layers.34.mlp.experts.21.up_proj", "model.layers.34.mlp.experts.22.up_proj", "model.layers.34.mlp.experts.23.up_proj", "model.layers.34.mlp.experts.24.up_proj", "model.layers.34.mlp.experts.25.up_proj", "model.layers.34.mlp.experts.26.up_proj", "model.layers.34.mlp.experts.27.up_proj", "model.layers.34.mlp.experts.28.up_proj", "model.layers.34.mlp.experts.29.up_proj", "model.layers.34.mlp.experts.30.up_proj", "model.layers.34.mlp.experts.31.up_proj", "model.layers.34.mlp.experts.32.up_proj", "model.layers.34.mlp.experts.33.up_proj", "model.layers.34.mlp.experts.34.up_proj", "model.layers.34.mlp.experts.35.up_proj", "model.layers.34.mlp.experts.36.up_proj", "model.layers.34.mlp.experts.37.up_proj", "model.layers.34.mlp.experts.38.up_proj", "model.layers.34.mlp.experts.39.up_proj", "model.layers.34.mlp.experts.40.up_proj", "model.layers.34.mlp.experts.41.up_proj", "model.layers.34.mlp.experts.42.up_proj", "model.layers.34.mlp.experts.43.up_proj", "model.layers.34.mlp.experts.44.up_proj", "model.layers.34.mlp.experts.45.up_proj", "model.layers.34.mlp.experts.46.up_proj", "model.layers.34.mlp.experts.47.up_proj", "model.layers.34.mlp.experts.48.up_proj", "model.layers.34.mlp.experts.49.up_proj", "model.layers.34.mlp.experts.50.up_proj", "model.layers.34.mlp.experts.51.up_proj", "model.layers.34.mlp.experts.52.up_proj", "model.layers.34.mlp.experts.53.up_proj", "model.layers.34.mlp.experts.54.up_proj", "model.layers.34.mlp.experts.55.up_proj", "model.layers.34.mlp.experts.56.up_proj", "model.layers.34.mlp.experts.57.up_proj", "model.layers.34.mlp.experts.58.up_proj", "model.layers.34.mlp.experts.59.up_proj", "model.layers.34.mlp.experts.60.up_proj", "model.layers.34.mlp.experts.61.up_proj", "model.layers.34.mlp.experts.62.up_proj", "model.layers.34.mlp.experts.63.up_proj", "model.layers.34.mlp.experts.64.up_proj", "model.layers.34.mlp.experts.65.up_proj", "model.layers.34.mlp.experts.66.up_proj", "model.layers.34.mlp.experts.67.up_proj", "model.layers.34.mlp.experts.68.up_proj", "model.layers.34.mlp.experts.69.up_proj", "model.layers.34.mlp.experts.70.up_proj", "model.layers.34.mlp.experts.71.up_proj", "model.layers.34.mlp.experts.72.up_proj", "model.layers.34.mlp.experts.73.up_proj", "model.layers.34.mlp.experts.74.up_proj", "model.layers.34.mlp.experts.75.up_proj", "model.layers.34.mlp.experts.76.up_proj", "model.layers.34.mlp.experts.77.up_proj", "model.layers.34.mlp.experts.78.up_proj", "model.layers.34.mlp.experts.79.up_proj", "model.layers.34.mlp.experts.80.up_proj", "model.layers.34.mlp.experts.81.up_proj", "model.layers.34.mlp.experts.82.up_proj", "model.layers.34.mlp.experts.83.up_proj", "model.layers.34.mlp.experts.84.up_proj", "model.layers.34.mlp.experts.85.up_proj", "model.layers.34.mlp.experts.86.up_proj", "model.layers.34.mlp.experts.87.up_proj", "model.layers.34.mlp.experts.88.up_proj", "model.layers.34.mlp.experts.89.up_proj", "model.layers.34.mlp.experts.90.up_proj", "model.layers.34.mlp.experts.91.up_proj", "model.layers.34.mlp.experts.92.up_proj", "model.layers.34.mlp.experts.93.up_proj", "model.layers.34.mlp.experts.94.up_proj", "model.layers.34.mlp.experts.95.up_proj", "model.layers.34.mlp.experts.96.up_proj", "model.layers.34.mlp.experts.97.up_proj", "model.layers.34.mlp.experts.98.up_proj", "model.layers.34.mlp.experts.99.up_proj", "model.layers.34.mlp.experts.100.up_proj", "model.layers.34.mlp.experts.101.up_proj", "model.layers.34.mlp.experts.102.up_proj", "model.layers.34.mlp.experts.103.up_proj", "model.layers.34.mlp.experts.104.up_proj", "model.layers.34.mlp.experts.105.up_proj", "model.layers.34.mlp.experts.106.up_proj", "model.layers.34.mlp.experts.107.up_proj", "model.layers.34.mlp.experts.108.up_proj", "model.layers.34.mlp.experts.109.up_proj", "model.layers.34.mlp.experts.110.up_proj", "model.layers.34.mlp.experts.111.up_proj", "model.layers.34.mlp.experts.112.up_proj", "model.layers.34.mlp.experts.113.up_proj", "model.layers.34.mlp.experts.114.up_proj", "model.layers.34.mlp.experts.115.up_proj", "model.layers.34.mlp.experts.116.up_proj", "model.layers.34.mlp.experts.117.up_proj", "model.layers.34.mlp.experts.118.up_proj", "model.layers.34.mlp.experts.119.up_proj", "model.layers.34.mlp.experts.120.up_proj", "model.layers.34.mlp.experts.121.up_proj", "model.layers.34.mlp.experts.122.up_proj", "model.layers.34.mlp.experts.123.up_proj", "model.layers.34.mlp.experts.124.up_proj", "model.layers.34.mlp.experts.125.up_proj", "model.layers.34.mlp.experts.126.up_proj", "model.layers.34.mlp.experts.127.up_proj", "model.layers.34.mlp.experts.128.up_proj", "model.layers.34.mlp.experts.129.up_proj", "model.layers.34.mlp.experts.130.up_proj", "model.layers.34.mlp.experts.131.up_proj", "model.layers.34.mlp.experts.132.up_proj", "model.layers.34.mlp.experts.133.up_proj", "model.layers.34.mlp.experts.134.up_proj", "model.layers.34.mlp.experts.135.up_proj", "model.layers.34.mlp.experts.136.up_proj", "model.layers.34.mlp.experts.137.up_proj", "model.layers.34.mlp.experts.138.up_proj", "model.layers.34.mlp.experts.139.up_proj", "model.layers.34.mlp.experts.140.up_proj", "model.layers.34.mlp.experts.141.up_proj", "model.layers.34.mlp.experts.142.up_proj", "model.layers.34.mlp.experts.143.up_proj", "model.layers.34.mlp.experts.144.up_proj", "model.layers.34.mlp.experts.145.up_proj", "model.layers.34.mlp.experts.146.up_proj", "model.layers.34.mlp.experts.147.up_proj", "model.layers.34.mlp.experts.148.up_proj", "model.layers.34.mlp.experts.149.up_proj", "model.layers.34.mlp.experts.150.up_proj", "model.layers.34.mlp.experts.151.up_proj", "model.layers.34.mlp.experts.152.up_proj", "model.layers.34.mlp.experts.153.up_proj", "model.layers.34.mlp.experts.154.up_proj", "model.layers.34.mlp.experts.155.up_proj", "model.layers.34.mlp.experts.156.up_proj", "model.layers.34.mlp.experts.157.up_proj", "model.layers.34.mlp.experts.158.up_proj", "model.layers.34.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.003520536422729481, "dbits": 2516582400 } ] }, { "idx": 206, "layers": [ "model.layers.34.mlp.experts.0.down_proj", "model.layers.34.mlp.experts.1.down_proj", "model.layers.34.mlp.experts.2.down_proj", "model.layers.34.mlp.experts.3.down_proj", "model.layers.34.mlp.experts.4.down_proj", "model.layers.34.mlp.experts.5.down_proj", "model.layers.34.mlp.experts.6.down_proj", "model.layers.34.mlp.experts.7.down_proj", "model.layers.34.mlp.experts.8.down_proj", "model.layers.34.mlp.experts.9.down_proj", "model.layers.34.mlp.experts.10.down_proj", "model.layers.34.mlp.experts.11.down_proj", "model.layers.34.mlp.experts.12.down_proj", "model.layers.34.mlp.experts.13.down_proj", "model.layers.34.mlp.experts.14.down_proj", "model.layers.34.mlp.experts.15.down_proj", "model.layers.34.mlp.experts.16.down_proj", "model.layers.34.mlp.experts.17.down_proj", "model.layers.34.mlp.experts.18.down_proj", "model.layers.34.mlp.experts.19.down_proj", "model.layers.34.mlp.experts.20.down_proj", "model.layers.34.mlp.experts.21.down_proj", "model.layers.34.mlp.experts.22.down_proj", "model.layers.34.mlp.experts.23.down_proj", "model.layers.34.mlp.experts.24.down_proj", "model.layers.34.mlp.experts.25.down_proj", "model.layers.34.mlp.experts.26.down_proj", "model.layers.34.mlp.experts.27.down_proj", "model.layers.34.mlp.experts.28.down_proj", "model.layers.34.mlp.experts.29.down_proj", "model.layers.34.mlp.experts.30.down_proj", "model.layers.34.mlp.experts.31.down_proj", "model.layers.34.mlp.experts.32.down_proj", "model.layers.34.mlp.experts.33.down_proj", "model.layers.34.mlp.experts.34.down_proj", "model.layers.34.mlp.experts.35.down_proj", "model.layers.34.mlp.experts.36.down_proj", "model.layers.34.mlp.experts.37.down_proj", "model.layers.34.mlp.experts.38.down_proj", "model.layers.34.mlp.experts.39.down_proj", "model.layers.34.mlp.experts.40.down_proj", "model.layers.34.mlp.experts.41.down_proj", "model.layers.34.mlp.experts.42.down_proj", "model.layers.34.mlp.experts.43.down_proj", "model.layers.34.mlp.experts.44.down_proj", "model.layers.34.mlp.experts.45.down_proj", "model.layers.34.mlp.experts.46.down_proj", "model.layers.34.mlp.experts.47.down_proj", "model.layers.34.mlp.experts.48.down_proj", "model.layers.34.mlp.experts.49.down_proj", "model.layers.34.mlp.experts.50.down_proj", "model.layers.34.mlp.experts.51.down_proj", "model.layers.34.mlp.experts.52.down_proj", "model.layers.34.mlp.experts.53.down_proj", "model.layers.34.mlp.experts.54.down_proj", "model.layers.34.mlp.experts.55.down_proj", "model.layers.34.mlp.experts.56.down_proj", "model.layers.34.mlp.experts.57.down_proj", "model.layers.34.mlp.experts.58.down_proj", "model.layers.34.mlp.experts.59.down_proj", "model.layers.34.mlp.experts.60.down_proj", "model.layers.34.mlp.experts.61.down_proj", "model.layers.34.mlp.experts.62.down_proj", "model.layers.34.mlp.experts.63.down_proj", "model.layers.34.mlp.experts.64.down_proj", "model.layers.34.mlp.experts.65.down_proj", "model.layers.34.mlp.experts.66.down_proj", "model.layers.34.mlp.experts.67.down_proj", "model.layers.34.mlp.experts.68.down_proj", "model.layers.34.mlp.experts.69.down_proj", "model.layers.34.mlp.experts.70.down_proj", "model.layers.34.mlp.experts.71.down_proj", "model.layers.34.mlp.experts.72.down_proj", "model.layers.34.mlp.experts.73.down_proj", "model.layers.34.mlp.experts.74.down_proj", "model.layers.34.mlp.experts.75.down_proj", "model.layers.34.mlp.experts.76.down_proj", "model.layers.34.mlp.experts.77.down_proj", "model.layers.34.mlp.experts.78.down_proj", "model.layers.34.mlp.experts.79.down_proj", "model.layers.34.mlp.experts.80.down_proj", "model.layers.34.mlp.experts.81.down_proj", "model.layers.34.mlp.experts.82.down_proj", "model.layers.34.mlp.experts.83.down_proj", "model.layers.34.mlp.experts.84.down_proj", "model.layers.34.mlp.experts.85.down_proj", "model.layers.34.mlp.experts.86.down_proj", "model.layers.34.mlp.experts.87.down_proj", "model.layers.34.mlp.experts.88.down_proj", "model.layers.34.mlp.experts.89.down_proj", "model.layers.34.mlp.experts.90.down_proj", "model.layers.34.mlp.experts.91.down_proj", "model.layers.34.mlp.experts.92.down_proj", "model.layers.34.mlp.experts.93.down_proj", "model.layers.34.mlp.experts.94.down_proj", "model.layers.34.mlp.experts.95.down_proj", "model.layers.34.mlp.experts.96.down_proj", "model.layers.34.mlp.experts.97.down_proj", "model.layers.34.mlp.experts.98.down_proj", "model.layers.34.mlp.experts.99.down_proj", "model.layers.34.mlp.experts.100.down_proj", "model.layers.34.mlp.experts.101.down_proj", "model.layers.34.mlp.experts.102.down_proj", "model.layers.34.mlp.experts.103.down_proj", "model.layers.34.mlp.experts.104.down_proj", "model.layers.34.mlp.experts.105.down_proj", "model.layers.34.mlp.experts.106.down_proj", "model.layers.34.mlp.experts.107.down_proj", "model.layers.34.mlp.experts.108.down_proj", "model.layers.34.mlp.experts.109.down_proj", "model.layers.34.mlp.experts.110.down_proj", "model.layers.34.mlp.experts.111.down_proj", "model.layers.34.mlp.experts.112.down_proj", "model.layers.34.mlp.experts.113.down_proj", "model.layers.34.mlp.experts.114.down_proj", "model.layers.34.mlp.experts.115.down_proj", "model.layers.34.mlp.experts.116.down_proj", "model.layers.34.mlp.experts.117.down_proj", "model.layers.34.mlp.experts.118.down_proj", "model.layers.34.mlp.experts.119.down_proj", "model.layers.34.mlp.experts.120.down_proj", "model.layers.34.mlp.experts.121.down_proj", "model.layers.34.mlp.experts.122.down_proj", "model.layers.34.mlp.experts.123.down_proj", "model.layers.34.mlp.experts.124.down_proj", "model.layers.34.mlp.experts.125.down_proj", "model.layers.34.mlp.experts.126.down_proj", "model.layers.34.mlp.experts.127.down_proj", "model.layers.34.mlp.experts.128.down_proj", "model.layers.34.mlp.experts.129.down_proj", "model.layers.34.mlp.experts.130.down_proj", "model.layers.34.mlp.experts.131.down_proj", "model.layers.34.mlp.experts.132.down_proj", "model.layers.34.mlp.experts.133.down_proj", "model.layers.34.mlp.experts.134.down_proj", "model.layers.34.mlp.experts.135.down_proj", "model.layers.34.mlp.experts.136.down_proj", "model.layers.34.mlp.experts.137.down_proj", "model.layers.34.mlp.experts.138.down_proj", "model.layers.34.mlp.experts.139.down_proj", "model.layers.34.mlp.experts.140.down_proj", "model.layers.34.mlp.experts.141.down_proj", "model.layers.34.mlp.experts.142.down_proj", "model.layers.34.mlp.experts.143.down_proj", "model.layers.34.mlp.experts.144.down_proj", "model.layers.34.mlp.experts.145.down_proj", "model.layers.34.mlp.experts.146.down_proj", "model.layers.34.mlp.experts.147.down_proj", "model.layers.34.mlp.experts.148.down_proj", "model.layers.34.mlp.experts.149.down_proj", "model.layers.34.mlp.experts.150.down_proj", "model.layers.34.mlp.experts.151.down_proj", "model.layers.34.mlp.experts.152.down_proj", "model.layers.34.mlp.experts.153.down_proj", "model.layers.34.mlp.experts.154.down_proj", "model.layers.34.mlp.experts.155.down_proj", "model.layers.34.mlp.experts.156.down_proj", "model.layers.34.mlp.experts.157.down_proj", "model.layers.34.mlp.experts.158.down_proj", "model.layers.34.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0038401290774345287, "dbits": 1258291200 } ] }, { "idx": 207, "layers": [ "model.layers.35.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00217465162277225, "dbits": 62914560 } ] }, { "idx": 208, "layers": [ "model.layers.35.self_attn.k_proj", "model.layers.35.self_attn.v_proj" ], "candidates": [ { "dkld": -0.006159700453281403, "dbits": 10485760 } ] }, { "idx": 209, "layers": [ "model.layers.35.self_attn.o_proj" ], "candidates": [ { "dkld": -0.004738509654998779, "dbits": 62914560 } ] }, { "idx": 210, "layers": [ "model.layers.35.mlp.shared_experts.gate_proj", "model.layers.35.mlp.shared_experts.up_proj", "model.layers.35.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0022104717791080475, "dbits": 23592960 } ] }, { "idx": 211, "layers": [ "model.layers.35.mlp.experts.0.gate_proj", "model.layers.35.mlp.experts.1.gate_proj", "model.layers.35.mlp.experts.2.gate_proj", "model.layers.35.mlp.experts.3.gate_proj", "model.layers.35.mlp.experts.4.gate_proj", "model.layers.35.mlp.experts.5.gate_proj", "model.layers.35.mlp.experts.6.gate_proj", "model.layers.35.mlp.experts.7.gate_proj", "model.layers.35.mlp.experts.8.gate_proj", "model.layers.35.mlp.experts.9.gate_proj", "model.layers.35.mlp.experts.10.gate_proj", "model.layers.35.mlp.experts.11.gate_proj", "model.layers.35.mlp.experts.12.gate_proj", "model.layers.35.mlp.experts.13.gate_proj", "model.layers.35.mlp.experts.14.gate_proj", "model.layers.35.mlp.experts.15.gate_proj", "model.layers.35.mlp.experts.16.gate_proj", "model.layers.35.mlp.experts.17.gate_proj", "model.layers.35.mlp.experts.18.gate_proj", "model.layers.35.mlp.experts.19.gate_proj", "model.layers.35.mlp.experts.20.gate_proj", "model.layers.35.mlp.experts.21.gate_proj", "model.layers.35.mlp.experts.22.gate_proj", "model.layers.35.mlp.experts.23.gate_proj", "model.layers.35.mlp.experts.24.gate_proj", "model.layers.35.mlp.experts.25.gate_proj", "model.layers.35.mlp.experts.26.gate_proj", "model.layers.35.mlp.experts.27.gate_proj", "model.layers.35.mlp.experts.28.gate_proj", "model.layers.35.mlp.experts.29.gate_proj", "model.layers.35.mlp.experts.30.gate_proj", "model.layers.35.mlp.experts.31.gate_proj", "model.layers.35.mlp.experts.32.gate_proj", "model.layers.35.mlp.experts.33.gate_proj", "model.layers.35.mlp.experts.34.gate_proj", "model.layers.35.mlp.experts.35.gate_proj", "model.layers.35.mlp.experts.36.gate_proj", "model.layers.35.mlp.experts.37.gate_proj", "model.layers.35.mlp.experts.38.gate_proj", "model.layers.35.mlp.experts.39.gate_proj", "model.layers.35.mlp.experts.40.gate_proj", "model.layers.35.mlp.experts.41.gate_proj", "model.layers.35.mlp.experts.42.gate_proj", "model.layers.35.mlp.experts.43.gate_proj", "model.layers.35.mlp.experts.44.gate_proj", "model.layers.35.mlp.experts.45.gate_proj", "model.layers.35.mlp.experts.46.gate_proj", "model.layers.35.mlp.experts.47.gate_proj", "model.layers.35.mlp.experts.48.gate_proj", "model.layers.35.mlp.experts.49.gate_proj", "model.layers.35.mlp.experts.50.gate_proj", "model.layers.35.mlp.experts.51.gate_proj", "model.layers.35.mlp.experts.52.gate_proj", "model.layers.35.mlp.experts.53.gate_proj", "model.layers.35.mlp.experts.54.gate_proj", "model.layers.35.mlp.experts.55.gate_proj", "model.layers.35.mlp.experts.56.gate_proj", "model.layers.35.mlp.experts.57.gate_proj", "model.layers.35.mlp.experts.58.gate_proj", "model.layers.35.mlp.experts.59.gate_proj", "model.layers.35.mlp.experts.60.gate_proj", "model.layers.35.mlp.experts.61.gate_proj", "model.layers.35.mlp.experts.62.gate_proj", "model.layers.35.mlp.experts.63.gate_proj", "model.layers.35.mlp.experts.64.gate_proj", "model.layers.35.mlp.experts.65.gate_proj", "model.layers.35.mlp.experts.66.gate_proj", "model.layers.35.mlp.experts.67.gate_proj", "model.layers.35.mlp.experts.68.gate_proj", "model.layers.35.mlp.experts.69.gate_proj", "model.layers.35.mlp.experts.70.gate_proj", "model.layers.35.mlp.experts.71.gate_proj", "model.layers.35.mlp.experts.72.gate_proj", "model.layers.35.mlp.experts.73.gate_proj", "model.layers.35.mlp.experts.74.gate_proj", "model.layers.35.mlp.experts.75.gate_proj", "model.layers.35.mlp.experts.76.gate_proj", "model.layers.35.mlp.experts.77.gate_proj", "model.layers.35.mlp.experts.78.gate_proj", "model.layers.35.mlp.experts.79.gate_proj", "model.layers.35.mlp.experts.80.gate_proj", "model.layers.35.mlp.experts.81.gate_proj", "model.layers.35.mlp.experts.82.gate_proj", "model.layers.35.mlp.experts.83.gate_proj", "model.layers.35.mlp.experts.84.gate_proj", "model.layers.35.mlp.experts.85.gate_proj", "model.layers.35.mlp.experts.86.gate_proj", "model.layers.35.mlp.experts.87.gate_proj", "model.layers.35.mlp.experts.88.gate_proj", "model.layers.35.mlp.experts.89.gate_proj", "model.layers.35.mlp.experts.90.gate_proj", "model.layers.35.mlp.experts.91.gate_proj", "model.layers.35.mlp.experts.92.gate_proj", "model.layers.35.mlp.experts.93.gate_proj", "model.layers.35.mlp.experts.94.gate_proj", "model.layers.35.mlp.experts.95.gate_proj", "model.layers.35.mlp.experts.96.gate_proj", "model.layers.35.mlp.experts.97.gate_proj", "model.layers.35.mlp.experts.98.gate_proj", "model.layers.35.mlp.experts.99.gate_proj", "model.layers.35.mlp.experts.100.gate_proj", "model.layers.35.mlp.experts.101.gate_proj", "model.layers.35.mlp.experts.102.gate_proj", "model.layers.35.mlp.experts.103.gate_proj", "model.layers.35.mlp.experts.104.gate_proj", "model.layers.35.mlp.experts.105.gate_proj", "model.layers.35.mlp.experts.106.gate_proj", "model.layers.35.mlp.experts.107.gate_proj", "model.layers.35.mlp.experts.108.gate_proj", "model.layers.35.mlp.experts.109.gate_proj", "model.layers.35.mlp.experts.110.gate_proj", "model.layers.35.mlp.experts.111.gate_proj", "model.layers.35.mlp.experts.112.gate_proj", "model.layers.35.mlp.experts.113.gate_proj", "model.layers.35.mlp.experts.114.gate_proj", "model.layers.35.mlp.experts.115.gate_proj", "model.layers.35.mlp.experts.116.gate_proj", "model.layers.35.mlp.experts.117.gate_proj", "model.layers.35.mlp.experts.118.gate_proj", "model.layers.35.mlp.experts.119.gate_proj", "model.layers.35.mlp.experts.120.gate_proj", "model.layers.35.mlp.experts.121.gate_proj", "model.layers.35.mlp.experts.122.gate_proj", "model.layers.35.mlp.experts.123.gate_proj", "model.layers.35.mlp.experts.124.gate_proj", "model.layers.35.mlp.experts.125.gate_proj", "model.layers.35.mlp.experts.126.gate_proj", "model.layers.35.mlp.experts.127.gate_proj", "model.layers.35.mlp.experts.128.gate_proj", "model.layers.35.mlp.experts.129.gate_proj", "model.layers.35.mlp.experts.130.gate_proj", "model.layers.35.mlp.experts.131.gate_proj", "model.layers.35.mlp.experts.132.gate_proj", "model.layers.35.mlp.experts.133.gate_proj", "model.layers.35.mlp.experts.134.gate_proj", "model.layers.35.mlp.experts.135.gate_proj", "model.layers.35.mlp.experts.136.gate_proj", "model.layers.35.mlp.experts.137.gate_proj", "model.layers.35.mlp.experts.138.gate_proj", "model.layers.35.mlp.experts.139.gate_proj", "model.layers.35.mlp.experts.140.gate_proj", "model.layers.35.mlp.experts.141.gate_proj", "model.layers.35.mlp.experts.142.gate_proj", "model.layers.35.mlp.experts.143.gate_proj", "model.layers.35.mlp.experts.144.gate_proj", "model.layers.35.mlp.experts.145.gate_proj", "model.layers.35.mlp.experts.146.gate_proj", "model.layers.35.mlp.experts.147.gate_proj", "model.layers.35.mlp.experts.148.gate_proj", "model.layers.35.mlp.experts.149.gate_proj", "model.layers.35.mlp.experts.150.gate_proj", "model.layers.35.mlp.experts.151.gate_proj", "model.layers.35.mlp.experts.152.gate_proj", "model.layers.35.mlp.experts.153.gate_proj", "model.layers.35.mlp.experts.154.gate_proj", "model.layers.35.mlp.experts.155.gate_proj", "model.layers.35.mlp.experts.156.gate_proj", "model.layers.35.mlp.experts.157.gate_proj", "model.layers.35.mlp.experts.158.gate_proj", "model.layers.35.mlp.experts.159.gate_proj", "model.layers.35.mlp.experts.0.up_proj", "model.layers.35.mlp.experts.1.up_proj", "model.layers.35.mlp.experts.2.up_proj", "model.layers.35.mlp.experts.3.up_proj", "model.layers.35.mlp.experts.4.up_proj", "model.layers.35.mlp.experts.5.up_proj", "model.layers.35.mlp.experts.6.up_proj", "model.layers.35.mlp.experts.7.up_proj", "model.layers.35.mlp.experts.8.up_proj", "model.layers.35.mlp.experts.9.up_proj", "model.layers.35.mlp.experts.10.up_proj", "model.layers.35.mlp.experts.11.up_proj", "model.layers.35.mlp.experts.12.up_proj", "model.layers.35.mlp.experts.13.up_proj", "model.layers.35.mlp.experts.14.up_proj", "model.layers.35.mlp.experts.15.up_proj", "model.layers.35.mlp.experts.16.up_proj", "model.layers.35.mlp.experts.17.up_proj", "model.layers.35.mlp.experts.18.up_proj", "model.layers.35.mlp.experts.19.up_proj", "model.layers.35.mlp.experts.20.up_proj", "model.layers.35.mlp.experts.21.up_proj", "model.layers.35.mlp.experts.22.up_proj", "model.layers.35.mlp.experts.23.up_proj", "model.layers.35.mlp.experts.24.up_proj", "model.layers.35.mlp.experts.25.up_proj", "model.layers.35.mlp.experts.26.up_proj", "model.layers.35.mlp.experts.27.up_proj", "model.layers.35.mlp.experts.28.up_proj", "model.layers.35.mlp.experts.29.up_proj", "model.layers.35.mlp.experts.30.up_proj", "model.layers.35.mlp.experts.31.up_proj", "model.layers.35.mlp.experts.32.up_proj", "model.layers.35.mlp.experts.33.up_proj", "model.layers.35.mlp.experts.34.up_proj", "model.layers.35.mlp.experts.35.up_proj", "model.layers.35.mlp.experts.36.up_proj", "model.layers.35.mlp.experts.37.up_proj", "model.layers.35.mlp.experts.38.up_proj", "model.layers.35.mlp.experts.39.up_proj", "model.layers.35.mlp.experts.40.up_proj", "model.layers.35.mlp.experts.41.up_proj", "model.layers.35.mlp.experts.42.up_proj", "model.layers.35.mlp.experts.43.up_proj", "model.layers.35.mlp.experts.44.up_proj", "model.layers.35.mlp.experts.45.up_proj", "model.layers.35.mlp.experts.46.up_proj", "model.layers.35.mlp.experts.47.up_proj", "model.layers.35.mlp.experts.48.up_proj", "model.layers.35.mlp.experts.49.up_proj", "model.layers.35.mlp.experts.50.up_proj", "model.layers.35.mlp.experts.51.up_proj", "model.layers.35.mlp.experts.52.up_proj", "model.layers.35.mlp.experts.53.up_proj", "model.layers.35.mlp.experts.54.up_proj", "model.layers.35.mlp.experts.55.up_proj", "model.layers.35.mlp.experts.56.up_proj", "model.layers.35.mlp.experts.57.up_proj", "model.layers.35.mlp.experts.58.up_proj", "model.layers.35.mlp.experts.59.up_proj", "model.layers.35.mlp.experts.60.up_proj", "model.layers.35.mlp.experts.61.up_proj", "model.layers.35.mlp.experts.62.up_proj", "model.layers.35.mlp.experts.63.up_proj", "model.layers.35.mlp.experts.64.up_proj", "model.layers.35.mlp.experts.65.up_proj", "model.layers.35.mlp.experts.66.up_proj", "model.layers.35.mlp.experts.67.up_proj", "model.layers.35.mlp.experts.68.up_proj", "model.layers.35.mlp.experts.69.up_proj", "model.layers.35.mlp.experts.70.up_proj", "model.layers.35.mlp.experts.71.up_proj", "model.layers.35.mlp.experts.72.up_proj", "model.layers.35.mlp.experts.73.up_proj", "model.layers.35.mlp.experts.74.up_proj", "model.layers.35.mlp.experts.75.up_proj", "model.layers.35.mlp.experts.76.up_proj", "model.layers.35.mlp.experts.77.up_proj", "model.layers.35.mlp.experts.78.up_proj", "model.layers.35.mlp.experts.79.up_proj", "model.layers.35.mlp.experts.80.up_proj", "model.layers.35.mlp.experts.81.up_proj", "model.layers.35.mlp.experts.82.up_proj", "model.layers.35.mlp.experts.83.up_proj", "model.layers.35.mlp.experts.84.up_proj", "model.layers.35.mlp.experts.85.up_proj", "model.layers.35.mlp.experts.86.up_proj", "model.layers.35.mlp.experts.87.up_proj", "model.layers.35.mlp.experts.88.up_proj", "model.layers.35.mlp.experts.89.up_proj", "model.layers.35.mlp.experts.90.up_proj", "model.layers.35.mlp.experts.91.up_proj", "model.layers.35.mlp.experts.92.up_proj", "model.layers.35.mlp.experts.93.up_proj", "model.layers.35.mlp.experts.94.up_proj", "model.layers.35.mlp.experts.95.up_proj", "model.layers.35.mlp.experts.96.up_proj", "model.layers.35.mlp.experts.97.up_proj", "model.layers.35.mlp.experts.98.up_proj", "model.layers.35.mlp.experts.99.up_proj", "model.layers.35.mlp.experts.100.up_proj", "model.layers.35.mlp.experts.101.up_proj", "model.layers.35.mlp.experts.102.up_proj", "model.layers.35.mlp.experts.103.up_proj", "model.layers.35.mlp.experts.104.up_proj", "model.layers.35.mlp.experts.105.up_proj", "model.layers.35.mlp.experts.106.up_proj", "model.layers.35.mlp.experts.107.up_proj", "model.layers.35.mlp.experts.108.up_proj", "model.layers.35.mlp.experts.109.up_proj", "model.layers.35.mlp.experts.110.up_proj", "model.layers.35.mlp.experts.111.up_proj", "model.layers.35.mlp.experts.112.up_proj", "model.layers.35.mlp.experts.113.up_proj", "model.layers.35.mlp.experts.114.up_proj", "model.layers.35.mlp.experts.115.up_proj", "model.layers.35.mlp.experts.116.up_proj", "model.layers.35.mlp.experts.117.up_proj", "model.layers.35.mlp.experts.118.up_proj", "model.layers.35.mlp.experts.119.up_proj", "model.layers.35.mlp.experts.120.up_proj", "model.layers.35.mlp.experts.121.up_proj", "model.layers.35.mlp.experts.122.up_proj", "model.layers.35.mlp.experts.123.up_proj", "model.layers.35.mlp.experts.124.up_proj", "model.layers.35.mlp.experts.125.up_proj", "model.layers.35.mlp.experts.126.up_proj", "model.layers.35.mlp.experts.127.up_proj", "model.layers.35.mlp.experts.128.up_proj", "model.layers.35.mlp.experts.129.up_proj", "model.layers.35.mlp.experts.130.up_proj", "model.layers.35.mlp.experts.131.up_proj", "model.layers.35.mlp.experts.132.up_proj", "model.layers.35.mlp.experts.133.up_proj", "model.layers.35.mlp.experts.134.up_proj", "model.layers.35.mlp.experts.135.up_proj", "model.layers.35.mlp.experts.136.up_proj", "model.layers.35.mlp.experts.137.up_proj", "model.layers.35.mlp.experts.138.up_proj", "model.layers.35.mlp.experts.139.up_proj", "model.layers.35.mlp.experts.140.up_proj", "model.layers.35.mlp.experts.141.up_proj", "model.layers.35.mlp.experts.142.up_proj", "model.layers.35.mlp.experts.143.up_proj", "model.layers.35.mlp.experts.144.up_proj", "model.layers.35.mlp.experts.145.up_proj", "model.layers.35.mlp.experts.146.up_proj", "model.layers.35.mlp.experts.147.up_proj", "model.layers.35.mlp.experts.148.up_proj", "model.layers.35.mlp.experts.149.up_proj", "model.layers.35.mlp.experts.150.up_proj", "model.layers.35.mlp.experts.151.up_proj", "model.layers.35.mlp.experts.152.up_proj", "model.layers.35.mlp.experts.153.up_proj", "model.layers.35.mlp.experts.154.up_proj", "model.layers.35.mlp.experts.155.up_proj", "model.layers.35.mlp.experts.156.up_proj", "model.layers.35.mlp.experts.157.up_proj", "model.layers.35.mlp.experts.158.up_proj", "model.layers.35.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.004939404129982006, "dbits": 2516582400 } ] }, { "idx": 212, "layers": [ "model.layers.35.mlp.experts.0.down_proj", "model.layers.35.mlp.experts.1.down_proj", "model.layers.35.mlp.experts.2.down_proj", "model.layers.35.mlp.experts.3.down_proj", "model.layers.35.mlp.experts.4.down_proj", "model.layers.35.mlp.experts.5.down_proj", "model.layers.35.mlp.experts.6.down_proj", "model.layers.35.mlp.experts.7.down_proj", "model.layers.35.mlp.experts.8.down_proj", "model.layers.35.mlp.experts.9.down_proj", "model.layers.35.mlp.experts.10.down_proj", "model.layers.35.mlp.experts.11.down_proj", "model.layers.35.mlp.experts.12.down_proj", "model.layers.35.mlp.experts.13.down_proj", "model.layers.35.mlp.experts.14.down_proj", "model.layers.35.mlp.experts.15.down_proj", "model.layers.35.mlp.experts.16.down_proj", "model.layers.35.mlp.experts.17.down_proj", "model.layers.35.mlp.experts.18.down_proj", "model.layers.35.mlp.experts.19.down_proj", "model.layers.35.mlp.experts.20.down_proj", "model.layers.35.mlp.experts.21.down_proj", "model.layers.35.mlp.experts.22.down_proj", "model.layers.35.mlp.experts.23.down_proj", "model.layers.35.mlp.experts.24.down_proj", "model.layers.35.mlp.experts.25.down_proj", "model.layers.35.mlp.experts.26.down_proj", "model.layers.35.mlp.experts.27.down_proj", "model.layers.35.mlp.experts.28.down_proj", "model.layers.35.mlp.experts.29.down_proj", "model.layers.35.mlp.experts.30.down_proj", "model.layers.35.mlp.experts.31.down_proj", "model.layers.35.mlp.experts.32.down_proj", "model.layers.35.mlp.experts.33.down_proj", "model.layers.35.mlp.experts.34.down_proj", "model.layers.35.mlp.experts.35.down_proj", "model.layers.35.mlp.experts.36.down_proj", "model.layers.35.mlp.experts.37.down_proj", "model.layers.35.mlp.experts.38.down_proj", "model.layers.35.mlp.experts.39.down_proj", "model.layers.35.mlp.experts.40.down_proj", "model.layers.35.mlp.experts.41.down_proj", "model.layers.35.mlp.experts.42.down_proj", "model.layers.35.mlp.experts.43.down_proj", "model.layers.35.mlp.experts.44.down_proj", "model.layers.35.mlp.experts.45.down_proj", "model.layers.35.mlp.experts.46.down_proj", "model.layers.35.mlp.experts.47.down_proj", "model.layers.35.mlp.experts.48.down_proj", "model.layers.35.mlp.experts.49.down_proj", "model.layers.35.mlp.experts.50.down_proj", "model.layers.35.mlp.experts.51.down_proj", "model.layers.35.mlp.experts.52.down_proj", "model.layers.35.mlp.experts.53.down_proj", "model.layers.35.mlp.experts.54.down_proj", "model.layers.35.mlp.experts.55.down_proj", "model.layers.35.mlp.experts.56.down_proj", "model.layers.35.mlp.experts.57.down_proj", "model.layers.35.mlp.experts.58.down_proj", "model.layers.35.mlp.experts.59.down_proj", "model.layers.35.mlp.experts.60.down_proj", "model.layers.35.mlp.experts.61.down_proj", "model.layers.35.mlp.experts.62.down_proj", "model.layers.35.mlp.experts.63.down_proj", "model.layers.35.mlp.experts.64.down_proj", "model.layers.35.mlp.experts.65.down_proj", "model.layers.35.mlp.experts.66.down_proj", "model.layers.35.mlp.experts.67.down_proj", "model.layers.35.mlp.experts.68.down_proj", "model.layers.35.mlp.experts.69.down_proj", "model.layers.35.mlp.experts.70.down_proj", "model.layers.35.mlp.experts.71.down_proj", "model.layers.35.mlp.experts.72.down_proj", "model.layers.35.mlp.experts.73.down_proj", "model.layers.35.mlp.experts.74.down_proj", "model.layers.35.mlp.experts.75.down_proj", "model.layers.35.mlp.experts.76.down_proj", "model.layers.35.mlp.experts.77.down_proj", "model.layers.35.mlp.experts.78.down_proj", "model.layers.35.mlp.experts.79.down_proj", "model.layers.35.mlp.experts.80.down_proj", "model.layers.35.mlp.experts.81.down_proj", "model.layers.35.mlp.experts.82.down_proj", "model.layers.35.mlp.experts.83.down_proj", "model.layers.35.mlp.experts.84.down_proj", "model.layers.35.mlp.experts.85.down_proj", "model.layers.35.mlp.experts.86.down_proj", "model.layers.35.mlp.experts.87.down_proj", "model.layers.35.mlp.experts.88.down_proj", "model.layers.35.mlp.experts.89.down_proj", "model.layers.35.mlp.experts.90.down_proj", "model.layers.35.mlp.experts.91.down_proj", "model.layers.35.mlp.experts.92.down_proj", "model.layers.35.mlp.experts.93.down_proj", "model.layers.35.mlp.experts.94.down_proj", "model.layers.35.mlp.experts.95.down_proj", "model.layers.35.mlp.experts.96.down_proj", "model.layers.35.mlp.experts.97.down_proj", "model.layers.35.mlp.experts.98.down_proj", "model.layers.35.mlp.experts.99.down_proj", "model.layers.35.mlp.experts.100.down_proj", "model.layers.35.mlp.experts.101.down_proj", "model.layers.35.mlp.experts.102.down_proj", "model.layers.35.mlp.experts.103.down_proj", "model.layers.35.mlp.experts.104.down_proj", "model.layers.35.mlp.experts.105.down_proj", "model.layers.35.mlp.experts.106.down_proj", "model.layers.35.mlp.experts.107.down_proj", "model.layers.35.mlp.experts.108.down_proj", "model.layers.35.mlp.experts.109.down_proj", "model.layers.35.mlp.experts.110.down_proj", "model.layers.35.mlp.experts.111.down_proj", "model.layers.35.mlp.experts.112.down_proj", "model.layers.35.mlp.experts.113.down_proj", "model.layers.35.mlp.experts.114.down_proj", "model.layers.35.mlp.experts.115.down_proj", "model.layers.35.mlp.experts.116.down_proj", "model.layers.35.mlp.experts.117.down_proj", "model.layers.35.mlp.experts.118.down_proj", "model.layers.35.mlp.experts.119.down_proj", "model.layers.35.mlp.experts.120.down_proj", "model.layers.35.mlp.experts.121.down_proj", "model.layers.35.mlp.experts.122.down_proj", "model.layers.35.mlp.experts.123.down_proj", "model.layers.35.mlp.experts.124.down_proj", "model.layers.35.mlp.experts.125.down_proj", "model.layers.35.mlp.experts.126.down_proj", "model.layers.35.mlp.experts.127.down_proj", "model.layers.35.mlp.experts.128.down_proj", "model.layers.35.mlp.experts.129.down_proj", "model.layers.35.mlp.experts.130.down_proj", "model.layers.35.mlp.experts.131.down_proj", "model.layers.35.mlp.experts.132.down_proj", "model.layers.35.mlp.experts.133.down_proj", "model.layers.35.mlp.experts.134.down_proj", "model.layers.35.mlp.experts.135.down_proj", "model.layers.35.mlp.experts.136.down_proj", "model.layers.35.mlp.experts.137.down_proj", "model.layers.35.mlp.experts.138.down_proj", "model.layers.35.mlp.experts.139.down_proj", "model.layers.35.mlp.experts.140.down_proj", "model.layers.35.mlp.experts.141.down_proj", "model.layers.35.mlp.experts.142.down_proj", "model.layers.35.mlp.experts.143.down_proj", "model.layers.35.mlp.experts.144.down_proj", "model.layers.35.mlp.experts.145.down_proj", "model.layers.35.mlp.experts.146.down_proj", "model.layers.35.mlp.experts.147.down_proj", "model.layers.35.mlp.experts.148.down_proj", "model.layers.35.mlp.experts.149.down_proj", "model.layers.35.mlp.experts.150.down_proj", "model.layers.35.mlp.experts.151.down_proj", "model.layers.35.mlp.experts.152.down_proj", "model.layers.35.mlp.experts.153.down_proj", "model.layers.35.mlp.experts.154.down_proj", "model.layers.35.mlp.experts.155.down_proj", "model.layers.35.mlp.experts.156.down_proj", "model.layers.35.mlp.experts.157.down_proj", "model.layers.35.mlp.experts.158.down_proj", "model.layers.35.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.003334052860736847, "dbits": 1258291200 } ] }, { "idx": 213, "layers": [ "model.layers.36.self_attn.q_proj" ], "candidates": [ { "dkld": -0.005372165143489871, "dbits": 62914560 } ] }, { "idx": 214, "layers": [ "model.layers.36.self_attn.k_proj", "model.layers.36.self_attn.v_proj" ], "candidates": [ { "dkld": 0.003542559593915917, "dbits": 10485760 } ] }, { "idx": 215, "layers": [ "model.layers.36.self_attn.o_proj" ], "candidates": [ { "dkld": -0.006466989964246772, "dbits": 62914560 } ] }, { "idx": 216, "layers": [ "model.layers.36.mlp.shared_experts.gate_proj", "model.layers.36.mlp.shared_experts.up_proj", "model.layers.36.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.007626178115606341, "dbits": 23592960 } ] }, { "idx": 217, "layers": [ "model.layers.36.mlp.experts.0.gate_proj", "model.layers.36.mlp.experts.1.gate_proj", "model.layers.36.mlp.experts.2.gate_proj", "model.layers.36.mlp.experts.3.gate_proj", "model.layers.36.mlp.experts.4.gate_proj", "model.layers.36.mlp.experts.5.gate_proj", "model.layers.36.mlp.experts.6.gate_proj", "model.layers.36.mlp.experts.7.gate_proj", "model.layers.36.mlp.experts.8.gate_proj", "model.layers.36.mlp.experts.9.gate_proj", "model.layers.36.mlp.experts.10.gate_proj", "model.layers.36.mlp.experts.11.gate_proj", "model.layers.36.mlp.experts.12.gate_proj", "model.layers.36.mlp.experts.13.gate_proj", "model.layers.36.mlp.experts.14.gate_proj", "model.layers.36.mlp.experts.15.gate_proj", "model.layers.36.mlp.experts.16.gate_proj", "model.layers.36.mlp.experts.17.gate_proj", "model.layers.36.mlp.experts.18.gate_proj", "model.layers.36.mlp.experts.19.gate_proj", "model.layers.36.mlp.experts.20.gate_proj", "model.layers.36.mlp.experts.21.gate_proj", "model.layers.36.mlp.experts.22.gate_proj", "model.layers.36.mlp.experts.23.gate_proj", "model.layers.36.mlp.experts.24.gate_proj", "model.layers.36.mlp.experts.25.gate_proj", "model.layers.36.mlp.experts.26.gate_proj", "model.layers.36.mlp.experts.27.gate_proj", "model.layers.36.mlp.experts.28.gate_proj", "model.layers.36.mlp.experts.29.gate_proj", "model.layers.36.mlp.experts.30.gate_proj", "model.layers.36.mlp.experts.31.gate_proj", "model.layers.36.mlp.experts.32.gate_proj", "model.layers.36.mlp.experts.33.gate_proj", "model.layers.36.mlp.experts.34.gate_proj", "model.layers.36.mlp.experts.35.gate_proj", "model.layers.36.mlp.experts.36.gate_proj", "model.layers.36.mlp.experts.37.gate_proj", "model.layers.36.mlp.experts.38.gate_proj", "model.layers.36.mlp.experts.39.gate_proj", "model.layers.36.mlp.experts.40.gate_proj", "model.layers.36.mlp.experts.41.gate_proj", "model.layers.36.mlp.experts.42.gate_proj", "model.layers.36.mlp.experts.43.gate_proj", "model.layers.36.mlp.experts.44.gate_proj", "model.layers.36.mlp.experts.45.gate_proj", "model.layers.36.mlp.experts.46.gate_proj", "model.layers.36.mlp.experts.47.gate_proj", "model.layers.36.mlp.experts.48.gate_proj", "model.layers.36.mlp.experts.49.gate_proj", "model.layers.36.mlp.experts.50.gate_proj", "model.layers.36.mlp.experts.51.gate_proj", "model.layers.36.mlp.experts.52.gate_proj", "model.layers.36.mlp.experts.53.gate_proj", "model.layers.36.mlp.experts.54.gate_proj", "model.layers.36.mlp.experts.55.gate_proj", "model.layers.36.mlp.experts.56.gate_proj", "model.layers.36.mlp.experts.57.gate_proj", "model.layers.36.mlp.experts.58.gate_proj", "model.layers.36.mlp.experts.59.gate_proj", "model.layers.36.mlp.experts.60.gate_proj", "model.layers.36.mlp.experts.61.gate_proj", "model.layers.36.mlp.experts.62.gate_proj", "model.layers.36.mlp.experts.63.gate_proj", "model.layers.36.mlp.experts.64.gate_proj", "model.layers.36.mlp.experts.65.gate_proj", "model.layers.36.mlp.experts.66.gate_proj", "model.layers.36.mlp.experts.67.gate_proj", "model.layers.36.mlp.experts.68.gate_proj", "model.layers.36.mlp.experts.69.gate_proj", "model.layers.36.mlp.experts.70.gate_proj", "model.layers.36.mlp.experts.71.gate_proj", "model.layers.36.mlp.experts.72.gate_proj", "model.layers.36.mlp.experts.73.gate_proj", "model.layers.36.mlp.experts.74.gate_proj", "model.layers.36.mlp.experts.75.gate_proj", "model.layers.36.mlp.experts.76.gate_proj", "model.layers.36.mlp.experts.77.gate_proj", "model.layers.36.mlp.experts.78.gate_proj", "model.layers.36.mlp.experts.79.gate_proj", "model.layers.36.mlp.experts.80.gate_proj", "model.layers.36.mlp.experts.81.gate_proj", "model.layers.36.mlp.experts.82.gate_proj", "model.layers.36.mlp.experts.83.gate_proj", "model.layers.36.mlp.experts.84.gate_proj", "model.layers.36.mlp.experts.85.gate_proj", "model.layers.36.mlp.experts.86.gate_proj", "model.layers.36.mlp.experts.87.gate_proj", "model.layers.36.mlp.experts.88.gate_proj", "model.layers.36.mlp.experts.89.gate_proj", "model.layers.36.mlp.experts.90.gate_proj", "model.layers.36.mlp.experts.91.gate_proj", "model.layers.36.mlp.experts.92.gate_proj", "model.layers.36.mlp.experts.93.gate_proj", "model.layers.36.mlp.experts.94.gate_proj", "model.layers.36.mlp.experts.95.gate_proj", "model.layers.36.mlp.experts.96.gate_proj", "model.layers.36.mlp.experts.97.gate_proj", "model.layers.36.mlp.experts.98.gate_proj", "model.layers.36.mlp.experts.99.gate_proj", "model.layers.36.mlp.experts.100.gate_proj", "model.layers.36.mlp.experts.101.gate_proj", "model.layers.36.mlp.experts.102.gate_proj", "model.layers.36.mlp.experts.103.gate_proj", "model.layers.36.mlp.experts.104.gate_proj", "model.layers.36.mlp.experts.105.gate_proj", "model.layers.36.mlp.experts.106.gate_proj", "model.layers.36.mlp.experts.107.gate_proj", "model.layers.36.mlp.experts.108.gate_proj", "model.layers.36.mlp.experts.109.gate_proj", "model.layers.36.mlp.experts.110.gate_proj", "model.layers.36.mlp.experts.111.gate_proj", "model.layers.36.mlp.experts.112.gate_proj", "model.layers.36.mlp.experts.113.gate_proj", "model.layers.36.mlp.experts.114.gate_proj", "model.layers.36.mlp.experts.115.gate_proj", "model.layers.36.mlp.experts.116.gate_proj", "model.layers.36.mlp.experts.117.gate_proj", "model.layers.36.mlp.experts.118.gate_proj", "model.layers.36.mlp.experts.119.gate_proj", "model.layers.36.mlp.experts.120.gate_proj", "model.layers.36.mlp.experts.121.gate_proj", "model.layers.36.mlp.experts.122.gate_proj", "model.layers.36.mlp.experts.123.gate_proj", "model.layers.36.mlp.experts.124.gate_proj", "model.layers.36.mlp.experts.125.gate_proj", "model.layers.36.mlp.experts.126.gate_proj", "model.layers.36.mlp.experts.127.gate_proj", "model.layers.36.mlp.experts.128.gate_proj", "model.layers.36.mlp.experts.129.gate_proj", "model.layers.36.mlp.experts.130.gate_proj", "model.layers.36.mlp.experts.131.gate_proj", "model.layers.36.mlp.experts.132.gate_proj", "model.layers.36.mlp.experts.133.gate_proj", "model.layers.36.mlp.experts.134.gate_proj", "model.layers.36.mlp.experts.135.gate_proj", "model.layers.36.mlp.experts.136.gate_proj", "model.layers.36.mlp.experts.137.gate_proj", "model.layers.36.mlp.experts.138.gate_proj", "model.layers.36.mlp.experts.139.gate_proj", "model.layers.36.mlp.experts.140.gate_proj", "model.layers.36.mlp.experts.141.gate_proj", "model.layers.36.mlp.experts.142.gate_proj", "model.layers.36.mlp.experts.143.gate_proj", "model.layers.36.mlp.experts.144.gate_proj", "model.layers.36.mlp.experts.145.gate_proj", "model.layers.36.mlp.experts.146.gate_proj", "model.layers.36.mlp.experts.147.gate_proj", "model.layers.36.mlp.experts.148.gate_proj", "model.layers.36.mlp.experts.149.gate_proj", "model.layers.36.mlp.experts.150.gate_proj", "model.layers.36.mlp.experts.151.gate_proj", "model.layers.36.mlp.experts.152.gate_proj", "model.layers.36.mlp.experts.153.gate_proj", "model.layers.36.mlp.experts.154.gate_proj", "model.layers.36.mlp.experts.155.gate_proj", "model.layers.36.mlp.experts.156.gate_proj", "model.layers.36.mlp.experts.157.gate_proj", "model.layers.36.mlp.experts.158.gate_proj", "model.layers.36.mlp.experts.159.gate_proj", "model.layers.36.mlp.experts.0.up_proj", "model.layers.36.mlp.experts.1.up_proj", "model.layers.36.mlp.experts.2.up_proj", "model.layers.36.mlp.experts.3.up_proj", "model.layers.36.mlp.experts.4.up_proj", "model.layers.36.mlp.experts.5.up_proj", "model.layers.36.mlp.experts.6.up_proj", "model.layers.36.mlp.experts.7.up_proj", "model.layers.36.mlp.experts.8.up_proj", "model.layers.36.mlp.experts.9.up_proj", "model.layers.36.mlp.experts.10.up_proj", "model.layers.36.mlp.experts.11.up_proj", "model.layers.36.mlp.experts.12.up_proj", "model.layers.36.mlp.experts.13.up_proj", "model.layers.36.mlp.experts.14.up_proj", "model.layers.36.mlp.experts.15.up_proj", "model.layers.36.mlp.experts.16.up_proj", "model.layers.36.mlp.experts.17.up_proj", "model.layers.36.mlp.experts.18.up_proj", "model.layers.36.mlp.experts.19.up_proj", "model.layers.36.mlp.experts.20.up_proj", "model.layers.36.mlp.experts.21.up_proj", "model.layers.36.mlp.experts.22.up_proj", "model.layers.36.mlp.experts.23.up_proj", "model.layers.36.mlp.experts.24.up_proj", "model.layers.36.mlp.experts.25.up_proj", "model.layers.36.mlp.experts.26.up_proj", "model.layers.36.mlp.experts.27.up_proj", "model.layers.36.mlp.experts.28.up_proj", "model.layers.36.mlp.experts.29.up_proj", "model.layers.36.mlp.experts.30.up_proj", "model.layers.36.mlp.experts.31.up_proj", "model.layers.36.mlp.experts.32.up_proj", "model.layers.36.mlp.experts.33.up_proj", "model.layers.36.mlp.experts.34.up_proj", "model.layers.36.mlp.experts.35.up_proj", "model.layers.36.mlp.experts.36.up_proj", "model.layers.36.mlp.experts.37.up_proj", "model.layers.36.mlp.experts.38.up_proj", "model.layers.36.mlp.experts.39.up_proj", "model.layers.36.mlp.experts.40.up_proj", "model.layers.36.mlp.experts.41.up_proj", "model.layers.36.mlp.experts.42.up_proj", "model.layers.36.mlp.experts.43.up_proj", "model.layers.36.mlp.experts.44.up_proj", "model.layers.36.mlp.experts.45.up_proj", "model.layers.36.mlp.experts.46.up_proj", "model.layers.36.mlp.experts.47.up_proj", "model.layers.36.mlp.experts.48.up_proj", "model.layers.36.mlp.experts.49.up_proj", "model.layers.36.mlp.experts.50.up_proj", "model.layers.36.mlp.experts.51.up_proj", "model.layers.36.mlp.experts.52.up_proj", "model.layers.36.mlp.experts.53.up_proj", "model.layers.36.mlp.experts.54.up_proj", "model.layers.36.mlp.experts.55.up_proj", "model.layers.36.mlp.experts.56.up_proj", "model.layers.36.mlp.experts.57.up_proj", "model.layers.36.mlp.experts.58.up_proj", "model.layers.36.mlp.experts.59.up_proj", "model.layers.36.mlp.experts.60.up_proj", "model.layers.36.mlp.experts.61.up_proj", "model.layers.36.mlp.experts.62.up_proj", "model.layers.36.mlp.experts.63.up_proj", "model.layers.36.mlp.experts.64.up_proj", "model.layers.36.mlp.experts.65.up_proj", "model.layers.36.mlp.experts.66.up_proj", "model.layers.36.mlp.experts.67.up_proj", "model.layers.36.mlp.experts.68.up_proj", "model.layers.36.mlp.experts.69.up_proj", "model.layers.36.mlp.experts.70.up_proj", "model.layers.36.mlp.experts.71.up_proj", "model.layers.36.mlp.experts.72.up_proj", "model.layers.36.mlp.experts.73.up_proj", "model.layers.36.mlp.experts.74.up_proj", "model.layers.36.mlp.experts.75.up_proj", "model.layers.36.mlp.experts.76.up_proj", "model.layers.36.mlp.experts.77.up_proj", "model.layers.36.mlp.experts.78.up_proj", "model.layers.36.mlp.experts.79.up_proj", "model.layers.36.mlp.experts.80.up_proj", "model.layers.36.mlp.experts.81.up_proj", "model.layers.36.mlp.experts.82.up_proj", "model.layers.36.mlp.experts.83.up_proj", "model.layers.36.mlp.experts.84.up_proj", "model.layers.36.mlp.experts.85.up_proj", "model.layers.36.mlp.experts.86.up_proj", "model.layers.36.mlp.experts.87.up_proj", "model.layers.36.mlp.experts.88.up_proj", "model.layers.36.mlp.experts.89.up_proj", "model.layers.36.mlp.experts.90.up_proj", "model.layers.36.mlp.experts.91.up_proj", "model.layers.36.mlp.experts.92.up_proj", "model.layers.36.mlp.experts.93.up_proj", "model.layers.36.mlp.experts.94.up_proj", "model.layers.36.mlp.experts.95.up_proj", "model.layers.36.mlp.experts.96.up_proj", "model.layers.36.mlp.experts.97.up_proj", "model.layers.36.mlp.experts.98.up_proj", "model.layers.36.mlp.experts.99.up_proj", "model.layers.36.mlp.experts.100.up_proj", "model.layers.36.mlp.experts.101.up_proj", "model.layers.36.mlp.experts.102.up_proj", "model.layers.36.mlp.experts.103.up_proj", "model.layers.36.mlp.experts.104.up_proj", "model.layers.36.mlp.experts.105.up_proj", "model.layers.36.mlp.experts.106.up_proj", "model.layers.36.mlp.experts.107.up_proj", "model.layers.36.mlp.experts.108.up_proj", "model.layers.36.mlp.experts.109.up_proj", "model.layers.36.mlp.experts.110.up_proj", "model.layers.36.mlp.experts.111.up_proj", "model.layers.36.mlp.experts.112.up_proj", "model.layers.36.mlp.experts.113.up_proj", "model.layers.36.mlp.experts.114.up_proj", "model.layers.36.mlp.experts.115.up_proj", "model.layers.36.mlp.experts.116.up_proj", "model.layers.36.mlp.experts.117.up_proj", "model.layers.36.mlp.experts.118.up_proj", "model.layers.36.mlp.experts.119.up_proj", "model.layers.36.mlp.experts.120.up_proj", "model.layers.36.mlp.experts.121.up_proj", "model.layers.36.mlp.experts.122.up_proj", "model.layers.36.mlp.experts.123.up_proj", "model.layers.36.mlp.experts.124.up_proj", "model.layers.36.mlp.experts.125.up_proj", "model.layers.36.mlp.experts.126.up_proj", "model.layers.36.mlp.experts.127.up_proj", "model.layers.36.mlp.experts.128.up_proj", "model.layers.36.mlp.experts.129.up_proj", "model.layers.36.mlp.experts.130.up_proj", "model.layers.36.mlp.experts.131.up_proj", "model.layers.36.mlp.experts.132.up_proj", "model.layers.36.mlp.experts.133.up_proj", "model.layers.36.mlp.experts.134.up_proj", "model.layers.36.mlp.experts.135.up_proj", "model.layers.36.mlp.experts.136.up_proj", "model.layers.36.mlp.experts.137.up_proj", "model.layers.36.mlp.experts.138.up_proj", "model.layers.36.mlp.experts.139.up_proj", "model.layers.36.mlp.experts.140.up_proj", "model.layers.36.mlp.experts.141.up_proj", "model.layers.36.mlp.experts.142.up_proj", "model.layers.36.mlp.experts.143.up_proj", "model.layers.36.mlp.experts.144.up_proj", "model.layers.36.mlp.experts.145.up_proj", "model.layers.36.mlp.experts.146.up_proj", "model.layers.36.mlp.experts.147.up_proj", "model.layers.36.mlp.experts.148.up_proj", "model.layers.36.mlp.experts.149.up_proj", "model.layers.36.mlp.experts.150.up_proj", "model.layers.36.mlp.experts.151.up_proj", "model.layers.36.mlp.experts.152.up_proj", "model.layers.36.mlp.experts.153.up_proj", "model.layers.36.mlp.experts.154.up_proj", "model.layers.36.mlp.experts.155.up_proj", "model.layers.36.mlp.experts.156.up_proj", "model.layers.36.mlp.experts.157.up_proj", "model.layers.36.mlp.experts.158.up_proj", "model.layers.36.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0009216360747814067, "dbits": 2516582400 } ] }, { "idx": 218, "layers": [ "model.layers.36.mlp.experts.0.down_proj", "model.layers.36.mlp.experts.1.down_proj", "model.layers.36.mlp.experts.2.down_proj", "model.layers.36.mlp.experts.3.down_proj", "model.layers.36.mlp.experts.4.down_proj", "model.layers.36.mlp.experts.5.down_proj", "model.layers.36.mlp.experts.6.down_proj", "model.layers.36.mlp.experts.7.down_proj", "model.layers.36.mlp.experts.8.down_proj", "model.layers.36.mlp.experts.9.down_proj", "model.layers.36.mlp.experts.10.down_proj", "model.layers.36.mlp.experts.11.down_proj", "model.layers.36.mlp.experts.12.down_proj", "model.layers.36.mlp.experts.13.down_proj", "model.layers.36.mlp.experts.14.down_proj", "model.layers.36.mlp.experts.15.down_proj", "model.layers.36.mlp.experts.16.down_proj", "model.layers.36.mlp.experts.17.down_proj", "model.layers.36.mlp.experts.18.down_proj", "model.layers.36.mlp.experts.19.down_proj", "model.layers.36.mlp.experts.20.down_proj", "model.layers.36.mlp.experts.21.down_proj", "model.layers.36.mlp.experts.22.down_proj", "model.layers.36.mlp.experts.23.down_proj", "model.layers.36.mlp.experts.24.down_proj", "model.layers.36.mlp.experts.25.down_proj", "model.layers.36.mlp.experts.26.down_proj", "model.layers.36.mlp.experts.27.down_proj", "model.layers.36.mlp.experts.28.down_proj", "model.layers.36.mlp.experts.29.down_proj", "model.layers.36.mlp.experts.30.down_proj", "model.layers.36.mlp.experts.31.down_proj", "model.layers.36.mlp.experts.32.down_proj", "model.layers.36.mlp.experts.33.down_proj", "model.layers.36.mlp.experts.34.down_proj", "model.layers.36.mlp.experts.35.down_proj", "model.layers.36.mlp.experts.36.down_proj", "model.layers.36.mlp.experts.37.down_proj", "model.layers.36.mlp.experts.38.down_proj", "model.layers.36.mlp.experts.39.down_proj", "model.layers.36.mlp.experts.40.down_proj", "model.layers.36.mlp.experts.41.down_proj", "model.layers.36.mlp.experts.42.down_proj", "model.layers.36.mlp.experts.43.down_proj", "model.layers.36.mlp.experts.44.down_proj", "model.layers.36.mlp.experts.45.down_proj", "model.layers.36.mlp.experts.46.down_proj", "model.layers.36.mlp.experts.47.down_proj", "model.layers.36.mlp.experts.48.down_proj", "model.layers.36.mlp.experts.49.down_proj", "model.layers.36.mlp.experts.50.down_proj", "model.layers.36.mlp.experts.51.down_proj", "model.layers.36.mlp.experts.52.down_proj", "model.layers.36.mlp.experts.53.down_proj", "model.layers.36.mlp.experts.54.down_proj", "model.layers.36.mlp.experts.55.down_proj", "model.layers.36.mlp.experts.56.down_proj", "model.layers.36.mlp.experts.57.down_proj", "model.layers.36.mlp.experts.58.down_proj", "model.layers.36.mlp.experts.59.down_proj", "model.layers.36.mlp.experts.60.down_proj", "model.layers.36.mlp.experts.61.down_proj", "model.layers.36.mlp.experts.62.down_proj", "model.layers.36.mlp.experts.63.down_proj", "model.layers.36.mlp.experts.64.down_proj", "model.layers.36.mlp.experts.65.down_proj", "model.layers.36.mlp.experts.66.down_proj", "model.layers.36.mlp.experts.67.down_proj", "model.layers.36.mlp.experts.68.down_proj", "model.layers.36.mlp.experts.69.down_proj", "model.layers.36.mlp.experts.70.down_proj", "model.layers.36.mlp.experts.71.down_proj", "model.layers.36.mlp.experts.72.down_proj", "model.layers.36.mlp.experts.73.down_proj", "model.layers.36.mlp.experts.74.down_proj", "model.layers.36.mlp.experts.75.down_proj", "model.layers.36.mlp.experts.76.down_proj", "model.layers.36.mlp.experts.77.down_proj", "model.layers.36.mlp.experts.78.down_proj", "model.layers.36.mlp.experts.79.down_proj", "model.layers.36.mlp.experts.80.down_proj", "model.layers.36.mlp.experts.81.down_proj", "model.layers.36.mlp.experts.82.down_proj", "model.layers.36.mlp.experts.83.down_proj", "model.layers.36.mlp.experts.84.down_proj", "model.layers.36.mlp.experts.85.down_proj", "model.layers.36.mlp.experts.86.down_proj", "model.layers.36.mlp.experts.87.down_proj", "model.layers.36.mlp.experts.88.down_proj", "model.layers.36.mlp.experts.89.down_proj", "model.layers.36.mlp.experts.90.down_proj", "model.layers.36.mlp.experts.91.down_proj", "model.layers.36.mlp.experts.92.down_proj", "model.layers.36.mlp.experts.93.down_proj", "model.layers.36.mlp.experts.94.down_proj", "model.layers.36.mlp.experts.95.down_proj", "model.layers.36.mlp.experts.96.down_proj", "model.layers.36.mlp.experts.97.down_proj", "model.layers.36.mlp.experts.98.down_proj", "model.layers.36.mlp.experts.99.down_proj", "model.layers.36.mlp.experts.100.down_proj", "model.layers.36.mlp.experts.101.down_proj", "model.layers.36.mlp.experts.102.down_proj", "model.layers.36.mlp.experts.103.down_proj", "model.layers.36.mlp.experts.104.down_proj", "model.layers.36.mlp.experts.105.down_proj", "model.layers.36.mlp.experts.106.down_proj", "model.layers.36.mlp.experts.107.down_proj", "model.layers.36.mlp.experts.108.down_proj", "model.layers.36.mlp.experts.109.down_proj", "model.layers.36.mlp.experts.110.down_proj", "model.layers.36.mlp.experts.111.down_proj", "model.layers.36.mlp.experts.112.down_proj", "model.layers.36.mlp.experts.113.down_proj", "model.layers.36.mlp.experts.114.down_proj", "model.layers.36.mlp.experts.115.down_proj", "model.layers.36.mlp.experts.116.down_proj", "model.layers.36.mlp.experts.117.down_proj", "model.layers.36.mlp.experts.118.down_proj", "model.layers.36.mlp.experts.119.down_proj", "model.layers.36.mlp.experts.120.down_proj", "model.layers.36.mlp.experts.121.down_proj", "model.layers.36.mlp.experts.122.down_proj", "model.layers.36.mlp.experts.123.down_proj", "model.layers.36.mlp.experts.124.down_proj", "model.layers.36.mlp.experts.125.down_proj", "model.layers.36.mlp.experts.126.down_proj", "model.layers.36.mlp.experts.127.down_proj", "model.layers.36.mlp.experts.128.down_proj", "model.layers.36.mlp.experts.129.down_proj", "model.layers.36.mlp.experts.130.down_proj", "model.layers.36.mlp.experts.131.down_proj", "model.layers.36.mlp.experts.132.down_proj", "model.layers.36.mlp.experts.133.down_proj", "model.layers.36.mlp.experts.134.down_proj", "model.layers.36.mlp.experts.135.down_proj", "model.layers.36.mlp.experts.136.down_proj", "model.layers.36.mlp.experts.137.down_proj", "model.layers.36.mlp.experts.138.down_proj", "model.layers.36.mlp.experts.139.down_proj", "model.layers.36.mlp.experts.140.down_proj", "model.layers.36.mlp.experts.141.down_proj", "model.layers.36.mlp.experts.142.down_proj", "model.layers.36.mlp.experts.143.down_proj", "model.layers.36.mlp.experts.144.down_proj", "model.layers.36.mlp.experts.145.down_proj", "model.layers.36.mlp.experts.146.down_proj", "model.layers.36.mlp.experts.147.down_proj", "model.layers.36.mlp.experts.148.down_proj", "model.layers.36.mlp.experts.149.down_proj", "model.layers.36.mlp.experts.150.down_proj", "model.layers.36.mlp.experts.151.down_proj", "model.layers.36.mlp.experts.152.down_proj", "model.layers.36.mlp.experts.153.down_proj", "model.layers.36.mlp.experts.154.down_proj", "model.layers.36.mlp.experts.155.down_proj", "model.layers.36.mlp.experts.156.down_proj", "model.layers.36.mlp.experts.157.down_proj", "model.layers.36.mlp.experts.158.down_proj", "model.layers.36.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002906306087970767, "dbits": 1258291200 } ] }, { "idx": 219, "layers": [ "model.layers.37.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0024503044784069394, "dbits": 62914560 } ] }, { "idx": 220, "layers": [ "model.layers.37.self_attn.k_proj", "model.layers.37.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0033378757536411396, "dbits": 10485760 } ] }, { "idx": 221, "layers": [ "model.layers.37.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0010030522942542919, "dbits": 62914560 } ] }, { "idx": 222, "layers": [ "model.layers.37.mlp.shared_experts.gate_proj", "model.layers.37.mlp.shared_experts.up_proj", "model.layers.37.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.004523053020238865, "dbits": 23592960 } ] }, { "idx": 223, "layers": [ "model.layers.37.mlp.experts.0.gate_proj", "model.layers.37.mlp.experts.1.gate_proj", "model.layers.37.mlp.experts.2.gate_proj", "model.layers.37.mlp.experts.3.gate_proj", "model.layers.37.mlp.experts.4.gate_proj", "model.layers.37.mlp.experts.5.gate_proj", "model.layers.37.mlp.experts.6.gate_proj", "model.layers.37.mlp.experts.7.gate_proj", "model.layers.37.mlp.experts.8.gate_proj", "model.layers.37.mlp.experts.9.gate_proj", "model.layers.37.mlp.experts.10.gate_proj", "model.layers.37.mlp.experts.11.gate_proj", "model.layers.37.mlp.experts.12.gate_proj", "model.layers.37.mlp.experts.13.gate_proj", "model.layers.37.mlp.experts.14.gate_proj", "model.layers.37.mlp.experts.15.gate_proj", "model.layers.37.mlp.experts.16.gate_proj", "model.layers.37.mlp.experts.17.gate_proj", "model.layers.37.mlp.experts.18.gate_proj", "model.layers.37.mlp.experts.19.gate_proj", "model.layers.37.mlp.experts.20.gate_proj", "model.layers.37.mlp.experts.21.gate_proj", "model.layers.37.mlp.experts.22.gate_proj", "model.layers.37.mlp.experts.23.gate_proj", "model.layers.37.mlp.experts.24.gate_proj", "model.layers.37.mlp.experts.25.gate_proj", "model.layers.37.mlp.experts.26.gate_proj", "model.layers.37.mlp.experts.27.gate_proj", "model.layers.37.mlp.experts.28.gate_proj", "model.layers.37.mlp.experts.29.gate_proj", "model.layers.37.mlp.experts.30.gate_proj", "model.layers.37.mlp.experts.31.gate_proj", "model.layers.37.mlp.experts.32.gate_proj", "model.layers.37.mlp.experts.33.gate_proj", "model.layers.37.mlp.experts.34.gate_proj", "model.layers.37.mlp.experts.35.gate_proj", "model.layers.37.mlp.experts.36.gate_proj", "model.layers.37.mlp.experts.37.gate_proj", "model.layers.37.mlp.experts.38.gate_proj", "model.layers.37.mlp.experts.39.gate_proj", "model.layers.37.mlp.experts.40.gate_proj", "model.layers.37.mlp.experts.41.gate_proj", "model.layers.37.mlp.experts.42.gate_proj", "model.layers.37.mlp.experts.43.gate_proj", "model.layers.37.mlp.experts.44.gate_proj", "model.layers.37.mlp.experts.45.gate_proj", "model.layers.37.mlp.experts.46.gate_proj", "model.layers.37.mlp.experts.47.gate_proj", "model.layers.37.mlp.experts.48.gate_proj", "model.layers.37.mlp.experts.49.gate_proj", "model.layers.37.mlp.experts.50.gate_proj", "model.layers.37.mlp.experts.51.gate_proj", "model.layers.37.mlp.experts.52.gate_proj", "model.layers.37.mlp.experts.53.gate_proj", "model.layers.37.mlp.experts.54.gate_proj", "model.layers.37.mlp.experts.55.gate_proj", "model.layers.37.mlp.experts.56.gate_proj", "model.layers.37.mlp.experts.57.gate_proj", "model.layers.37.mlp.experts.58.gate_proj", "model.layers.37.mlp.experts.59.gate_proj", "model.layers.37.mlp.experts.60.gate_proj", "model.layers.37.mlp.experts.61.gate_proj", "model.layers.37.mlp.experts.62.gate_proj", "model.layers.37.mlp.experts.63.gate_proj", "model.layers.37.mlp.experts.64.gate_proj", "model.layers.37.mlp.experts.65.gate_proj", "model.layers.37.mlp.experts.66.gate_proj", "model.layers.37.mlp.experts.67.gate_proj", "model.layers.37.mlp.experts.68.gate_proj", "model.layers.37.mlp.experts.69.gate_proj", "model.layers.37.mlp.experts.70.gate_proj", "model.layers.37.mlp.experts.71.gate_proj", "model.layers.37.mlp.experts.72.gate_proj", "model.layers.37.mlp.experts.73.gate_proj", "model.layers.37.mlp.experts.74.gate_proj", "model.layers.37.mlp.experts.75.gate_proj", "model.layers.37.mlp.experts.76.gate_proj", "model.layers.37.mlp.experts.77.gate_proj", "model.layers.37.mlp.experts.78.gate_proj", "model.layers.37.mlp.experts.79.gate_proj", "model.layers.37.mlp.experts.80.gate_proj", "model.layers.37.mlp.experts.81.gate_proj", "model.layers.37.mlp.experts.82.gate_proj", "model.layers.37.mlp.experts.83.gate_proj", "model.layers.37.mlp.experts.84.gate_proj", "model.layers.37.mlp.experts.85.gate_proj", "model.layers.37.mlp.experts.86.gate_proj", "model.layers.37.mlp.experts.87.gate_proj", "model.layers.37.mlp.experts.88.gate_proj", "model.layers.37.mlp.experts.89.gate_proj", "model.layers.37.mlp.experts.90.gate_proj", "model.layers.37.mlp.experts.91.gate_proj", "model.layers.37.mlp.experts.92.gate_proj", "model.layers.37.mlp.experts.93.gate_proj", "model.layers.37.mlp.experts.94.gate_proj", "model.layers.37.mlp.experts.95.gate_proj", "model.layers.37.mlp.experts.96.gate_proj", "model.layers.37.mlp.experts.97.gate_proj", "model.layers.37.mlp.experts.98.gate_proj", "model.layers.37.mlp.experts.99.gate_proj", "model.layers.37.mlp.experts.100.gate_proj", "model.layers.37.mlp.experts.101.gate_proj", "model.layers.37.mlp.experts.102.gate_proj", "model.layers.37.mlp.experts.103.gate_proj", "model.layers.37.mlp.experts.104.gate_proj", "model.layers.37.mlp.experts.105.gate_proj", "model.layers.37.mlp.experts.106.gate_proj", "model.layers.37.mlp.experts.107.gate_proj", "model.layers.37.mlp.experts.108.gate_proj", "model.layers.37.mlp.experts.109.gate_proj", "model.layers.37.mlp.experts.110.gate_proj", "model.layers.37.mlp.experts.111.gate_proj", "model.layers.37.mlp.experts.112.gate_proj", "model.layers.37.mlp.experts.113.gate_proj", "model.layers.37.mlp.experts.114.gate_proj", "model.layers.37.mlp.experts.115.gate_proj", "model.layers.37.mlp.experts.116.gate_proj", "model.layers.37.mlp.experts.117.gate_proj", "model.layers.37.mlp.experts.118.gate_proj", "model.layers.37.mlp.experts.119.gate_proj", "model.layers.37.mlp.experts.120.gate_proj", "model.layers.37.mlp.experts.121.gate_proj", "model.layers.37.mlp.experts.122.gate_proj", "model.layers.37.mlp.experts.123.gate_proj", "model.layers.37.mlp.experts.124.gate_proj", "model.layers.37.mlp.experts.125.gate_proj", "model.layers.37.mlp.experts.126.gate_proj", "model.layers.37.mlp.experts.127.gate_proj", "model.layers.37.mlp.experts.128.gate_proj", "model.layers.37.mlp.experts.129.gate_proj", "model.layers.37.mlp.experts.130.gate_proj", "model.layers.37.mlp.experts.131.gate_proj", "model.layers.37.mlp.experts.132.gate_proj", "model.layers.37.mlp.experts.133.gate_proj", "model.layers.37.mlp.experts.134.gate_proj", "model.layers.37.mlp.experts.135.gate_proj", "model.layers.37.mlp.experts.136.gate_proj", "model.layers.37.mlp.experts.137.gate_proj", "model.layers.37.mlp.experts.138.gate_proj", "model.layers.37.mlp.experts.139.gate_proj", "model.layers.37.mlp.experts.140.gate_proj", "model.layers.37.mlp.experts.141.gate_proj", "model.layers.37.mlp.experts.142.gate_proj", "model.layers.37.mlp.experts.143.gate_proj", "model.layers.37.mlp.experts.144.gate_proj", "model.layers.37.mlp.experts.145.gate_proj", "model.layers.37.mlp.experts.146.gate_proj", "model.layers.37.mlp.experts.147.gate_proj", "model.layers.37.mlp.experts.148.gate_proj", "model.layers.37.mlp.experts.149.gate_proj", "model.layers.37.mlp.experts.150.gate_proj", "model.layers.37.mlp.experts.151.gate_proj", "model.layers.37.mlp.experts.152.gate_proj", "model.layers.37.mlp.experts.153.gate_proj", "model.layers.37.mlp.experts.154.gate_proj", "model.layers.37.mlp.experts.155.gate_proj", "model.layers.37.mlp.experts.156.gate_proj", "model.layers.37.mlp.experts.157.gate_proj", "model.layers.37.mlp.experts.158.gate_proj", "model.layers.37.mlp.experts.159.gate_proj", "model.layers.37.mlp.experts.0.up_proj", "model.layers.37.mlp.experts.1.up_proj", "model.layers.37.mlp.experts.2.up_proj", "model.layers.37.mlp.experts.3.up_proj", "model.layers.37.mlp.experts.4.up_proj", "model.layers.37.mlp.experts.5.up_proj", "model.layers.37.mlp.experts.6.up_proj", "model.layers.37.mlp.experts.7.up_proj", "model.layers.37.mlp.experts.8.up_proj", "model.layers.37.mlp.experts.9.up_proj", "model.layers.37.mlp.experts.10.up_proj", "model.layers.37.mlp.experts.11.up_proj", "model.layers.37.mlp.experts.12.up_proj", "model.layers.37.mlp.experts.13.up_proj", "model.layers.37.mlp.experts.14.up_proj", "model.layers.37.mlp.experts.15.up_proj", "model.layers.37.mlp.experts.16.up_proj", "model.layers.37.mlp.experts.17.up_proj", "model.layers.37.mlp.experts.18.up_proj", "model.layers.37.mlp.experts.19.up_proj", "model.layers.37.mlp.experts.20.up_proj", "model.layers.37.mlp.experts.21.up_proj", "model.layers.37.mlp.experts.22.up_proj", "model.layers.37.mlp.experts.23.up_proj", "model.layers.37.mlp.experts.24.up_proj", "model.layers.37.mlp.experts.25.up_proj", "model.layers.37.mlp.experts.26.up_proj", "model.layers.37.mlp.experts.27.up_proj", "model.layers.37.mlp.experts.28.up_proj", "model.layers.37.mlp.experts.29.up_proj", "model.layers.37.mlp.experts.30.up_proj", "model.layers.37.mlp.experts.31.up_proj", "model.layers.37.mlp.experts.32.up_proj", "model.layers.37.mlp.experts.33.up_proj", "model.layers.37.mlp.experts.34.up_proj", "model.layers.37.mlp.experts.35.up_proj", "model.layers.37.mlp.experts.36.up_proj", "model.layers.37.mlp.experts.37.up_proj", "model.layers.37.mlp.experts.38.up_proj", "model.layers.37.mlp.experts.39.up_proj", "model.layers.37.mlp.experts.40.up_proj", "model.layers.37.mlp.experts.41.up_proj", "model.layers.37.mlp.experts.42.up_proj", "model.layers.37.mlp.experts.43.up_proj", "model.layers.37.mlp.experts.44.up_proj", "model.layers.37.mlp.experts.45.up_proj", "model.layers.37.mlp.experts.46.up_proj", "model.layers.37.mlp.experts.47.up_proj", "model.layers.37.mlp.experts.48.up_proj", "model.layers.37.mlp.experts.49.up_proj", "model.layers.37.mlp.experts.50.up_proj", "model.layers.37.mlp.experts.51.up_proj", "model.layers.37.mlp.experts.52.up_proj", "model.layers.37.mlp.experts.53.up_proj", "model.layers.37.mlp.experts.54.up_proj", "model.layers.37.mlp.experts.55.up_proj", "model.layers.37.mlp.experts.56.up_proj", "model.layers.37.mlp.experts.57.up_proj", "model.layers.37.mlp.experts.58.up_proj", "model.layers.37.mlp.experts.59.up_proj", "model.layers.37.mlp.experts.60.up_proj", "model.layers.37.mlp.experts.61.up_proj", "model.layers.37.mlp.experts.62.up_proj", "model.layers.37.mlp.experts.63.up_proj", "model.layers.37.mlp.experts.64.up_proj", "model.layers.37.mlp.experts.65.up_proj", "model.layers.37.mlp.experts.66.up_proj", "model.layers.37.mlp.experts.67.up_proj", "model.layers.37.mlp.experts.68.up_proj", "model.layers.37.mlp.experts.69.up_proj", "model.layers.37.mlp.experts.70.up_proj", "model.layers.37.mlp.experts.71.up_proj", "model.layers.37.mlp.experts.72.up_proj", "model.layers.37.mlp.experts.73.up_proj", "model.layers.37.mlp.experts.74.up_proj", "model.layers.37.mlp.experts.75.up_proj", "model.layers.37.mlp.experts.76.up_proj", "model.layers.37.mlp.experts.77.up_proj", "model.layers.37.mlp.experts.78.up_proj", "model.layers.37.mlp.experts.79.up_proj", "model.layers.37.mlp.experts.80.up_proj", "model.layers.37.mlp.experts.81.up_proj", "model.layers.37.mlp.experts.82.up_proj", "model.layers.37.mlp.experts.83.up_proj", "model.layers.37.mlp.experts.84.up_proj", "model.layers.37.mlp.experts.85.up_proj", "model.layers.37.mlp.experts.86.up_proj", "model.layers.37.mlp.experts.87.up_proj", "model.layers.37.mlp.experts.88.up_proj", "model.layers.37.mlp.experts.89.up_proj", "model.layers.37.mlp.experts.90.up_proj", "model.layers.37.mlp.experts.91.up_proj", "model.layers.37.mlp.experts.92.up_proj", "model.layers.37.mlp.experts.93.up_proj", "model.layers.37.mlp.experts.94.up_proj", "model.layers.37.mlp.experts.95.up_proj", "model.layers.37.mlp.experts.96.up_proj", "model.layers.37.mlp.experts.97.up_proj", "model.layers.37.mlp.experts.98.up_proj", "model.layers.37.mlp.experts.99.up_proj", "model.layers.37.mlp.experts.100.up_proj", "model.layers.37.mlp.experts.101.up_proj", "model.layers.37.mlp.experts.102.up_proj", "model.layers.37.mlp.experts.103.up_proj", "model.layers.37.mlp.experts.104.up_proj", "model.layers.37.mlp.experts.105.up_proj", "model.layers.37.mlp.experts.106.up_proj", "model.layers.37.mlp.experts.107.up_proj", "model.layers.37.mlp.experts.108.up_proj", "model.layers.37.mlp.experts.109.up_proj", "model.layers.37.mlp.experts.110.up_proj", "model.layers.37.mlp.experts.111.up_proj", "model.layers.37.mlp.experts.112.up_proj", "model.layers.37.mlp.experts.113.up_proj", "model.layers.37.mlp.experts.114.up_proj", "model.layers.37.mlp.experts.115.up_proj", "model.layers.37.mlp.experts.116.up_proj", "model.layers.37.mlp.experts.117.up_proj", "model.layers.37.mlp.experts.118.up_proj", "model.layers.37.mlp.experts.119.up_proj", "model.layers.37.mlp.experts.120.up_proj", "model.layers.37.mlp.experts.121.up_proj", "model.layers.37.mlp.experts.122.up_proj", "model.layers.37.mlp.experts.123.up_proj", "model.layers.37.mlp.experts.124.up_proj", "model.layers.37.mlp.experts.125.up_proj", "model.layers.37.mlp.experts.126.up_proj", "model.layers.37.mlp.experts.127.up_proj", "model.layers.37.mlp.experts.128.up_proj", "model.layers.37.mlp.experts.129.up_proj", "model.layers.37.mlp.experts.130.up_proj", "model.layers.37.mlp.experts.131.up_proj", "model.layers.37.mlp.experts.132.up_proj", "model.layers.37.mlp.experts.133.up_proj", "model.layers.37.mlp.experts.134.up_proj", "model.layers.37.mlp.experts.135.up_proj", "model.layers.37.mlp.experts.136.up_proj", "model.layers.37.mlp.experts.137.up_proj", "model.layers.37.mlp.experts.138.up_proj", "model.layers.37.mlp.experts.139.up_proj", "model.layers.37.mlp.experts.140.up_proj", "model.layers.37.mlp.experts.141.up_proj", "model.layers.37.mlp.experts.142.up_proj", "model.layers.37.mlp.experts.143.up_proj", "model.layers.37.mlp.experts.144.up_proj", "model.layers.37.mlp.experts.145.up_proj", "model.layers.37.mlp.experts.146.up_proj", "model.layers.37.mlp.experts.147.up_proj", "model.layers.37.mlp.experts.148.up_proj", "model.layers.37.mlp.experts.149.up_proj", "model.layers.37.mlp.experts.150.up_proj", "model.layers.37.mlp.experts.151.up_proj", "model.layers.37.mlp.experts.152.up_proj", "model.layers.37.mlp.experts.153.up_proj", "model.layers.37.mlp.experts.154.up_proj", "model.layers.37.mlp.experts.155.up_proj", "model.layers.37.mlp.experts.156.up_proj", "model.layers.37.mlp.experts.157.up_proj", "model.layers.37.mlp.experts.158.up_proj", "model.layers.37.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0037399917840957864, "dbits": 2516582400 } ] }, { "idx": 224, "layers": [ "model.layers.37.mlp.experts.0.down_proj", "model.layers.37.mlp.experts.1.down_proj", "model.layers.37.mlp.experts.2.down_proj", "model.layers.37.mlp.experts.3.down_proj", "model.layers.37.mlp.experts.4.down_proj", "model.layers.37.mlp.experts.5.down_proj", "model.layers.37.mlp.experts.6.down_proj", "model.layers.37.mlp.experts.7.down_proj", "model.layers.37.mlp.experts.8.down_proj", "model.layers.37.mlp.experts.9.down_proj", "model.layers.37.mlp.experts.10.down_proj", "model.layers.37.mlp.experts.11.down_proj", "model.layers.37.mlp.experts.12.down_proj", "model.layers.37.mlp.experts.13.down_proj", "model.layers.37.mlp.experts.14.down_proj", "model.layers.37.mlp.experts.15.down_proj", "model.layers.37.mlp.experts.16.down_proj", "model.layers.37.mlp.experts.17.down_proj", "model.layers.37.mlp.experts.18.down_proj", "model.layers.37.mlp.experts.19.down_proj", "model.layers.37.mlp.experts.20.down_proj", "model.layers.37.mlp.experts.21.down_proj", "model.layers.37.mlp.experts.22.down_proj", "model.layers.37.mlp.experts.23.down_proj", "model.layers.37.mlp.experts.24.down_proj", "model.layers.37.mlp.experts.25.down_proj", "model.layers.37.mlp.experts.26.down_proj", "model.layers.37.mlp.experts.27.down_proj", "model.layers.37.mlp.experts.28.down_proj", "model.layers.37.mlp.experts.29.down_proj", "model.layers.37.mlp.experts.30.down_proj", "model.layers.37.mlp.experts.31.down_proj", "model.layers.37.mlp.experts.32.down_proj", "model.layers.37.mlp.experts.33.down_proj", "model.layers.37.mlp.experts.34.down_proj", "model.layers.37.mlp.experts.35.down_proj", "model.layers.37.mlp.experts.36.down_proj", "model.layers.37.mlp.experts.37.down_proj", "model.layers.37.mlp.experts.38.down_proj", "model.layers.37.mlp.experts.39.down_proj", "model.layers.37.mlp.experts.40.down_proj", "model.layers.37.mlp.experts.41.down_proj", "model.layers.37.mlp.experts.42.down_proj", "model.layers.37.mlp.experts.43.down_proj", "model.layers.37.mlp.experts.44.down_proj", "model.layers.37.mlp.experts.45.down_proj", "model.layers.37.mlp.experts.46.down_proj", "model.layers.37.mlp.experts.47.down_proj", "model.layers.37.mlp.experts.48.down_proj", "model.layers.37.mlp.experts.49.down_proj", "model.layers.37.mlp.experts.50.down_proj", "model.layers.37.mlp.experts.51.down_proj", "model.layers.37.mlp.experts.52.down_proj", "model.layers.37.mlp.experts.53.down_proj", "model.layers.37.mlp.experts.54.down_proj", "model.layers.37.mlp.experts.55.down_proj", "model.layers.37.mlp.experts.56.down_proj", "model.layers.37.mlp.experts.57.down_proj", "model.layers.37.mlp.experts.58.down_proj", "model.layers.37.mlp.experts.59.down_proj", "model.layers.37.mlp.experts.60.down_proj", "model.layers.37.mlp.experts.61.down_proj", "model.layers.37.mlp.experts.62.down_proj", "model.layers.37.mlp.experts.63.down_proj", "model.layers.37.mlp.experts.64.down_proj", "model.layers.37.mlp.experts.65.down_proj", "model.layers.37.mlp.experts.66.down_proj", "model.layers.37.mlp.experts.67.down_proj", "model.layers.37.mlp.experts.68.down_proj", "model.layers.37.mlp.experts.69.down_proj", "model.layers.37.mlp.experts.70.down_proj", "model.layers.37.mlp.experts.71.down_proj", "model.layers.37.mlp.experts.72.down_proj", "model.layers.37.mlp.experts.73.down_proj", "model.layers.37.mlp.experts.74.down_proj", "model.layers.37.mlp.experts.75.down_proj", "model.layers.37.mlp.experts.76.down_proj", "model.layers.37.mlp.experts.77.down_proj", "model.layers.37.mlp.experts.78.down_proj", "model.layers.37.mlp.experts.79.down_proj", "model.layers.37.mlp.experts.80.down_proj", "model.layers.37.mlp.experts.81.down_proj", "model.layers.37.mlp.experts.82.down_proj", "model.layers.37.mlp.experts.83.down_proj", "model.layers.37.mlp.experts.84.down_proj", "model.layers.37.mlp.experts.85.down_proj", "model.layers.37.mlp.experts.86.down_proj", "model.layers.37.mlp.experts.87.down_proj", "model.layers.37.mlp.experts.88.down_proj", "model.layers.37.mlp.experts.89.down_proj", "model.layers.37.mlp.experts.90.down_proj", "model.layers.37.mlp.experts.91.down_proj", "model.layers.37.mlp.experts.92.down_proj", "model.layers.37.mlp.experts.93.down_proj", "model.layers.37.mlp.experts.94.down_proj", "model.layers.37.mlp.experts.95.down_proj", "model.layers.37.mlp.experts.96.down_proj", "model.layers.37.mlp.experts.97.down_proj", "model.layers.37.mlp.experts.98.down_proj", "model.layers.37.mlp.experts.99.down_proj", "model.layers.37.mlp.experts.100.down_proj", "model.layers.37.mlp.experts.101.down_proj", "model.layers.37.mlp.experts.102.down_proj", "model.layers.37.mlp.experts.103.down_proj", "model.layers.37.mlp.experts.104.down_proj", "model.layers.37.mlp.experts.105.down_proj", "model.layers.37.mlp.experts.106.down_proj", "model.layers.37.mlp.experts.107.down_proj", "model.layers.37.mlp.experts.108.down_proj", "model.layers.37.mlp.experts.109.down_proj", "model.layers.37.mlp.experts.110.down_proj", "model.layers.37.mlp.experts.111.down_proj", "model.layers.37.mlp.experts.112.down_proj", "model.layers.37.mlp.experts.113.down_proj", "model.layers.37.mlp.experts.114.down_proj", "model.layers.37.mlp.experts.115.down_proj", "model.layers.37.mlp.experts.116.down_proj", "model.layers.37.mlp.experts.117.down_proj", "model.layers.37.mlp.experts.118.down_proj", "model.layers.37.mlp.experts.119.down_proj", "model.layers.37.mlp.experts.120.down_proj", "model.layers.37.mlp.experts.121.down_proj", "model.layers.37.mlp.experts.122.down_proj", "model.layers.37.mlp.experts.123.down_proj", "model.layers.37.mlp.experts.124.down_proj", "model.layers.37.mlp.experts.125.down_proj", "model.layers.37.mlp.experts.126.down_proj", "model.layers.37.mlp.experts.127.down_proj", "model.layers.37.mlp.experts.128.down_proj", "model.layers.37.mlp.experts.129.down_proj", "model.layers.37.mlp.experts.130.down_proj", "model.layers.37.mlp.experts.131.down_proj", "model.layers.37.mlp.experts.132.down_proj", "model.layers.37.mlp.experts.133.down_proj", "model.layers.37.mlp.experts.134.down_proj", "model.layers.37.mlp.experts.135.down_proj", "model.layers.37.mlp.experts.136.down_proj", "model.layers.37.mlp.experts.137.down_proj", "model.layers.37.mlp.experts.138.down_proj", "model.layers.37.mlp.experts.139.down_proj", "model.layers.37.mlp.experts.140.down_proj", "model.layers.37.mlp.experts.141.down_proj", "model.layers.37.mlp.experts.142.down_proj", "model.layers.37.mlp.experts.143.down_proj", "model.layers.37.mlp.experts.144.down_proj", "model.layers.37.mlp.experts.145.down_proj", "model.layers.37.mlp.experts.146.down_proj", "model.layers.37.mlp.experts.147.down_proj", "model.layers.37.mlp.experts.148.down_proj", "model.layers.37.mlp.experts.149.down_proj", "model.layers.37.mlp.experts.150.down_proj", "model.layers.37.mlp.experts.151.down_proj", "model.layers.37.mlp.experts.152.down_proj", "model.layers.37.mlp.experts.153.down_proj", "model.layers.37.mlp.experts.154.down_proj", "model.layers.37.mlp.experts.155.down_proj", "model.layers.37.mlp.experts.156.down_proj", "model.layers.37.mlp.experts.157.down_proj", "model.layers.37.mlp.experts.158.down_proj", "model.layers.37.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0020439669489860646, "dbits": 1258291200 } ] }, { "idx": 225, "layers": [ "model.layers.38.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0015438966453075187, "dbits": 62914560 } ] }, { "idx": 226, "layers": [ "model.layers.38.self_attn.k_proj", "model.layers.38.self_attn.v_proj" ], "candidates": [ { "dkld": -0.005437389761209499, "dbits": 10485760 } ] }, { "idx": 227, "layers": [ "model.layers.38.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0030263110995292775, "dbits": 62914560 } ] }, { "idx": 228, "layers": [ "model.layers.38.mlp.shared_experts.gate_proj", "model.layers.38.mlp.shared_experts.up_proj", "model.layers.38.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0047277376055717135, "dbits": 23592960 } ] }, { "idx": 229, "layers": [ "model.layers.38.mlp.experts.0.gate_proj", "model.layers.38.mlp.experts.1.gate_proj", "model.layers.38.mlp.experts.2.gate_proj", "model.layers.38.mlp.experts.3.gate_proj", "model.layers.38.mlp.experts.4.gate_proj", "model.layers.38.mlp.experts.5.gate_proj", "model.layers.38.mlp.experts.6.gate_proj", "model.layers.38.mlp.experts.7.gate_proj", "model.layers.38.mlp.experts.8.gate_proj", "model.layers.38.mlp.experts.9.gate_proj", "model.layers.38.mlp.experts.10.gate_proj", "model.layers.38.mlp.experts.11.gate_proj", "model.layers.38.mlp.experts.12.gate_proj", "model.layers.38.mlp.experts.13.gate_proj", "model.layers.38.mlp.experts.14.gate_proj", "model.layers.38.mlp.experts.15.gate_proj", "model.layers.38.mlp.experts.16.gate_proj", "model.layers.38.mlp.experts.17.gate_proj", "model.layers.38.mlp.experts.18.gate_proj", "model.layers.38.mlp.experts.19.gate_proj", "model.layers.38.mlp.experts.20.gate_proj", "model.layers.38.mlp.experts.21.gate_proj", "model.layers.38.mlp.experts.22.gate_proj", "model.layers.38.mlp.experts.23.gate_proj", "model.layers.38.mlp.experts.24.gate_proj", "model.layers.38.mlp.experts.25.gate_proj", "model.layers.38.mlp.experts.26.gate_proj", "model.layers.38.mlp.experts.27.gate_proj", "model.layers.38.mlp.experts.28.gate_proj", "model.layers.38.mlp.experts.29.gate_proj", "model.layers.38.mlp.experts.30.gate_proj", "model.layers.38.mlp.experts.31.gate_proj", "model.layers.38.mlp.experts.32.gate_proj", "model.layers.38.mlp.experts.33.gate_proj", "model.layers.38.mlp.experts.34.gate_proj", "model.layers.38.mlp.experts.35.gate_proj", "model.layers.38.mlp.experts.36.gate_proj", "model.layers.38.mlp.experts.37.gate_proj", "model.layers.38.mlp.experts.38.gate_proj", "model.layers.38.mlp.experts.39.gate_proj", "model.layers.38.mlp.experts.40.gate_proj", "model.layers.38.mlp.experts.41.gate_proj", "model.layers.38.mlp.experts.42.gate_proj", "model.layers.38.mlp.experts.43.gate_proj", "model.layers.38.mlp.experts.44.gate_proj", "model.layers.38.mlp.experts.45.gate_proj", "model.layers.38.mlp.experts.46.gate_proj", "model.layers.38.mlp.experts.47.gate_proj", "model.layers.38.mlp.experts.48.gate_proj", "model.layers.38.mlp.experts.49.gate_proj", "model.layers.38.mlp.experts.50.gate_proj", "model.layers.38.mlp.experts.51.gate_proj", "model.layers.38.mlp.experts.52.gate_proj", "model.layers.38.mlp.experts.53.gate_proj", "model.layers.38.mlp.experts.54.gate_proj", "model.layers.38.mlp.experts.55.gate_proj", "model.layers.38.mlp.experts.56.gate_proj", "model.layers.38.mlp.experts.57.gate_proj", "model.layers.38.mlp.experts.58.gate_proj", "model.layers.38.mlp.experts.59.gate_proj", "model.layers.38.mlp.experts.60.gate_proj", "model.layers.38.mlp.experts.61.gate_proj", "model.layers.38.mlp.experts.62.gate_proj", "model.layers.38.mlp.experts.63.gate_proj", "model.layers.38.mlp.experts.64.gate_proj", "model.layers.38.mlp.experts.65.gate_proj", "model.layers.38.mlp.experts.66.gate_proj", "model.layers.38.mlp.experts.67.gate_proj", "model.layers.38.mlp.experts.68.gate_proj", "model.layers.38.mlp.experts.69.gate_proj", "model.layers.38.mlp.experts.70.gate_proj", "model.layers.38.mlp.experts.71.gate_proj", "model.layers.38.mlp.experts.72.gate_proj", "model.layers.38.mlp.experts.73.gate_proj", "model.layers.38.mlp.experts.74.gate_proj", "model.layers.38.mlp.experts.75.gate_proj", "model.layers.38.mlp.experts.76.gate_proj", "model.layers.38.mlp.experts.77.gate_proj", "model.layers.38.mlp.experts.78.gate_proj", "model.layers.38.mlp.experts.79.gate_proj", "model.layers.38.mlp.experts.80.gate_proj", "model.layers.38.mlp.experts.81.gate_proj", "model.layers.38.mlp.experts.82.gate_proj", "model.layers.38.mlp.experts.83.gate_proj", "model.layers.38.mlp.experts.84.gate_proj", "model.layers.38.mlp.experts.85.gate_proj", "model.layers.38.mlp.experts.86.gate_proj", "model.layers.38.mlp.experts.87.gate_proj", "model.layers.38.mlp.experts.88.gate_proj", "model.layers.38.mlp.experts.89.gate_proj", "model.layers.38.mlp.experts.90.gate_proj", "model.layers.38.mlp.experts.91.gate_proj", "model.layers.38.mlp.experts.92.gate_proj", "model.layers.38.mlp.experts.93.gate_proj", "model.layers.38.mlp.experts.94.gate_proj", "model.layers.38.mlp.experts.95.gate_proj", "model.layers.38.mlp.experts.96.gate_proj", "model.layers.38.mlp.experts.97.gate_proj", "model.layers.38.mlp.experts.98.gate_proj", "model.layers.38.mlp.experts.99.gate_proj", "model.layers.38.mlp.experts.100.gate_proj", "model.layers.38.mlp.experts.101.gate_proj", "model.layers.38.mlp.experts.102.gate_proj", "model.layers.38.mlp.experts.103.gate_proj", "model.layers.38.mlp.experts.104.gate_proj", "model.layers.38.mlp.experts.105.gate_proj", "model.layers.38.mlp.experts.106.gate_proj", "model.layers.38.mlp.experts.107.gate_proj", "model.layers.38.mlp.experts.108.gate_proj", "model.layers.38.mlp.experts.109.gate_proj", "model.layers.38.mlp.experts.110.gate_proj", "model.layers.38.mlp.experts.111.gate_proj", "model.layers.38.mlp.experts.112.gate_proj", "model.layers.38.mlp.experts.113.gate_proj", "model.layers.38.mlp.experts.114.gate_proj", "model.layers.38.mlp.experts.115.gate_proj", "model.layers.38.mlp.experts.116.gate_proj", "model.layers.38.mlp.experts.117.gate_proj", "model.layers.38.mlp.experts.118.gate_proj", "model.layers.38.mlp.experts.119.gate_proj", "model.layers.38.mlp.experts.120.gate_proj", "model.layers.38.mlp.experts.121.gate_proj", "model.layers.38.mlp.experts.122.gate_proj", "model.layers.38.mlp.experts.123.gate_proj", "model.layers.38.mlp.experts.124.gate_proj", "model.layers.38.mlp.experts.125.gate_proj", "model.layers.38.mlp.experts.126.gate_proj", "model.layers.38.mlp.experts.127.gate_proj", "model.layers.38.mlp.experts.128.gate_proj", "model.layers.38.mlp.experts.129.gate_proj", "model.layers.38.mlp.experts.130.gate_proj", "model.layers.38.mlp.experts.131.gate_proj", "model.layers.38.mlp.experts.132.gate_proj", "model.layers.38.mlp.experts.133.gate_proj", "model.layers.38.mlp.experts.134.gate_proj", "model.layers.38.mlp.experts.135.gate_proj", "model.layers.38.mlp.experts.136.gate_proj", "model.layers.38.mlp.experts.137.gate_proj", "model.layers.38.mlp.experts.138.gate_proj", "model.layers.38.mlp.experts.139.gate_proj", "model.layers.38.mlp.experts.140.gate_proj", "model.layers.38.mlp.experts.141.gate_proj", "model.layers.38.mlp.experts.142.gate_proj", "model.layers.38.mlp.experts.143.gate_proj", "model.layers.38.mlp.experts.144.gate_proj", "model.layers.38.mlp.experts.145.gate_proj", "model.layers.38.mlp.experts.146.gate_proj", "model.layers.38.mlp.experts.147.gate_proj", "model.layers.38.mlp.experts.148.gate_proj", "model.layers.38.mlp.experts.149.gate_proj", "model.layers.38.mlp.experts.150.gate_proj", "model.layers.38.mlp.experts.151.gate_proj", "model.layers.38.mlp.experts.152.gate_proj", "model.layers.38.mlp.experts.153.gate_proj", "model.layers.38.mlp.experts.154.gate_proj", "model.layers.38.mlp.experts.155.gate_proj", "model.layers.38.mlp.experts.156.gate_proj", "model.layers.38.mlp.experts.157.gate_proj", "model.layers.38.mlp.experts.158.gate_proj", "model.layers.38.mlp.experts.159.gate_proj", "model.layers.38.mlp.experts.0.up_proj", "model.layers.38.mlp.experts.1.up_proj", "model.layers.38.mlp.experts.2.up_proj", "model.layers.38.mlp.experts.3.up_proj", "model.layers.38.mlp.experts.4.up_proj", "model.layers.38.mlp.experts.5.up_proj", "model.layers.38.mlp.experts.6.up_proj", "model.layers.38.mlp.experts.7.up_proj", "model.layers.38.mlp.experts.8.up_proj", "model.layers.38.mlp.experts.9.up_proj", "model.layers.38.mlp.experts.10.up_proj", "model.layers.38.mlp.experts.11.up_proj", "model.layers.38.mlp.experts.12.up_proj", "model.layers.38.mlp.experts.13.up_proj", "model.layers.38.mlp.experts.14.up_proj", "model.layers.38.mlp.experts.15.up_proj", "model.layers.38.mlp.experts.16.up_proj", "model.layers.38.mlp.experts.17.up_proj", "model.layers.38.mlp.experts.18.up_proj", "model.layers.38.mlp.experts.19.up_proj", "model.layers.38.mlp.experts.20.up_proj", "model.layers.38.mlp.experts.21.up_proj", "model.layers.38.mlp.experts.22.up_proj", "model.layers.38.mlp.experts.23.up_proj", "model.layers.38.mlp.experts.24.up_proj", "model.layers.38.mlp.experts.25.up_proj", "model.layers.38.mlp.experts.26.up_proj", "model.layers.38.mlp.experts.27.up_proj", "model.layers.38.mlp.experts.28.up_proj", "model.layers.38.mlp.experts.29.up_proj", "model.layers.38.mlp.experts.30.up_proj", "model.layers.38.mlp.experts.31.up_proj", "model.layers.38.mlp.experts.32.up_proj", "model.layers.38.mlp.experts.33.up_proj", "model.layers.38.mlp.experts.34.up_proj", "model.layers.38.mlp.experts.35.up_proj", "model.layers.38.mlp.experts.36.up_proj", "model.layers.38.mlp.experts.37.up_proj", "model.layers.38.mlp.experts.38.up_proj", "model.layers.38.mlp.experts.39.up_proj", "model.layers.38.mlp.experts.40.up_proj", "model.layers.38.mlp.experts.41.up_proj", "model.layers.38.mlp.experts.42.up_proj", "model.layers.38.mlp.experts.43.up_proj", "model.layers.38.mlp.experts.44.up_proj", "model.layers.38.mlp.experts.45.up_proj", "model.layers.38.mlp.experts.46.up_proj", "model.layers.38.mlp.experts.47.up_proj", "model.layers.38.mlp.experts.48.up_proj", "model.layers.38.mlp.experts.49.up_proj", "model.layers.38.mlp.experts.50.up_proj", "model.layers.38.mlp.experts.51.up_proj", "model.layers.38.mlp.experts.52.up_proj", "model.layers.38.mlp.experts.53.up_proj", "model.layers.38.mlp.experts.54.up_proj", "model.layers.38.mlp.experts.55.up_proj", "model.layers.38.mlp.experts.56.up_proj", "model.layers.38.mlp.experts.57.up_proj", "model.layers.38.mlp.experts.58.up_proj", "model.layers.38.mlp.experts.59.up_proj", "model.layers.38.mlp.experts.60.up_proj", "model.layers.38.mlp.experts.61.up_proj", "model.layers.38.mlp.experts.62.up_proj", "model.layers.38.mlp.experts.63.up_proj", "model.layers.38.mlp.experts.64.up_proj", "model.layers.38.mlp.experts.65.up_proj", "model.layers.38.mlp.experts.66.up_proj", "model.layers.38.mlp.experts.67.up_proj", "model.layers.38.mlp.experts.68.up_proj", "model.layers.38.mlp.experts.69.up_proj", "model.layers.38.mlp.experts.70.up_proj", "model.layers.38.mlp.experts.71.up_proj", "model.layers.38.mlp.experts.72.up_proj", "model.layers.38.mlp.experts.73.up_proj", "model.layers.38.mlp.experts.74.up_proj", "model.layers.38.mlp.experts.75.up_proj", "model.layers.38.mlp.experts.76.up_proj", "model.layers.38.mlp.experts.77.up_proj", "model.layers.38.mlp.experts.78.up_proj", "model.layers.38.mlp.experts.79.up_proj", "model.layers.38.mlp.experts.80.up_proj", "model.layers.38.mlp.experts.81.up_proj", "model.layers.38.mlp.experts.82.up_proj", "model.layers.38.mlp.experts.83.up_proj", "model.layers.38.mlp.experts.84.up_proj", "model.layers.38.mlp.experts.85.up_proj", "model.layers.38.mlp.experts.86.up_proj", "model.layers.38.mlp.experts.87.up_proj", "model.layers.38.mlp.experts.88.up_proj", "model.layers.38.mlp.experts.89.up_proj", "model.layers.38.mlp.experts.90.up_proj", "model.layers.38.mlp.experts.91.up_proj", "model.layers.38.mlp.experts.92.up_proj", "model.layers.38.mlp.experts.93.up_proj", "model.layers.38.mlp.experts.94.up_proj", "model.layers.38.mlp.experts.95.up_proj", "model.layers.38.mlp.experts.96.up_proj", "model.layers.38.mlp.experts.97.up_proj", "model.layers.38.mlp.experts.98.up_proj", "model.layers.38.mlp.experts.99.up_proj", "model.layers.38.mlp.experts.100.up_proj", "model.layers.38.mlp.experts.101.up_proj", "model.layers.38.mlp.experts.102.up_proj", "model.layers.38.mlp.experts.103.up_proj", "model.layers.38.mlp.experts.104.up_proj", "model.layers.38.mlp.experts.105.up_proj", "model.layers.38.mlp.experts.106.up_proj", "model.layers.38.mlp.experts.107.up_proj", "model.layers.38.mlp.experts.108.up_proj", "model.layers.38.mlp.experts.109.up_proj", "model.layers.38.mlp.experts.110.up_proj", "model.layers.38.mlp.experts.111.up_proj", "model.layers.38.mlp.experts.112.up_proj", "model.layers.38.mlp.experts.113.up_proj", "model.layers.38.mlp.experts.114.up_proj", "model.layers.38.mlp.experts.115.up_proj", "model.layers.38.mlp.experts.116.up_proj", "model.layers.38.mlp.experts.117.up_proj", "model.layers.38.mlp.experts.118.up_proj", "model.layers.38.mlp.experts.119.up_proj", "model.layers.38.mlp.experts.120.up_proj", "model.layers.38.mlp.experts.121.up_proj", "model.layers.38.mlp.experts.122.up_proj", "model.layers.38.mlp.experts.123.up_proj", "model.layers.38.mlp.experts.124.up_proj", "model.layers.38.mlp.experts.125.up_proj", "model.layers.38.mlp.experts.126.up_proj", "model.layers.38.mlp.experts.127.up_proj", "model.layers.38.mlp.experts.128.up_proj", "model.layers.38.mlp.experts.129.up_proj", "model.layers.38.mlp.experts.130.up_proj", "model.layers.38.mlp.experts.131.up_proj", "model.layers.38.mlp.experts.132.up_proj", "model.layers.38.mlp.experts.133.up_proj", "model.layers.38.mlp.experts.134.up_proj", "model.layers.38.mlp.experts.135.up_proj", "model.layers.38.mlp.experts.136.up_proj", "model.layers.38.mlp.experts.137.up_proj", "model.layers.38.mlp.experts.138.up_proj", "model.layers.38.mlp.experts.139.up_proj", "model.layers.38.mlp.experts.140.up_proj", "model.layers.38.mlp.experts.141.up_proj", "model.layers.38.mlp.experts.142.up_proj", "model.layers.38.mlp.experts.143.up_proj", "model.layers.38.mlp.experts.144.up_proj", "model.layers.38.mlp.experts.145.up_proj", "model.layers.38.mlp.experts.146.up_proj", "model.layers.38.mlp.experts.147.up_proj", "model.layers.38.mlp.experts.148.up_proj", "model.layers.38.mlp.experts.149.up_proj", "model.layers.38.mlp.experts.150.up_proj", "model.layers.38.mlp.experts.151.up_proj", "model.layers.38.mlp.experts.152.up_proj", "model.layers.38.mlp.experts.153.up_proj", "model.layers.38.mlp.experts.154.up_proj", "model.layers.38.mlp.experts.155.up_proj", "model.layers.38.mlp.experts.156.up_proj", "model.layers.38.mlp.experts.157.up_proj", "model.layers.38.mlp.experts.158.up_proj", "model.layers.38.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0010970659554004891, "dbits": 2516582400 } ] }, { "idx": 230, "layers": [ "model.layers.38.mlp.experts.0.down_proj", "model.layers.38.mlp.experts.1.down_proj", "model.layers.38.mlp.experts.2.down_proj", "model.layers.38.mlp.experts.3.down_proj", "model.layers.38.mlp.experts.4.down_proj", "model.layers.38.mlp.experts.5.down_proj", "model.layers.38.mlp.experts.6.down_proj", "model.layers.38.mlp.experts.7.down_proj", "model.layers.38.mlp.experts.8.down_proj", "model.layers.38.mlp.experts.9.down_proj", "model.layers.38.mlp.experts.10.down_proj", "model.layers.38.mlp.experts.11.down_proj", "model.layers.38.mlp.experts.12.down_proj", "model.layers.38.mlp.experts.13.down_proj", "model.layers.38.mlp.experts.14.down_proj", "model.layers.38.mlp.experts.15.down_proj", "model.layers.38.mlp.experts.16.down_proj", "model.layers.38.mlp.experts.17.down_proj", "model.layers.38.mlp.experts.18.down_proj", "model.layers.38.mlp.experts.19.down_proj", "model.layers.38.mlp.experts.20.down_proj", "model.layers.38.mlp.experts.21.down_proj", "model.layers.38.mlp.experts.22.down_proj", "model.layers.38.mlp.experts.23.down_proj", "model.layers.38.mlp.experts.24.down_proj", "model.layers.38.mlp.experts.25.down_proj", "model.layers.38.mlp.experts.26.down_proj", "model.layers.38.mlp.experts.27.down_proj", "model.layers.38.mlp.experts.28.down_proj", "model.layers.38.mlp.experts.29.down_proj", "model.layers.38.mlp.experts.30.down_proj", "model.layers.38.mlp.experts.31.down_proj", "model.layers.38.mlp.experts.32.down_proj", "model.layers.38.mlp.experts.33.down_proj", "model.layers.38.mlp.experts.34.down_proj", "model.layers.38.mlp.experts.35.down_proj", "model.layers.38.mlp.experts.36.down_proj", "model.layers.38.mlp.experts.37.down_proj", "model.layers.38.mlp.experts.38.down_proj", "model.layers.38.mlp.experts.39.down_proj", "model.layers.38.mlp.experts.40.down_proj", "model.layers.38.mlp.experts.41.down_proj", "model.layers.38.mlp.experts.42.down_proj", "model.layers.38.mlp.experts.43.down_proj", "model.layers.38.mlp.experts.44.down_proj", "model.layers.38.mlp.experts.45.down_proj", "model.layers.38.mlp.experts.46.down_proj", "model.layers.38.mlp.experts.47.down_proj", "model.layers.38.mlp.experts.48.down_proj", "model.layers.38.mlp.experts.49.down_proj", "model.layers.38.mlp.experts.50.down_proj", "model.layers.38.mlp.experts.51.down_proj", "model.layers.38.mlp.experts.52.down_proj", "model.layers.38.mlp.experts.53.down_proj", "model.layers.38.mlp.experts.54.down_proj", "model.layers.38.mlp.experts.55.down_proj", "model.layers.38.mlp.experts.56.down_proj", "model.layers.38.mlp.experts.57.down_proj", "model.layers.38.mlp.experts.58.down_proj", "model.layers.38.mlp.experts.59.down_proj", "model.layers.38.mlp.experts.60.down_proj", "model.layers.38.mlp.experts.61.down_proj", "model.layers.38.mlp.experts.62.down_proj", "model.layers.38.mlp.experts.63.down_proj", "model.layers.38.mlp.experts.64.down_proj", "model.layers.38.mlp.experts.65.down_proj", "model.layers.38.mlp.experts.66.down_proj", "model.layers.38.mlp.experts.67.down_proj", "model.layers.38.mlp.experts.68.down_proj", "model.layers.38.mlp.experts.69.down_proj", "model.layers.38.mlp.experts.70.down_proj", "model.layers.38.mlp.experts.71.down_proj", "model.layers.38.mlp.experts.72.down_proj", "model.layers.38.mlp.experts.73.down_proj", "model.layers.38.mlp.experts.74.down_proj", "model.layers.38.mlp.experts.75.down_proj", "model.layers.38.mlp.experts.76.down_proj", "model.layers.38.mlp.experts.77.down_proj", "model.layers.38.mlp.experts.78.down_proj", "model.layers.38.mlp.experts.79.down_proj", "model.layers.38.mlp.experts.80.down_proj", "model.layers.38.mlp.experts.81.down_proj", "model.layers.38.mlp.experts.82.down_proj", "model.layers.38.mlp.experts.83.down_proj", "model.layers.38.mlp.experts.84.down_proj", "model.layers.38.mlp.experts.85.down_proj", "model.layers.38.mlp.experts.86.down_proj", "model.layers.38.mlp.experts.87.down_proj", "model.layers.38.mlp.experts.88.down_proj", "model.layers.38.mlp.experts.89.down_proj", "model.layers.38.mlp.experts.90.down_proj", "model.layers.38.mlp.experts.91.down_proj", "model.layers.38.mlp.experts.92.down_proj", "model.layers.38.mlp.experts.93.down_proj", "model.layers.38.mlp.experts.94.down_proj", "model.layers.38.mlp.experts.95.down_proj", "model.layers.38.mlp.experts.96.down_proj", "model.layers.38.mlp.experts.97.down_proj", "model.layers.38.mlp.experts.98.down_proj", "model.layers.38.mlp.experts.99.down_proj", "model.layers.38.mlp.experts.100.down_proj", "model.layers.38.mlp.experts.101.down_proj", "model.layers.38.mlp.experts.102.down_proj", "model.layers.38.mlp.experts.103.down_proj", "model.layers.38.mlp.experts.104.down_proj", "model.layers.38.mlp.experts.105.down_proj", "model.layers.38.mlp.experts.106.down_proj", "model.layers.38.mlp.experts.107.down_proj", "model.layers.38.mlp.experts.108.down_proj", "model.layers.38.mlp.experts.109.down_proj", "model.layers.38.mlp.experts.110.down_proj", "model.layers.38.mlp.experts.111.down_proj", "model.layers.38.mlp.experts.112.down_proj", "model.layers.38.mlp.experts.113.down_proj", "model.layers.38.mlp.experts.114.down_proj", "model.layers.38.mlp.experts.115.down_proj", "model.layers.38.mlp.experts.116.down_proj", "model.layers.38.mlp.experts.117.down_proj", "model.layers.38.mlp.experts.118.down_proj", "model.layers.38.mlp.experts.119.down_proj", "model.layers.38.mlp.experts.120.down_proj", "model.layers.38.mlp.experts.121.down_proj", "model.layers.38.mlp.experts.122.down_proj", "model.layers.38.mlp.experts.123.down_proj", "model.layers.38.mlp.experts.124.down_proj", "model.layers.38.mlp.experts.125.down_proj", "model.layers.38.mlp.experts.126.down_proj", "model.layers.38.mlp.experts.127.down_proj", "model.layers.38.mlp.experts.128.down_proj", "model.layers.38.mlp.experts.129.down_proj", "model.layers.38.mlp.experts.130.down_proj", "model.layers.38.mlp.experts.131.down_proj", "model.layers.38.mlp.experts.132.down_proj", "model.layers.38.mlp.experts.133.down_proj", "model.layers.38.mlp.experts.134.down_proj", "model.layers.38.mlp.experts.135.down_proj", "model.layers.38.mlp.experts.136.down_proj", "model.layers.38.mlp.experts.137.down_proj", "model.layers.38.mlp.experts.138.down_proj", "model.layers.38.mlp.experts.139.down_proj", "model.layers.38.mlp.experts.140.down_proj", "model.layers.38.mlp.experts.141.down_proj", "model.layers.38.mlp.experts.142.down_proj", "model.layers.38.mlp.experts.143.down_proj", "model.layers.38.mlp.experts.144.down_proj", "model.layers.38.mlp.experts.145.down_proj", "model.layers.38.mlp.experts.146.down_proj", "model.layers.38.mlp.experts.147.down_proj", "model.layers.38.mlp.experts.148.down_proj", "model.layers.38.mlp.experts.149.down_proj", "model.layers.38.mlp.experts.150.down_proj", "model.layers.38.mlp.experts.151.down_proj", "model.layers.38.mlp.experts.152.down_proj", "model.layers.38.mlp.experts.153.down_proj", "model.layers.38.mlp.experts.154.down_proj", "model.layers.38.mlp.experts.155.down_proj", "model.layers.38.mlp.experts.156.down_proj", "model.layers.38.mlp.experts.157.down_proj", "model.layers.38.mlp.experts.158.down_proj", "model.layers.38.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002455139905214343, "dbits": 1258291200 } ] }, { "idx": 231, "layers": [ "model.layers.39.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0037478387355804332, "dbits": 62914560 } ] }, { "idx": 232, "layers": [ "model.layers.39.self_attn.k_proj", "model.layers.39.self_attn.v_proj" ], "candidates": [ { "dkld": 0.002388209104537964, "dbits": 10485760 } ] }, { "idx": 233, "layers": [ "model.layers.39.self_attn.o_proj" ], "candidates": [ { "dkld": -0.005784505605697654, "dbits": 62914560 } ] }, { "idx": 234, "layers": [ "model.layers.39.mlp.shared_experts.gate_proj", "model.layers.39.mlp.shared_experts.up_proj", "model.layers.39.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0028714224696159696, "dbits": 23592960 } ] }, { "idx": 235, "layers": [ "model.layers.39.mlp.experts.0.gate_proj", "model.layers.39.mlp.experts.1.gate_proj", "model.layers.39.mlp.experts.2.gate_proj", "model.layers.39.mlp.experts.3.gate_proj", "model.layers.39.mlp.experts.4.gate_proj", "model.layers.39.mlp.experts.5.gate_proj", "model.layers.39.mlp.experts.6.gate_proj", "model.layers.39.mlp.experts.7.gate_proj", "model.layers.39.mlp.experts.8.gate_proj", "model.layers.39.mlp.experts.9.gate_proj", "model.layers.39.mlp.experts.10.gate_proj", "model.layers.39.mlp.experts.11.gate_proj", "model.layers.39.mlp.experts.12.gate_proj", "model.layers.39.mlp.experts.13.gate_proj", "model.layers.39.mlp.experts.14.gate_proj", "model.layers.39.mlp.experts.15.gate_proj", "model.layers.39.mlp.experts.16.gate_proj", "model.layers.39.mlp.experts.17.gate_proj", "model.layers.39.mlp.experts.18.gate_proj", "model.layers.39.mlp.experts.19.gate_proj", "model.layers.39.mlp.experts.20.gate_proj", "model.layers.39.mlp.experts.21.gate_proj", "model.layers.39.mlp.experts.22.gate_proj", "model.layers.39.mlp.experts.23.gate_proj", "model.layers.39.mlp.experts.24.gate_proj", "model.layers.39.mlp.experts.25.gate_proj", "model.layers.39.mlp.experts.26.gate_proj", "model.layers.39.mlp.experts.27.gate_proj", "model.layers.39.mlp.experts.28.gate_proj", "model.layers.39.mlp.experts.29.gate_proj", "model.layers.39.mlp.experts.30.gate_proj", "model.layers.39.mlp.experts.31.gate_proj", "model.layers.39.mlp.experts.32.gate_proj", "model.layers.39.mlp.experts.33.gate_proj", "model.layers.39.mlp.experts.34.gate_proj", "model.layers.39.mlp.experts.35.gate_proj", "model.layers.39.mlp.experts.36.gate_proj", "model.layers.39.mlp.experts.37.gate_proj", "model.layers.39.mlp.experts.38.gate_proj", "model.layers.39.mlp.experts.39.gate_proj", "model.layers.39.mlp.experts.40.gate_proj", "model.layers.39.mlp.experts.41.gate_proj", "model.layers.39.mlp.experts.42.gate_proj", "model.layers.39.mlp.experts.43.gate_proj", "model.layers.39.mlp.experts.44.gate_proj", "model.layers.39.mlp.experts.45.gate_proj", "model.layers.39.mlp.experts.46.gate_proj", "model.layers.39.mlp.experts.47.gate_proj", "model.layers.39.mlp.experts.48.gate_proj", "model.layers.39.mlp.experts.49.gate_proj", "model.layers.39.mlp.experts.50.gate_proj", "model.layers.39.mlp.experts.51.gate_proj", "model.layers.39.mlp.experts.52.gate_proj", "model.layers.39.mlp.experts.53.gate_proj", "model.layers.39.mlp.experts.54.gate_proj", "model.layers.39.mlp.experts.55.gate_proj", "model.layers.39.mlp.experts.56.gate_proj", "model.layers.39.mlp.experts.57.gate_proj", "model.layers.39.mlp.experts.58.gate_proj", "model.layers.39.mlp.experts.59.gate_proj", "model.layers.39.mlp.experts.60.gate_proj", "model.layers.39.mlp.experts.61.gate_proj", "model.layers.39.mlp.experts.62.gate_proj", "model.layers.39.mlp.experts.63.gate_proj", "model.layers.39.mlp.experts.64.gate_proj", "model.layers.39.mlp.experts.65.gate_proj", "model.layers.39.mlp.experts.66.gate_proj", "model.layers.39.mlp.experts.67.gate_proj", "model.layers.39.mlp.experts.68.gate_proj", "model.layers.39.mlp.experts.69.gate_proj", "model.layers.39.mlp.experts.70.gate_proj", "model.layers.39.mlp.experts.71.gate_proj", "model.layers.39.mlp.experts.72.gate_proj", "model.layers.39.mlp.experts.73.gate_proj", "model.layers.39.mlp.experts.74.gate_proj", "model.layers.39.mlp.experts.75.gate_proj", "model.layers.39.mlp.experts.76.gate_proj", "model.layers.39.mlp.experts.77.gate_proj", "model.layers.39.mlp.experts.78.gate_proj", "model.layers.39.mlp.experts.79.gate_proj", "model.layers.39.mlp.experts.80.gate_proj", "model.layers.39.mlp.experts.81.gate_proj", "model.layers.39.mlp.experts.82.gate_proj", "model.layers.39.mlp.experts.83.gate_proj", "model.layers.39.mlp.experts.84.gate_proj", "model.layers.39.mlp.experts.85.gate_proj", "model.layers.39.mlp.experts.86.gate_proj", "model.layers.39.mlp.experts.87.gate_proj", "model.layers.39.mlp.experts.88.gate_proj", "model.layers.39.mlp.experts.89.gate_proj", "model.layers.39.mlp.experts.90.gate_proj", "model.layers.39.mlp.experts.91.gate_proj", "model.layers.39.mlp.experts.92.gate_proj", "model.layers.39.mlp.experts.93.gate_proj", "model.layers.39.mlp.experts.94.gate_proj", "model.layers.39.mlp.experts.95.gate_proj", "model.layers.39.mlp.experts.96.gate_proj", "model.layers.39.mlp.experts.97.gate_proj", "model.layers.39.mlp.experts.98.gate_proj", "model.layers.39.mlp.experts.99.gate_proj", "model.layers.39.mlp.experts.100.gate_proj", "model.layers.39.mlp.experts.101.gate_proj", "model.layers.39.mlp.experts.102.gate_proj", "model.layers.39.mlp.experts.103.gate_proj", "model.layers.39.mlp.experts.104.gate_proj", "model.layers.39.mlp.experts.105.gate_proj", "model.layers.39.mlp.experts.106.gate_proj", "model.layers.39.mlp.experts.107.gate_proj", "model.layers.39.mlp.experts.108.gate_proj", "model.layers.39.mlp.experts.109.gate_proj", "model.layers.39.mlp.experts.110.gate_proj", "model.layers.39.mlp.experts.111.gate_proj", "model.layers.39.mlp.experts.112.gate_proj", "model.layers.39.mlp.experts.113.gate_proj", "model.layers.39.mlp.experts.114.gate_proj", "model.layers.39.mlp.experts.115.gate_proj", "model.layers.39.mlp.experts.116.gate_proj", "model.layers.39.mlp.experts.117.gate_proj", "model.layers.39.mlp.experts.118.gate_proj", "model.layers.39.mlp.experts.119.gate_proj", "model.layers.39.mlp.experts.120.gate_proj", "model.layers.39.mlp.experts.121.gate_proj", "model.layers.39.mlp.experts.122.gate_proj", "model.layers.39.mlp.experts.123.gate_proj", "model.layers.39.mlp.experts.124.gate_proj", "model.layers.39.mlp.experts.125.gate_proj", "model.layers.39.mlp.experts.126.gate_proj", "model.layers.39.mlp.experts.127.gate_proj", "model.layers.39.mlp.experts.128.gate_proj", "model.layers.39.mlp.experts.129.gate_proj", "model.layers.39.mlp.experts.130.gate_proj", "model.layers.39.mlp.experts.131.gate_proj", "model.layers.39.mlp.experts.132.gate_proj", "model.layers.39.mlp.experts.133.gate_proj", "model.layers.39.mlp.experts.134.gate_proj", "model.layers.39.mlp.experts.135.gate_proj", "model.layers.39.mlp.experts.136.gate_proj", "model.layers.39.mlp.experts.137.gate_proj", "model.layers.39.mlp.experts.138.gate_proj", "model.layers.39.mlp.experts.139.gate_proj", "model.layers.39.mlp.experts.140.gate_proj", "model.layers.39.mlp.experts.141.gate_proj", "model.layers.39.mlp.experts.142.gate_proj", "model.layers.39.mlp.experts.143.gate_proj", "model.layers.39.mlp.experts.144.gate_proj", "model.layers.39.mlp.experts.145.gate_proj", "model.layers.39.mlp.experts.146.gate_proj", "model.layers.39.mlp.experts.147.gate_proj", "model.layers.39.mlp.experts.148.gate_proj", "model.layers.39.mlp.experts.149.gate_proj", "model.layers.39.mlp.experts.150.gate_proj", "model.layers.39.mlp.experts.151.gate_proj", "model.layers.39.mlp.experts.152.gate_proj", "model.layers.39.mlp.experts.153.gate_proj", "model.layers.39.mlp.experts.154.gate_proj", "model.layers.39.mlp.experts.155.gate_proj", "model.layers.39.mlp.experts.156.gate_proj", "model.layers.39.mlp.experts.157.gate_proj", "model.layers.39.mlp.experts.158.gate_proj", "model.layers.39.mlp.experts.159.gate_proj", "model.layers.39.mlp.experts.0.up_proj", "model.layers.39.mlp.experts.1.up_proj", "model.layers.39.mlp.experts.2.up_proj", "model.layers.39.mlp.experts.3.up_proj", "model.layers.39.mlp.experts.4.up_proj", "model.layers.39.mlp.experts.5.up_proj", "model.layers.39.mlp.experts.6.up_proj", "model.layers.39.mlp.experts.7.up_proj", "model.layers.39.mlp.experts.8.up_proj", "model.layers.39.mlp.experts.9.up_proj", "model.layers.39.mlp.experts.10.up_proj", "model.layers.39.mlp.experts.11.up_proj", "model.layers.39.mlp.experts.12.up_proj", "model.layers.39.mlp.experts.13.up_proj", "model.layers.39.mlp.experts.14.up_proj", "model.layers.39.mlp.experts.15.up_proj", "model.layers.39.mlp.experts.16.up_proj", "model.layers.39.mlp.experts.17.up_proj", "model.layers.39.mlp.experts.18.up_proj", "model.layers.39.mlp.experts.19.up_proj", "model.layers.39.mlp.experts.20.up_proj", "model.layers.39.mlp.experts.21.up_proj", "model.layers.39.mlp.experts.22.up_proj", "model.layers.39.mlp.experts.23.up_proj", "model.layers.39.mlp.experts.24.up_proj", "model.layers.39.mlp.experts.25.up_proj", "model.layers.39.mlp.experts.26.up_proj", "model.layers.39.mlp.experts.27.up_proj", "model.layers.39.mlp.experts.28.up_proj", "model.layers.39.mlp.experts.29.up_proj", "model.layers.39.mlp.experts.30.up_proj", "model.layers.39.mlp.experts.31.up_proj", "model.layers.39.mlp.experts.32.up_proj", "model.layers.39.mlp.experts.33.up_proj", "model.layers.39.mlp.experts.34.up_proj", "model.layers.39.mlp.experts.35.up_proj", "model.layers.39.mlp.experts.36.up_proj", "model.layers.39.mlp.experts.37.up_proj", "model.layers.39.mlp.experts.38.up_proj", "model.layers.39.mlp.experts.39.up_proj", "model.layers.39.mlp.experts.40.up_proj", "model.layers.39.mlp.experts.41.up_proj", "model.layers.39.mlp.experts.42.up_proj", "model.layers.39.mlp.experts.43.up_proj", "model.layers.39.mlp.experts.44.up_proj", "model.layers.39.mlp.experts.45.up_proj", "model.layers.39.mlp.experts.46.up_proj", "model.layers.39.mlp.experts.47.up_proj", "model.layers.39.mlp.experts.48.up_proj", "model.layers.39.mlp.experts.49.up_proj", "model.layers.39.mlp.experts.50.up_proj", "model.layers.39.mlp.experts.51.up_proj", "model.layers.39.mlp.experts.52.up_proj", "model.layers.39.mlp.experts.53.up_proj", "model.layers.39.mlp.experts.54.up_proj", "model.layers.39.mlp.experts.55.up_proj", "model.layers.39.mlp.experts.56.up_proj", "model.layers.39.mlp.experts.57.up_proj", "model.layers.39.mlp.experts.58.up_proj", "model.layers.39.mlp.experts.59.up_proj", "model.layers.39.mlp.experts.60.up_proj", "model.layers.39.mlp.experts.61.up_proj", "model.layers.39.mlp.experts.62.up_proj", "model.layers.39.mlp.experts.63.up_proj", "model.layers.39.mlp.experts.64.up_proj", "model.layers.39.mlp.experts.65.up_proj", "model.layers.39.mlp.experts.66.up_proj", "model.layers.39.mlp.experts.67.up_proj", "model.layers.39.mlp.experts.68.up_proj", "model.layers.39.mlp.experts.69.up_proj", "model.layers.39.mlp.experts.70.up_proj", "model.layers.39.mlp.experts.71.up_proj", "model.layers.39.mlp.experts.72.up_proj", "model.layers.39.mlp.experts.73.up_proj", "model.layers.39.mlp.experts.74.up_proj", "model.layers.39.mlp.experts.75.up_proj", "model.layers.39.mlp.experts.76.up_proj", "model.layers.39.mlp.experts.77.up_proj", "model.layers.39.mlp.experts.78.up_proj", "model.layers.39.mlp.experts.79.up_proj", "model.layers.39.mlp.experts.80.up_proj", "model.layers.39.mlp.experts.81.up_proj", "model.layers.39.mlp.experts.82.up_proj", "model.layers.39.mlp.experts.83.up_proj", "model.layers.39.mlp.experts.84.up_proj", "model.layers.39.mlp.experts.85.up_proj", "model.layers.39.mlp.experts.86.up_proj", "model.layers.39.mlp.experts.87.up_proj", "model.layers.39.mlp.experts.88.up_proj", "model.layers.39.mlp.experts.89.up_proj", "model.layers.39.mlp.experts.90.up_proj", "model.layers.39.mlp.experts.91.up_proj", "model.layers.39.mlp.experts.92.up_proj", "model.layers.39.mlp.experts.93.up_proj", "model.layers.39.mlp.experts.94.up_proj", "model.layers.39.mlp.experts.95.up_proj", "model.layers.39.mlp.experts.96.up_proj", "model.layers.39.mlp.experts.97.up_proj", "model.layers.39.mlp.experts.98.up_proj", "model.layers.39.mlp.experts.99.up_proj", "model.layers.39.mlp.experts.100.up_proj", "model.layers.39.mlp.experts.101.up_proj", "model.layers.39.mlp.experts.102.up_proj", "model.layers.39.mlp.experts.103.up_proj", "model.layers.39.mlp.experts.104.up_proj", "model.layers.39.mlp.experts.105.up_proj", "model.layers.39.mlp.experts.106.up_proj", "model.layers.39.mlp.experts.107.up_proj", "model.layers.39.mlp.experts.108.up_proj", "model.layers.39.mlp.experts.109.up_proj", "model.layers.39.mlp.experts.110.up_proj", "model.layers.39.mlp.experts.111.up_proj", "model.layers.39.mlp.experts.112.up_proj", "model.layers.39.mlp.experts.113.up_proj", "model.layers.39.mlp.experts.114.up_proj", "model.layers.39.mlp.experts.115.up_proj", "model.layers.39.mlp.experts.116.up_proj", "model.layers.39.mlp.experts.117.up_proj", "model.layers.39.mlp.experts.118.up_proj", "model.layers.39.mlp.experts.119.up_proj", "model.layers.39.mlp.experts.120.up_proj", "model.layers.39.mlp.experts.121.up_proj", "model.layers.39.mlp.experts.122.up_proj", "model.layers.39.mlp.experts.123.up_proj", "model.layers.39.mlp.experts.124.up_proj", "model.layers.39.mlp.experts.125.up_proj", "model.layers.39.mlp.experts.126.up_proj", "model.layers.39.mlp.experts.127.up_proj", "model.layers.39.mlp.experts.128.up_proj", "model.layers.39.mlp.experts.129.up_proj", "model.layers.39.mlp.experts.130.up_proj", "model.layers.39.mlp.experts.131.up_proj", "model.layers.39.mlp.experts.132.up_proj", "model.layers.39.mlp.experts.133.up_proj", "model.layers.39.mlp.experts.134.up_proj", "model.layers.39.mlp.experts.135.up_proj", "model.layers.39.mlp.experts.136.up_proj", "model.layers.39.mlp.experts.137.up_proj", "model.layers.39.mlp.experts.138.up_proj", "model.layers.39.mlp.experts.139.up_proj", "model.layers.39.mlp.experts.140.up_proj", "model.layers.39.mlp.experts.141.up_proj", "model.layers.39.mlp.experts.142.up_proj", "model.layers.39.mlp.experts.143.up_proj", "model.layers.39.mlp.experts.144.up_proj", "model.layers.39.mlp.experts.145.up_proj", "model.layers.39.mlp.experts.146.up_proj", "model.layers.39.mlp.experts.147.up_proj", "model.layers.39.mlp.experts.148.up_proj", "model.layers.39.mlp.experts.149.up_proj", "model.layers.39.mlp.experts.150.up_proj", "model.layers.39.mlp.experts.151.up_proj", "model.layers.39.mlp.experts.152.up_proj", "model.layers.39.mlp.experts.153.up_proj", "model.layers.39.mlp.experts.154.up_proj", "model.layers.39.mlp.experts.155.up_proj", "model.layers.39.mlp.experts.156.up_proj", "model.layers.39.mlp.experts.157.up_proj", "model.layers.39.mlp.experts.158.up_proj", "model.layers.39.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.002803371101617824, "dbits": 2516582400 } ] }, { "idx": 236, "layers": [ "model.layers.39.mlp.experts.0.down_proj", "model.layers.39.mlp.experts.1.down_proj", "model.layers.39.mlp.experts.2.down_proj", "model.layers.39.mlp.experts.3.down_proj", "model.layers.39.mlp.experts.4.down_proj", "model.layers.39.mlp.experts.5.down_proj", "model.layers.39.mlp.experts.6.down_proj", "model.layers.39.mlp.experts.7.down_proj", "model.layers.39.mlp.experts.8.down_proj", "model.layers.39.mlp.experts.9.down_proj", "model.layers.39.mlp.experts.10.down_proj", "model.layers.39.mlp.experts.11.down_proj", "model.layers.39.mlp.experts.12.down_proj", "model.layers.39.mlp.experts.13.down_proj", "model.layers.39.mlp.experts.14.down_proj", "model.layers.39.mlp.experts.15.down_proj", "model.layers.39.mlp.experts.16.down_proj", "model.layers.39.mlp.experts.17.down_proj", "model.layers.39.mlp.experts.18.down_proj", "model.layers.39.mlp.experts.19.down_proj", "model.layers.39.mlp.experts.20.down_proj", "model.layers.39.mlp.experts.21.down_proj", "model.layers.39.mlp.experts.22.down_proj", "model.layers.39.mlp.experts.23.down_proj", "model.layers.39.mlp.experts.24.down_proj", "model.layers.39.mlp.experts.25.down_proj", "model.layers.39.mlp.experts.26.down_proj", "model.layers.39.mlp.experts.27.down_proj", "model.layers.39.mlp.experts.28.down_proj", "model.layers.39.mlp.experts.29.down_proj", "model.layers.39.mlp.experts.30.down_proj", "model.layers.39.mlp.experts.31.down_proj", "model.layers.39.mlp.experts.32.down_proj", "model.layers.39.mlp.experts.33.down_proj", "model.layers.39.mlp.experts.34.down_proj", "model.layers.39.mlp.experts.35.down_proj", "model.layers.39.mlp.experts.36.down_proj", "model.layers.39.mlp.experts.37.down_proj", "model.layers.39.mlp.experts.38.down_proj", "model.layers.39.mlp.experts.39.down_proj", "model.layers.39.mlp.experts.40.down_proj", "model.layers.39.mlp.experts.41.down_proj", "model.layers.39.mlp.experts.42.down_proj", "model.layers.39.mlp.experts.43.down_proj", "model.layers.39.mlp.experts.44.down_proj", "model.layers.39.mlp.experts.45.down_proj", "model.layers.39.mlp.experts.46.down_proj", "model.layers.39.mlp.experts.47.down_proj", "model.layers.39.mlp.experts.48.down_proj", "model.layers.39.mlp.experts.49.down_proj", "model.layers.39.mlp.experts.50.down_proj", "model.layers.39.mlp.experts.51.down_proj", "model.layers.39.mlp.experts.52.down_proj", "model.layers.39.mlp.experts.53.down_proj", "model.layers.39.mlp.experts.54.down_proj", "model.layers.39.mlp.experts.55.down_proj", "model.layers.39.mlp.experts.56.down_proj", "model.layers.39.mlp.experts.57.down_proj", "model.layers.39.mlp.experts.58.down_proj", "model.layers.39.mlp.experts.59.down_proj", "model.layers.39.mlp.experts.60.down_proj", "model.layers.39.mlp.experts.61.down_proj", "model.layers.39.mlp.experts.62.down_proj", "model.layers.39.mlp.experts.63.down_proj", "model.layers.39.mlp.experts.64.down_proj", "model.layers.39.mlp.experts.65.down_proj", "model.layers.39.mlp.experts.66.down_proj", "model.layers.39.mlp.experts.67.down_proj", "model.layers.39.mlp.experts.68.down_proj", "model.layers.39.mlp.experts.69.down_proj", "model.layers.39.mlp.experts.70.down_proj", "model.layers.39.mlp.experts.71.down_proj", "model.layers.39.mlp.experts.72.down_proj", "model.layers.39.mlp.experts.73.down_proj", "model.layers.39.mlp.experts.74.down_proj", "model.layers.39.mlp.experts.75.down_proj", "model.layers.39.mlp.experts.76.down_proj", "model.layers.39.mlp.experts.77.down_proj", "model.layers.39.mlp.experts.78.down_proj", "model.layers.39.mlp.experts.79.down_proj", "model.layers.39.mlp.experts.80.down_proj", "model.layers.39.mlp.experts.81.down_proj", "model.layers.39.mlp.experts.82.down_proj", "model.layers.39.mlp.experts.83.down_proj", "model.layers.39.mlp.experts.84.down_proj", "model.layers.39.mlp.experts.85.down_proj", "model.layers.39.mlp.experts.86.down_proj", "model.layers.39.mlp.experts.87.down_proj", "model.layers.39.mlp.experts.88.down_proj", "model.layers.39.mlp.experts.89.down_proj", "model.layers.39.mlp.experts.90.down_proj", "model.layers.39.mlp.experts.91.down_proj", "model.layers.39.mlp.experts.92.down_proj", "model.layers.39.mlp.experts.93.down_proj", "model.layers.39.mlp.experts.94.down_proj", "model.layers.39.mlp.experts.95.down_proj", "model.layers.39.mlp.experts.96.down_proj", "model.layers.39.mlp.experts.97.down_proj", "model.layers.39.mlp.experts.98.down_proj", "model.layers.39.mlp.experts.99.down_proj", "model.layers.39.mlp.experts.100.down_proj", "model.layers.39.mlp.experts.101.down_proj", "model.layers.39.mlp.experts.102.down_proj", "model.layers.39.mlp.experts.103.down_proj", "model.layers.39.mlp.experts.104.down_proj", "model.layers.39.mlp.experts.105.down_proj", "model.layers.39.mlp.experts.106.down_proj", "model.layers.39.mlp.experts.107.down_proj", "model.layers.39.mlp.experts.108.down_proj", "model.layers.39.mlp.experts.109.down_proj", "model.layers.39.mlp.experts.110.down_proj", "model.layers.39.mlp.experts.111.down_proj", "model.layers.39.mlp.experts.112.down_proj", "model.layers.39.mlp.experts.113.down_proj", "model.layers.39.mlp.experts.114.down_proj", "model.layers.39.mlp.experts.115.down_proj", "model.layers.39.mlp.experts.116.down_proj", "model.layers.39.mlp.experts.117.down_proj", "model.layers.39.mlp.experts.118.down_proj", "model.layers.39.mlp.experts.119.down_proj", "model.layers.39.mlp.experts.120.down_proj", "model.layers.39.mlp.experts.121.down_proj", "model.layers.39.mlp.experts.122.down_proj", "model.layers.39.mlp.experts.123.down_proj", "model.layers.39.mlp.experts.124.down_proj", "model.layers.39.mlp.experts.125.down_proj", "model.layers.39.mlp.experts.126.down_proj", "model.layers.39.mlp.experts.127.down_proj", "model.layers.39.mlp.experts.128.down_proj", "model.layers.39.mlp.experts.129.down_proj", "model.layers.39.mlp.experts.130.down_proj", "model.layers.39.mlp.experts.131.down_proj", "model.layers.39.mlp.experts.132.down_proj", "model.layers.39.mlp.experts.133.down_proj", "model.layers.39.mlp.experts.134.down_proj", "model.layers.39.mlp.experts.135.down_proj", "model.layers.39.mlp.experts.136.down_proj", "model.layers.39.mlp.experts.137.down_proj", "model.layers.39.mlp.experts.138.down_proj", "model.layers.39.mlp.experts.139.down_proj", "model.layers.39.mlp.experts.140.down_proj", "model.layers.39.mlp.experts.141.down_proj", "model.layers.39.mlp.experts.142.down_proj", "model.layers.39.mlp.experts.143.down_proj", "model.layers.39.mlp.experts.144.down_proj", "model.layers.39.mlp.experts.145.down_proj", "model.layers.39.mlp.experts.146.down_proj", "model.layers.39.mlp.experts.147.down_proj", "model.layers.39.mlp.experts.148.down_proj", "model.layers.39.mlp.experts.149.down_proj", "model.layers.39.mlp.experts.150.down_proj", "model.layers.39.mlp.experts.151.down_proj", "model.layers.39.mlp.experts.152.down_proj", "model.layers.39.mlp.experts.153.down_proj", "model.layers.39.mlp.experts.154.down_proj", "model.layers.39.mlp.experts.155.down_proj", "model.layers.39.mlp.experts.156.down_proj", "model.layers.39.mlp.experts.157.down_proj", "model.layers.39.mlp.experts.158.down_proj", "model.layers.39.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0024181462824344746, "dbits": 1258291200 } ] }, { "idx": 237, "layers": [ "model.layers.40.self_attn.q_proj" ], "candidates": [ { "dkld": 0.00014904141426086426, "dbits": 62914560 } ] }, { "idx": 238, "layers": [ "model.layers.40.self_attn.k_proj", "model.layers.40.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0012115254998207314, "dbits": 10485760 } ] }, { "idx": 239, "layers": [ "model.layers.40.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00011253505945207909, "dbits": 62914560 } ] }, { "idx": 240, "layers": [ "model.layers.40.mlp.shared_experts.gate_proj", "model.layers.40.mlp.shared_experts.up_proj", "model.layers.40.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0018754579126834536, "dbits": 23592960 } ] }, { "idx": 241, "layers": [ "model.layers.40.mlp.experts.0.gate_proj", "model.layers.40.mlp.experts.1.gate_proj", "model.layers.40.mlp.experts.2.gate_proj", "model.layers.40.mlp.experts.3.gate_proj", "model.layers.40.mlp.experts.4.gate_proj", "model.layers.40.mlp.experts.5.gate_proj", "model.layers.40.mlp.experts.6.gate_proj", "model.layers.40.mlp.experts.7.gate_proj", "model.layers.40.mlp.experts.8.gate_proj", "model.layers.40.mlp.experts.9.gate_proj", "model.layers.40.mlp.experts.10.gate_proj", "model.layers.40.mlp.experts.11.gate_proj", "model.layers.40.mlp.experts.12.gate_proj", "model.layers.40.mlp.experts.13.gate_proj", "model.layers.40.mlp.experts.14.gate_proj", "model.layers.40.mlp.experts.15.gate_proj", "model.layers.40.mlp.experts.16.gate_proj", "model.layers.40.mlp.experts.17.gate_proj", "model.layers.40.mlp.experts.18.gate_proj", "model.layers.40.mlp.experts.19.gate_proj", "model.layers.40.mlp.experts.20.gate_proj", "model.layers.40.mlp.experts.21.gate_proj", "model.layers.40.mlp.experts.22.gate_proj", "model.layers.40.mlp.experts.23.gate_proj", "model.layers.40.mlp.experts.24.gate_proj", "model.layers.40.mlp.experts.25.gate_proj", "model.layers.40.mlp.experts.26.gate_proj", "model.layers.40.mlp.experts.27.gate_proj", "model.layers.40.mlp.experts.28.gate_proj", "model.layers.40.mlp.experts.29.gate_proj", "model.layers.40.mlp.experts.30.gate_proj", "model.layers.40.mlp.experts.31.gate_proj", "model.layers.40.mlp.experts.32.gate_proj", "model.layers.40.mlp.experts.33.gate_proj", "model.layers.40.mlp.experts.34.gate_proj", "model.layers.40.mlp.experts.35.gate_proj", "model.layers.40.mlp.experts.36.gate_proj", "model.layers.40.mlp.experts.37.gate_proj", "model.layers.40.mlp.experts.38.gate_proj", "model.layers.40.mlp.experts.39.gate_proj", "model.layers.40.mlp.experts.40.gate_proj", "model.layers.40.mlp.experts.41.gate_proj", "model.layers.40.mlp.experts.42.gate_proj", "model.layers.40.mlp.experts.43.gate_proj", "model.layers.40.mlp.experts.44.gate_proj", "model.layers.40.mlp.experts.45.gate_proj", "model.layers.40.mlp.experts.46.gate_proj", "model.layers.40.mlp.experts.47.gate_proj", "model.layers.40.mlp.experts.48.gate_proj", "model.layers.40.mlp.experts.49.gate_proj", "model.layers.40.mlp.experts.50.gate_proj", "model.layers.40.mlp.experts.51.gate_proj", "model.layers.40.mlp.experts.52.gate_proj", "model.layers.40.mlp.experts.53.gate_proj", "model.layers.40.mlp.experts.54.gate_proj", "model.layers.40.mlp.experts.55.gate_proj", "model.layers.40.mlp.experts.56.gate_proj", "model.layers.40.mlp.experts.57.gate_proj", "model.layers.40.mlp.experts.58.gate_proj", "model.layers.40.mlp.experts.59.gate_proj", "model.layers.40.mlp.experts.60.gate_proj", "model.layers.40.mlp.experts.61.gate_proj", "model.layers.40.mlp.experts.62.gate_proj", "model.layers.40.mlp.experts.63.gate_proj", "model.layers.40.mlp.experts.64.gate_proj", "model.layers.40.mlp.experts.65.gate_proj", "model.layers.40.mlp.experts.66.gate_proj", "model.layers.40.mlp.experts.67.gate_proj", "model.layers.40.mlp.experts.68.gate_proj", "model.layers.40.mlp.experts.69.gate_proj", "model.layers.40.mlp.experts.70.gate_proj", "model.layers.40.mlp.experts.71.gate_proj", "model.layers.40.mlp.experts.72.gate_proj", "model.layers.40.mlp.experts.73.gate_proj", "model.layers.40.mlp.experts.74.gate_proj", "model.layers.40.mlp.experts.75.gate_proj", "model.layers.40.mlp.experts.76.gate_proj", "model.layers.40.mlp.experts.77.gate_proj", "model.layers.40.mlp.experts.78.gate_proj", "model.layers.40.mlp.experts.79.gate_proj", "model.layers.40.mlp.experts.80.gate_proj", "model.layers.40.mlp.experts.81.gate_proj", "model.layers.40.mlp.experts.82.gate_proj", "model.layers.40.mlp.experts.83.gate_proj", "model.layers.40.mlp.experts.84.gate_proj", "model.layers.40.mlp.experts.85.gate_proj", "model.layers.40.mlp.experts.86.gate_proj", "model.layers.40.mlp.experts.87.gate_proj", "model.layers.40.mlp.experts.88.gate_proj", "model.layers.40.mlp.experts.89.gate_proj", "model.layers.40.mlp.experts.90.gate_proj", "model.layers.40.mlp.experts.91.gate_proj", "model.layers.40.mlp.experts.92.gate_proj", "model.layers.40.mlp.experts.93.gate_proj", "model.layers.40.mlp.experts.94.gate_proj", "model.layers.40.mlp.experts.95.gate_proj", "model.layers.40.mlp.experts.96.gate_proj", "model.layers.40.mlp.experts.97.gate_proj", "model.layers.40.mlp.experts.98.gate_proj", "model.layers.40.mlp.experts.99.gate_proj", "model.layers.40.mlp.experts.100.gate_proj", "model.layers.40.mlp.experts.101.gate_proj", "model.layers.40.mlp.experts.102.gate_proj", "model.layers.40.mlp.experts.103.gate_proj", "model.layers.40.mlp.experts.104.gate_proj", "model.layers.40.mlp.experts.105.gate_proj", "model.layers.40.mlp.experts.106.gate_proj", "model.layers.40.mlp.experts.107.gate_proj", "model.layers.40.mlp.experts.108.gate_proj", "model.layers.40.mlp.experts.109.gate_proj", "model.layers.40.mlp.experts.110.gate_proj", "model.layers.40.mlp.experts.111.gate_proj", "model.layers.40.mlp.experts.112.gate_proj", "model.layers.40.mlp.experts.113.gate_proj", "model.layers.40.mlp.experts.114.gate_proj", "model.layers.40.mlp.experts.115.gate_proj", "model.layers.40.mlp.experts.116.gate_proj", "model.layers.40.mlp.experts.117.gate_proj", "model.layers.40.mlp.experts.118.gate_proj", "model.layers.40.mlp.experts.119.gate_proj", "model.layers.40.mlp.experts.120.gate_proj", "model.layers.40.mlp.experts.121.gate_proj", "model.layers.40.mlp.experts.122.gate_proj", "model.layers.40.mlp.experts.123.gate_proj", "model.layers.40.mlp.experts.124.gate_proj", "model.layers.40.mlp.experts.125.gate_proj", "model.layers.40.mlp.experts.126.gate_proj", "model.layers.40.mlp.experts.127.gate_proj", "model.layers.40.mlp.experts.128.gate_proj", "model.layers.40.mlp.experts.129.gate_proj", "model.layers.40.mlp.experts.130.gate_proj", "model.layers.40.mlp.experts.131.gate_proj", "model.layers.40.mlp.experts.132.gate_proj", "model.layers.40.mlp.experts.133.gate_proj", "model.layers.40.mlp.experts.134.gate_proj", "model.layers.40.mlp.experts.135.gate_proj", "model.layers.40.mlp.experts.136.gate_proj", "model.layers.40.mlp.experts.137.gate_proj", "model.layers.40.mlp.experts.138.gate_proj", "model.layers.40.mlp.experts.139.gate_proj", "model.layers.40.mlp.experts.140.gate_proj", "model.layers.40.mlp.experts.141.gate_proj", "model.layers.40.mlp.experts.142.gate_proj", "model.layers.40.mlp.experts.143.gate_proj", "model.layers.40.mlp.experts.144.gate_proj", "model.layers.40.mlp.experts.145.gate_proj", "model.layers.40.mlp.experts.146.gate_proj", "model.layers.40.mlp.experts.147.gate_proj", "model.layers.40.mlp.experts.148.gate_proj", "model.layers.40.mlp.experts.149.gate_proj", "model.layers.40.mlp.experts.150.gate_proj", "model.layers.40.mlp.experts.151.gate_proj", "model.layers.40.mlp.experts.152.gate_proj", "model.layers.40.mlp.experts.153.gate_proj", "model.layers.40.mlp.experts.154.gate_proj", "model.layers.40.mlp.experts.155.gate_proj", "model.layers.40.mlp.experts.156.gate_proj", "model.layers.40.mlp.experts.157.gate_proj", "model.layers.40.mlp.experts.158.gate_proj", "model.layers.40.mlp.experts.159.gate_proj", "model.layers.40.mlp.experts.0.up_proj", "model.layers.40.mlp.experts.1.up_proj", "model.layers.40.mlp.experts.2.up_proj", "model.layers.40.mlp.experts.3.up_proj", "model.layers.40.mlp.experts.4.up_proj", "model.layers.40.mlp.experts.5.up_proj", "model.layers.40.mlp.experts.6.up_proj", "model.layers.40.mlp.experts.7.up_proj", "model.layers.40.mlp.experts.8.up_proj", "model.layers.40.mlp.experts.9.up_proj", "model.layers.40.mlp.experts.10.up_proj", "model.layers.40.mlp.experts.11.up_proj", "model.layers.40.mlp.experts.12.up_proj", "model.layers.40.mlp.experts.13.up_proj", "model.layers.40.mlp.experts.14.up_proj", "model.layers.40.mlp.experts.15.up_proj", "model.layers.40.mlp.experts.16.up_proj", "model.layers.40.mlp.experts.17.up_proj", "model.layers.40.mlp.experts.18.up_proj", "model.layers.40.mlp.experts.19.up_proj", "model.layers.40.mlp.experts.20.up_proj", "model.layers.40.mlp.experts.21.up_proj", "model.layers.40.mlp.experts.22.up_proj", "model.layers.40.mlp.experts.23.up_proj", "model.layers.40.mlp.experts.24.up_proj", "model.layers.40.mlp.experts.25.up_proj", "model.layers.40.mlp.experts.26.up_proj", "model.layers.40.mlp.experts.27.up_proj", "model.layers.40.mlp.experts.28.up_proj", "model.layers.40.mlp.experts.29.up_proj", "model.layers.40.mlp.experts.30.up_proj", "model.layers.40.mlp.experts.31.up_proj", "model.layers.40.mlp.experts.32.up_proj", "model.layers.40.mlp.experts.33.up_proj", "model.layers.40.mlp.experts.34.up_proj", "model.layers.40.mlp.experts.35.up_proj", "model.layers.40.mlp.experts.36.up_proj", "model.layers.40.mlp.experts.37.up_proj", "model.layers.40.mlp.experts.38.up_proj", "model.layers.40.mlp.experts.39.up_proj", "model.layers.40.mlp.experts.40.up_proj", "model.layers.40.mlp.experts.41.up_proj", "model.layers.40.mlp.experts.42.up_proj", "model.layers.40.mlp.experts.43.up_proj", "model.layers.40.mlp.experts.44.up_proj", "model.layers.40.mlp.experts.45.up_proj", "model.layers.40.mlp.experts.46.up_proj", "model.layers.40.mlp.experts.47.up_proj", "model.layers.40.mlp.experts.48.up_proj", "model.layers.40.mlp.experts.49.up_proj", "model.layers.40.mlp.experts.50.up_proj", "model.layers.40.mlp.experts.51.up_proj", "model.layers.40.mlp.experts.52.up_proj", "model.layers.40.mlp.experts.53.up_proj", "model.layers.40.mlp.experts.54.up_proj", "model.layers.40.mlp.experts.55.up_proj", "model.layers.40.mlp.experts.56.up_proj", "model.layers.40.mlp.experts.57.up_proj", "model.layers.40.mlp.experts.58.up_proj", "model.layers.40.mlp.experts.59.up_proj", "model.layers.40.mlp.experts.60.up_proj", "model.layers.40.mlp.experts.61.up_proj", "model.layers.40.mlp.experts.62.up_proj", "model.layers.40.mlp.experts.63.up_proj", "model.layers.40.mlp.experts.64.up_proj", "model.layers.40.mlp.experts.65.up_proj", "model.layers.40.mlp.experts.66.up_proj", "model.layers.40.mlp.experts.67.up_proj", "model.layers.40.mlp.experts.68.up_proj", "model.layers.40.mlp.experts.69.up_proj", "model.layers.40.mlp.experts.70.up_proj", "model.layers.40.mlp.experts.71.up_proj", "model.layers.40.mlp.experts.72.up_proj", "model.layers.40.mlp.experts.73.up_proj", "model.layers.40.mlp.experts.74.up_proj", "model.layers.40.mlp.experts.75.up_proj", "model.layers.40.mlp.experts.76.up_proj", "model.layers.40.mlp.experts.77.up_proj", "model.layers.40.mlp.experts.78.up_proj", "model.layers.40.mlp.experts.79.up_proj", "model.layers.40.mlp.experts.80.up_proj", "model.layers.40.mlp.experts.81.up_proj", "model.layers.40.mlp.experts.82.up_proj", "model.layers.40.mlp.experts.83.up_proj", "model.layers.40.mlp.experts.84.up_proj", "model.layers.40.mlp.experts.85.up_proj", "model.layers.40.mlp.experts.86.up_proj", "model.layers.40.mlp.experts.87.up_proj", "model.layers.40.mlp.experts.88.up_proj", "model.layers.40.mlp.experts.89.up_proj", "model.layers.40.mlp.experts.90.up_proj", "model.layers.40.mlp.experts.91.up_proj", "model.layers.40.mlp.experts.92.up_proj", "model.layers.40.mlp.experts.93.up_proj", "model.layers.40.mlp.experts.94.up_proj", "model.layers.40.mlp.experts.95.up_proj", "model.layers.40.mlp.experts.96.up_proj", "model.layers.40.mlp.experts.97.up_proj", "model.layers.40.mlp.experts.98.up_proj", "model.layers.40.mlp.experts.99.up_proj", "model.layers.40.mlp.experts.100.up_proj", "model.layers.40.mlp.experts.101.up_proj", "model.layers.40.mlp.experts.102.up_proj", "model.layers.40.mlp.experts.103.up_proj", "model.layers.40.mlp.experts.104.up_proj", "model.layers.40.mlp.experts.105.up_proj", "model.layers.40.mlp.experts.106.up_proj", "model.layers.40.mlp.experts.107.up_proj", "model.layers.40.mlp.experts.108.up_proj", "model.layers.40.mlp.experts.109.up_proj", "model.layers.40.mlp.experts.110.up_proj", "model.layers.40.mlp.experts.111.up_proj", "model.layers.40.mlp.experts.112.up_proj", "model.layers.40.mlp.experts.113.up_proj", "model.layers.40.mlp.experts.114.up_proj", "model.layers.40.mlp.experts.115.up_proj", "model.layers.40.mlp.experts.116.up_proj", "model.layers.40.mlp.experts.117.up_proj", "model.layers.40.mlp.experts.118.up_proj", "model.layers.40.mlp.experts.119.up_proj", "model.layers.40.mlp.experts.120.up_proj", "model.layers.40.mlp.experts.121.up_proj", "model.layers.40.mlp.experts.122.up_proj", "model.layers.40.mlp.experts.123.up_proj", "model.layers.40.mlp.experts.124.up_proj", "model.layers.40.mlp.experts.125.up_proj", "model.layers.40.mlp.experts.126.up_proj", "model.layers.40.mlp.experts.127.up_proj", "model.layers.40.mlp.experts.128.up_proj", "model.layers.40.mlp.experts.129.up_proj", "model.layers.40.mlp.experts.130.up_proj", "model.layers.40.mlp.experts.131.up_proj", "model.layers.40.mlp.experts.132.up_proj", "model.layers.40.mlp.experts.133.up_proj", "model.layers.40.mlp.experts.134.up_proj", "model.layers.40.mlp.experts.135.up_proj", "model.layers.40.mlp.experts.136.up_proj", "model.layers.40.mlp.experts.137.up_proj", "model.layers.40.mlp.experts.138.up_proj", "model.layers.40.mlp.experts.139.up_proj", "model.layers.40.mlp.experts.140.up_proj", "model.layers.40.mlp.experts.141.up_proj", "model.layers.40.mlp.experts.142.up_proj", "model.layers.40.mlp.experts.143.up_proj", "model.layers.40.mlp.experts.144.up_proj", "model.layers.40.mlp.experts.145.up_proj", "model.layers.40.mlp.experts.146.up_proj", "model.layers.40.mlp.experts.147.up_proj", "model.layers.40.mlp.experts.148.up_proj", "model.layers.40.mlp.experts.149.up_proj", "model.layers.40.mlp.experts.150.up_proj", "model.layers.40.mlp.experts.151.up_proj", "model.layers.40.mlp.experts.152.up_proj", "model.layers.40.mlp.experts.153.up_proj", "model.layers.40.mlp.experts.154.up_proj", "model.layers.40.mlp.experts.155.up_proj", "model.layers.40.mlp.experts.156.up_proj", "model.layers.40.mlp.experts.157.up_proj", "model.layers.40.mlp.experts.158.up_proj", "model.layers.40.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0031426481902599446, "dbits": 2516582400 } ] }, { "idx": 242, "layers": [ "model.layers.40.mlp.experts.0.down_proj", "model.layers.40.mlp.experts.1.down_proj", "model.layers.40.mlp.experts.2.down_proj", "model.layers.40.mlp.experts.3.down_proj", "model.layers.40.mlp.experts.4.down_proj", "model.layers.40.mlp.experts.5.down_proj", "model.layers.40.mlp.experts.6.down_proj", "model.layers.40.mlp.experts.7.down_proj", "model.layers.40.mlp.experts.8.down_proj", "model.layers.40.mlp.experts.9.down_proj", "model.layers.40.mlp.experts.10.down_proj", "model.layers.40.mlp.experts.11.down_proj", "model.layers.40.mlp.experts.12.down_proj", "model.layers.40.mlp.experts.13.down_proj", "model.layers.40.mlp.experts.14.down_proj", "model.layers.40.mlp.experts.15.down_proj", "model.layers.40.mlp.experts.16.down_proj", "model.layers.40.mlp.experts.17.down_proj", "model.layers.40.mlp.experts.18.down_proj", "model.layers.40.mlp.experts.19.down_proj", "model.layers.40.mlp.experts.20.down_proj", "model.layers.40.mlp.experts.21.down_proj", "model.layers.40.mlp.experts.22.down_proj", "model.layers.40.mlp.experts.23.down_proj", "model.layers.40.mlp.experts.24.down_proj", "model.layers.40.mlp.experts.25.down_proj", "model.layers.40.mlp.experts.26.down_proj", "model.layers.40.mlp.experts.27.down_proj", "model.layers.40.mlp.experts.28.down_proj", "model.layers.40.mlp.experts.29.down_proj", "model.layers.40.mlp.experts.30.down_proj", "model.layers.40.mlp.experts.31.down_proj", "model.layers.40.mlp.experts.32.down_proj", "model.layers.40.mlp.experts.33.down_proj", "model.layers.40.mlp.experts.34.down_proj", "model.layers.40.mlp.experts.35.down_proj", "model.layers.40.mlp.experts.36.down_proj", "model.layers.40.mlp.experts.37.down_proj", "model.layers.40.mlp.experts.38.down_proj", "model.layers.40.mlp.experts.39.down_proj", "model.layers.40.mlp.experts.40.down_proj", "model.layers.40.mlp.experts.41.down_proj", "model.layers.40.mlp.experts.42.down_proj", "model.layers.40.mlp.experts.43.down_proj", "model.layers.40.mlp.experts.44.down_proj", "model.layers.40.mlp.experts.45.down_proj", "model.layers.40.mlp.experts.46.down_proj", "model.layers.40.mlp.experts.47.down_proj", "model.layers.40.mlp.experts.48.down_proj", "model.layers.40.mlp.experts.49.down_proj", "model.layers.40.mlp.experts.50.down_proj", "model.layers.40.mlp.experts.51.down_proj", "model.layers.40.mlp.experts.52.down_proj", "model.layers.40.mlp.experts.53.down_proj", "model.layers.40.mlp.experts.54.down_proj", "model.layers.40.mlp.experts.55.down_proj", "model.layers.40.mlp.experts.56.down_proj", "model.layers.40.mlp.experts.57.down_proj", "model.layers.40.mlp.experts.58.down_proj", "model.layers.40.mlp.experts.59.down_proj", "model.layers.40.mlp.experts.60.down_proj", "model.layers.40.mlp.experts.61.down_proj", "model.layers.40.mlp.experts.62.down_proj", "model.layers.40.mlp.experts.63.down_proj", "model.layers.40.mlp.experts.64.down_proj", "model.layers.40.mlp.experts.65.down_proj", "model.layers.40.mlp.experts.66.down_proj", "model.layers.40.mlp.experts.67.down_proj", "model.layers.40.mlp.experts.68.down_proj", "model.layers.40.mlp.experts.69.down_proj", "model.layers.40.mlp.experts.70.down_proj", "model.layers.40.mlp.experts.71.down_proj", "model.layers.40.mlp.experts.72.down_proj", "model.layers.40.mlp.experts.73.down_proj", "model.layers.40.mlp.experts.74.down_proj", "model.layers.40.mlp.experts.75.down_proj", "model.layers.40.mlp.experts.76.down_proj", "model.layers.40.mlp.experts.77.down_proj", "model.layers.40.mlp.experts.78.down_proj", "model.layers.40.mlp.experts.79.down_proj", "model.layers.40.mlp.experts.80.down_proj", "model.layers.40.mlp.experts.81.down_proj", "model.layers.40.mlp.experts.82.down_proj", "model.layers.40.mlp.experts.83.down_proj", "model.layers.40.mlp.experts.84.down_proj", "model.layers.40.mlp.experts.85.down_proj", "model.layers.40.mlp.experts.86.down_proj", "model.layers.40.mlp.experts.87.down_proj", "model.layers.40.mlp.experts.88.down_proj", "model.layers.40.mlp.experts.89.down_proj", "model.layers.40.mlp.experts.90.down_proj", "model.layers.40.mlp.experts.91.down_proj", "model.layers.40.mlp.experts.92.down_proj", "model.layers.40.mlp.experts.93.down_proj", "model.layers.40.mlp.experts.94.down_proj", "model.layers.40.mlp.experts.95.down_proj", "model.layers.40.mlp.experts.96.down_proj", "model.layers.40.mlp.experts.97.down_proj", "model.layers.40.mlp.experts.98.down_proj", "model.layers.40.mlp.experts.99.down_proj", "model.layers.40.mlp.experts.100.down_proj", "model.layers.40.mlp.experts.101.down_proj", "model.layers.40.mlp.experts.102.down_proj", "model.layers.40.mlp.experts.103.down_proj", "model.layers.40.mlp.experts.104.down_proj", "model.layers.40.mlp.experts.105.down_proj", "model.layers.40.mlp.experts.106.down_proj", "model.layers.40.mlp.experts.107.down_proj", "model.layers.40.mlp.experts.108.down_proj", "model.layers.40.mlp.experts.109.down_proj", "model.layers.40.mlp.experts.110.down_proj", "model.layers.40.mlp.experts.111.down_proj", "model.layers.40.mlp.experts.112.down_proj", "model.layers.40.mlp.experts.113.down_proj", "model.layers.40.mlp.experts.114.down_proj", "model.layers.40.mlp.experts.115.down_proj", "model.layers.40.mlp.experts.116.down_proj", "model.layers.40.mlp.experts.117.down_proj", "model.layers.40.mlp.experts.118.down_proj", "model.layers.40.mlp.experts.119.down_proj", "model.layers.40.mlp.experts.120.down_proj", "model.layers.40.mlp.experts.121.down_proj", "model.layers.40.mlp.experts.122.down_proj", "model.layers.40.mlp.experts.123.down_proj", "model.layers.40.mlp.experts.124.down_proj", "model.layers.40.mlp.experts.125.down_proj", "model.layers.40.mlp.experts.126.down_proj", "model.layers.40.mlp.experts.127.down_proj", "model.layers.40.mlp.experts.128.down_proj", "model.layers.40.mlp.experts.129.down_proj", "model.layers.40.mlp.experts.130.down_proj", "model.layers.40.mlp.experts.131.down_proj", "model.layers.40.mlp.experts.132.down_proj", "model.layers.40.mlp.experts.133.down_proj", "model.layers.40.mlp.experts.134.down_proj", "model.layers.40.mlp.experts.135.down_proj", "model.layers.40.mlp.experts.136.down_proj", "model.layers.40.mlp.experts.137.down_proj", "model.layers.40.mlp.experts.138.down_proj", "model.layers.40.mlp.experts.139.down_proj", "model.layers.40.mlp.experts.140.down_proj", "model.layers.40.mlp.experts.141.down_proj", "model.layers.40.mlp.experts.142.down_proj", "model.layers.40.mlp.experts.143.down_proj", "model.layers.40.mlp.experts.144.down_proj", "model.layers.40.mlp.experts.145.down_proj", "model.layers.40.mlp.experts.146.down_proj", "model.layers.40.mlp.experts.147.down_proj", "model.layers.40.mlp.experts.148.down_proj", "model.layers.40.mlp.experts.149.down_proj", "model.layers.40.mlp.experts.150.down_proj", "model.layers.40.mlp.experts.151.down_proj", "model.layers.40.mlp.experts.152.down_proj", "model.layers.40.mlp.experts.153.down_proj", "model.layers.40.mlp.experts.154.down_proj", "model.layers.40.mlp.experts.155.down_proj", "model.layers.40.mlp.experts.156.down_proj", "model.layers.40.mlp.experts.157.down_proj", "model.layers.40.mlp.experts.158.down_proj", "model.layers.40.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.002164656668901477, "dbits": 1258291200 } ] }, { "idx": 243, "layers": [ "model.layers.41.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0003833651542663463, "dbits": 62914560 } ] }, { "idx": 244, "layers": [ "model.layers.41.self_attn.k_proj", "model.layers.41.self_attn.v_proj" ], "candidates": [ { "dkld": 0.002871137857437145, "dbits": 10485760 } ] }, { "idx": 245, "layers": [ "model.layers.41.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0037468045949936024, "dbits": 62914560 } ] }, { "idx": 246, "layers": [ "model.layers.41.mlp.shared_experts.gate_proj", "model.layers.41.mlp.shared_experts.up_proj", "model.layers.41.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0005133822560310364, "dbits": 23592960 } ] }, { "idx": 247, "layers": [ "model.layers.41.mlp.experts.0.gate_proj", "model.layers.41.mlp.experts.1.gate_proj", "model.layers.41.mlp.experts.2.gate_proj", "model.layers.41.mlp.experts.3.gate_proj", "model.layers.41.mlp.experts.4.gate_proj", "model.layers.41.mlp.experts.5.gate_proj", "model.layers.41.mlp.experts.6.gate_proj", "model.layers.41.mlp.experts.7.gate_proj", "model.layers.41.mlp.experts.8.gate_proj", "model.layers.41.mlp.experts.9.gate_proj", "model.layers.41.mlp.experts.10.gate_proj", "model.layers.41.mlp.experts.11.gate_proj", "model.layers.41.mlp.experts.12.gate_proj", "model.layers.41.mlp.experts.13.gate_proj", "model.layers.41.mlp.experts.14.gate_proj", "model.layers.41.mlp.experts.15.gate_proj", "model.layers.41.mlp.experts.16.gate_proj", "model.layers.41.mlp.experts.17.gate_proj", "model.layers.41.mlp.experts.18.gate_proj", "model.layers.41.mlp.experts.19.gate_proj", "model.layers.41.mlp.experts.20.gate_proj", "model.layers.41.mlp.experts.21.gate_proj", "model.layers.41.mlp.experts.22.gate_proj", "model.layers.41.mlp.experts.23.gate_proj", "model.layers.41.mlp.experts.24.gate_proj", "model.layers.41.mlp.experts.25.gate_proj", "model.layers.41.mlp.experts.26.gate_proj", "model.layers.41.mlp.experts.27.gate_proj", "model.layers.41.mlp.experts.28.gate_proj", "model.layers.41.mlp.experts.29.gate_proj", "model.layers.41.mlp.experts.30.gate_proj", "model.layers.41.mlp.experts.31.gate_proj", "model.layers.41.mlp.experts.32.gate_proj", "model.layers.41.mlp.experts.33.gate_proj", "model.layers.41.mlp.experts.34.gate_proj", "model.layers.41.mlp.experts.35.gate_proj", "model.layers.41.mlp.experts.36.gate_proj", "model.layers.41.mlp.experts.37.gate_proj", "model.layers.41.mlp.experts.38.gate_proj", "model.layers.41.mlp.experts.39.gate_proj", "model.layers.41.mlp.experts.40.gate_proj", "model.layers.41.mlp.experts.41.gate_proj", "model.layers.41.mlp.experts.42.gate_proj", "model.layers.41.mlp.experts.43.gate_proj", "model.layers.41.mlp.experts.44.gate_proj", "model.layers.41.mlp.experts.45.gate_proj", "model.layers.41.mlp.experts.46.gate_proj", "model.layers.41.mlp.experts.47.gate_proj", "model.layers.41.mlp.experts.48.gate_proj", "model.layers.41.mlp.experts.49.gate_proj", "model.layers.41.mlp.experts.50.gate_proj", "model.layers.41.mlp.experts.51.gate_proj", "model.layers.41.mlp.experts.52.gate_proj", "model.layers.41.mlp.experts.53.gate_proj", "model.layers.41.mlp.experts.54.gate_proj", "model.layers.41.mlp.experts.55.gate_proj", "model.layers.41.mlp.experts.56.gate_proj", "model.layers.41.mlp.experts.57.gate_proj", "model.layers.41.mlp.experts.58.gate_proj", "model.layers.41.mlp.experts.59.gate_proj", "model.layers.41.mlp.experts.60.gate_proj", "model.layers.41.mlp.experts.61.gate_proj", "model.layers.41.mlp.experts.62.gate_proj", "model.layers.41.mlp.experts.63.gate_proj", "model.layers.41.mlp.experts.64.gate_proj", "model.layers.41.mlp.experts.65.gate_proj", "model.layers.41.mlp.experts.66.gate_proj", "model.layers.41.mlp.experts.67.gate_proj", "model.layers.41.mlp.experts.68.gate_proj", "model.layers.41.mlp.experts.69.gate_proj", "model.layers.41.mlp.experts.70.gate_proj", "model.layers.41.mlp.experts.71.gate_proj", "model.layers.41.mlp.experts.72.gate_proj", "model.layers.41.mlp.experts.73.gate_proj", "model.layers.41.mlp.experts.74.gate_proj", "model.layers.41.mlp.experts.75.gate_proj", "model.layers.41.mlp.experts.76.gate_proj", "model.layers.41.mlp.experts.77.gate_proj", "model.layers.41.mlp.experts.78.gate_proj", "model.layers.41.mlp.experts.79.gate_proj", "model.layers.41.mlp.experts.80.gate_proj", "model.layers.41.mlp.experts.81.gate_proj", "model.layers.41.mlp.experts.82.gate_proj", "model.layers.41.mlp.experts.83.gate_proj", "model.layers.41.mlp.experts.84.gate_proj", "model.layers.41.mlp.experts.85.gate_proj", "model.layers.41.mlp.experts.86.gate_proj", "model.layers.41.mlp.experts.87.gate_proj", "model.layers.41.mlp.experts.88.gate_proj", "model.layers.41.mlp.experts.89.gate_proj", "model.layers.41.mlp.experts.90.gate_proj", "model.layers.41.mlp.experts.91.gate_proj", "model.layers.41.mlp.experts.92.gate_proj", "model.layers.41.mlp.experts.93.gate_proj", "model.layers.41.mlp.experts.94.gate_proj", "model.layers.41.mlp.experts.95.gate_proj", "model.layers.41.mlp.experts.96.gate_proj", "model.layers.41.mlp.experts.97.gate_proj", "model.layers.41.mlp.experts.98.gate_proj", "model.layers.41.mlp.experts.99.gate_proj", "model.layers.41.mlp.experts.100.gate_proj", "model.layers.41.mlp.experts.101.gate_proj", "model.layers.41.mlp.experts.102.gate_proj", "model.layers.41.mlp.experts.103.gate_proj", "model.layers.41.mlp.experts.104.gate_proj", "model.layers.41.mlp.experts.105.gate_proj", "model.layers.41.mlp.experts.106.gate_proj", "model.layers.41.mlp.experts.107.gate_proj", "model.layers.41.mlp.experts.108.gate_proj", "model.layers.41.mlp.experts.109.gate_proj", "model.layers.41.mlp.experts.110.gate_proj", "model.layers.41.mlp.experts.111.gate_proj", "model.layers.41.mlp.experts.112.gate_proj", "model.layers.41.mlp.experts.113.gate_proj", "model.layers.41.mlp.experts.114.gate_proj", "model.layers.41.mlp.experts.115.gate_proj", "model.layers.41.mlp.experts.116.gate_proj", "model.layers.41.mlp.experts.117.gate_proj", "model.layers.41.mlp.experts.118.gate_proj", "model.layers.41.mlp.experts.119.gate_proj", "model.layers.41.mlp.experts.120.gate_proj", "model.layers.41.mlp.experts.121.gate_proj", "model.layers.41.mlp.experts.122.gate_proj", "model.layers.41.mlp.experts.123.gate_proj", "model.layers.41.mlp.experts.124.gate_proj", "model.layers.41.mlp.experts.125.gate_proj", "model.layers.41.mlp.experts.126.gate_proj", "model.layers.41.mlp.experts.127.gate_proj", "model.layers.41.mlp.experts.128.gate_proj", "model.layers.41.mlp.experts.129.gate_proj", "model.layers.41.mlp.experts.130.gate_proj", "model.layers.41.mlp.experts.131.gate_proj", "model.layers.41.mlp.experts.132.gate_proj", "model.layers.41.mlp.experts.133.gate_proj", "model.layers.41.mlp.experts.134.gate_proj", "model.layers.41.mlp.experts.135.gate_proj", "model.layers.41.mlp.experts.136.gate_proj", "model.layers.41.mlp.experts.137.gate_proj", "model.layers.41.mlp.experts.138.gate_proj", "model.layers.41.mlp.experts.139.gate_proj", "model.layers.41.mlp.experts.140.gate_proj", "model.layers.41.mlp.experts.141.gate_proj", "model.layers.41.mlp.experts.142.gate_proj", "model.layers.41.mlp.experts.143.gate_proj", "model.layers.41.mlp.experts.144.gate_proj", "model.layers.41.mlp.experts.145.gate_proj", "model.layers.41.mlp.experts.146.gate_proj", "model.layers.41.mlp.experts.147.gate_proj", "model.layers.41.mlp.experts.148.gate_proj", "model.layers.41.mlp.experts.149.gate_proj", "model.layers.41.mlp.experts.150.gate_proj", "model.layers.41.mlp.experts.151.gate_proj", "model.layers.41.mlp.experts.152.gate_proj", "model.layers.41.mlp.experts.153.gate_proj", "model.layers.41.mlp.experts.154.gate_proj", "model.layers.41.mlp.experts.155.gate_proj", "model.layers.41.mlp.experts.156.gate_proj", "model.layers.41.mlp.experts.157.gate_proj", "model.layers.41.mlp.experts.158.gate_proj", "model.layers.41.mlp.experts.159.gate_proj", "model.layers.41.mlp.experts.0.up_proj", "model.layers.41.mlp.experts.1.up_proj", "model.layers.41.mlp.experts.2.up_proj", "model.layers.41.mlp.experts.3.up_proj", "model.layers.41.mlp.experts.4.up_proj", "model.layers.41.mlp.experts.5.up_proj", "model.layers.41.mlp.experts.6.up_proj", "model.layers.41.mlp.experts.7.up_proj", "model.layers.41.mlp.experts.8.up_proj", "model.layers.41.mlp.experts.9.up_proj", "model.layers.41.mlp.experts.10.up_proj", "model.layers.41.mlp.experts.11.up_proj", "model.layers.41.mlp.experts.12.up_proj", "model.layers.41.mlp.experts.13.up_proj", "model.layers.41.mlp.experts.14.up_proj", "model.layers.41.mlp.experts.15.up_proj", "model.layers.41.mlp.experts.16.up_proj", "model.layers.41.mlp.experts.17.up_proj", "model.layers.41.mlp.experts.18.up_proj", "model.layers.41.mlp.experts.19.up_proj", "model.layers.41.mlp.experts.20.up_proj", "model.layers.41.mlp.experts.21.up_proj", "model.layers.41.mlp.experts.22.up_proj", "model.layers.41.mlp.experts.23.up_proj", "model.layers.41.mlp.experts.24.up_proj", "model.layers.41.mlp.experts.25.up_proj", "model.layers.41.mlp.experts.26.up_proj", "model.layers.41.mlp.experts.27.up_proj", "model.layers.41.mlp.experts.28.up_proj", "model.layers.41.mlp.experts.29.up_proj", "model.layers.41.mlp.experts.30.up_proj", "model.layers.41.mlp.experts.31.up_proj", "model.layers.41.mlp.experts.32.up_proj", "model.layers.41.mlp.experts.33.up_proj", "model.layers.41.mlp.experts.34.up_proj", "model.layers.41.mlp.experts.35.up_proj", "model.layers.41.mlp.experts.36.up_proj", "model.layers.41.mlp.experts.37.up_proj", "model.layers.41.mlp.experts.38.up_proj", "model.layers.41.mlp.experts.39.up_proj", "model.layers.41.mlp.experts.40.up_proj", "model.layers.41.mlp.experts.41.up_proj", "model.layers.41.mlp.experts.42.up_proj", "model.layers.41.mlp.experts.43.up_proj", "model.layers.41.mlp.experts.44.up_proj", "model.layers.41.mlp.experts.45.up_proj", "model.layers.41.mlp.experts.46.up_proj", "model.layers.41.mlp.experts.47.up_proj", "model.layers.41.mlp.experts.48.up_proj", "model.layers.41.mlp.experts.49.up_proj", "model.layers.41.mlp.experts.50.up_proj", "model.layers.41.mlp.experts.51.up_proj", "model.layers.41.mlp.experts.52.up_proj", "model.layers.41.mlp.experts.53.up_proj", "model.layers.41.mlp.experts.54.up_proj", "model.layers.41.mlp.experts.55.up_proj", "model.layers.41.mlp.experts.56.up_proj", "model.layers.41.mlp.experts.57.up_proj", "model.layers.41.mlp.experts.58.up_proj", "model.layers.41.mlp.experts.59.up_proj", "model.layers.41.mlp.experts.60.up_proj", "model.layers.41.mlp.experts.61.up_proj", "model.layers.41.mlp.experts.62.up_proj", "model.layers.41.mlp.experts.63.up_proj", "model.layers.41.mlp.experts.64.up_proj", "model.layers.41.mlp.experts.65.up_proj", "model.layers.41.mlp.experts.66.up_proj", "model.layers.41.mlp.experts.67.up_proj", "model.layers.41.mlp.experts.68.up_proj", "model.layers.41.mlp.experts.69.up_proj", "model.layers.41.mlp.experts.70.up_proj", "model.layers.41.mlp.experts.71.up_proj", "model.layers.41.mlp.experts.72.up_proj", "model.layers.41.mlp.experts.73.up_proj", "model.layers.41.mlp.experts.74.up_proj", "model.layers.41.mlp.experts.75.up_proj", "model.layers.41.mlp.experts.76.up_proj", "model.layers.41.mlp.experts.77.up_proj", "model.layers.41.mlp.experts.78.up_proj", "model.layers.41.mlp.experts.79.up_proj", "model.layers.41.mlp.experts.80.up_proj", "model.layers.41.mlp.experts.81.up_proj", "model.layers.41.mlp.experts.82.up_proj", "model.layers.41.mlp.experts.83.up_proj", "model.layers.41.mlp.experts.84.up_proj", "model.layers.41.mlp.experts.85.up_proj", "model.layers.41.mlp.experts.86.up_proj", "model.layers.41.mlp.experts.87.up_proj", "model.layers.41.mlp.experts.88.up_proj", "model.layers.41.mlp.experts.89.up_proj", "model.layers.41.mlp.experts.90.up_proj", "model.layers.41.mlp.experts.91.up_proj", "model.layers.41.mlp.experts.92.up_proj", "model.layers.41.mlp.experts.93.up_proj", "model.layers.41.mlp.experts.94.up_proj", "model.layers.41.mlp.experts.95.up_proj", "model.layers.41.mlp.experts.96.up_proj", "model.layers.41.mlp.experts.97.up_proj", "model.layers.41.mlp.experts.98.up_proj", "model.layers.41.mlp.experts.99.up_proj", "model.layers.41.mlp.experts.100.up_proj", "model.layers.41.mlp.experts.101.up_proj", "model.layers.41.mlp.experts.102.up_proj", "model.layers.41.mlp.experts.103.up_proj", "model.layers.41.mlp.experts.104.up_proj", "model.layers.41.mlp.experts.105.up_proj", "model.layers.41.mlp.experts.106.up_proj", "model.layers.41.mlp.experts.107.up_proj", "model.layers.41.mlp.experts.108.up_proj", "model.layers.41.mlp.experts.109.up_proj", "model.layers.41.mlp.experts.110.up_proj", "model.layers.41.mlp.experts.111.up_proj", "model.layers.41.mlp.experts.112.up_proj", "model.layers.41.mlp.experts.113.up_proj", "model.layers.41.mlp.experts.114.up_proj", "model.layers.41.mlp.experts.115.up_proj", "model.layers.41.mlp.experts.116.up_proj", "model.layers.41.mlp.experts.117.up_proj", "model.layers.41.mlp.experts.118.up_proj", "model.layers.41.mlp.experts.119.up_proj", "model.layers.41.mlp.experts.120.up_proj", "model.layers.41.mlp.experts.121.up_proj", "model.layers.41.mlp.experts.122.up_proj", "model.layers.41.mlp.experts.123.up_proj", "model.layers.41.mlp.experts.124.up_proj", "model.layers.41.mlp.experts.125.up_proj", "model.layers.41.mlp.experts.126.up_proj", "model.layers.41.mlp.experts.127.up_proj", "model.layers.41.mlp.experts.128.up_proj", "model.layers.41.mlp.experts.129.up_proj", "model.layers.41.mlp.experts.130.up_proj", "model.layers.41.mlp.experts.131.up_proj", "model.layers.41.mlp.experts.132.up_proj", "model.layers.41.mlp.experts.133.up_proj", "model.layers.41.mlp.experts.134.up_proj", "model.layers.41.mlp.experts.135.up_proj", "model.layers.41.mlp.experts.136.up_proj", "model.layers.41.mlp.experts.137.up_proj", "model.layers.41.mlp.experts.138.up_proj", "model.layers.41.mlp.experts.139.up_proj", "model.layers.41.mlp.experts.140.up_proj", "model.layers.41.mlp.experts.141.up_proj", "model.layers.41.mlp.experts.142.up_proj", "model.layers.41.mlp.experts.143.up_proj", "model.layers.41.mlp.experts.144.up_proj", "model.layers.41.mlp.experts.145.up_proj", "model.layers.41.mlp.experts.146.up_proj", "model.layers.41.mlp.experts.147.up_proj", "model.layers.41.mlp.experts.148.up_proj", "model.layers.41.mlp.experts.149.up_proj", "model.layers.41.mlp.experts.150.up_proj", "model.layers.41.mlp.experts.151.up_proj", "model.layers.41.mlp.experts.152.up_proj", "model.layers.41.mlp.experts.153.up_proj", "model.layers.41.mlp.experts.154.up_proj", "model.layers.41.mlp.experts.155.up_proj", "model.layers.41.mlp.experts.156.up_proj", "model.layers.41.mlp.experts.157.up_proj", "model.layers.41.mlp.experts.158.up_proj", "model.layers.41.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.004016762226820014, "dbits": 2516582400 } ] }, { "idx": 248, "layers": [ "model.layers.41.mlp.experts.0.down_proj", "model.layers.41.mlp.experts.1.down_proj", "model.layers.41.mlp.experts.2.down_proj", "model.layers.41.mlp.experts.3.down_proj", "model.layers.41.mlp.experts.4.down_proj", "model.layers.41.mlp.experts.5.down_proj", "model.layers.41.mlp.experts.6.down_proj", "model.layers.41.mlp.experts.7.down_proj", "model.layers.41.mlp.experts.8.down_proj", "model.layers.41.mlp.experts.9.down_proj", "model.layers.41.mlp.experts.10.down_proj", "model.layers.41.mlp.experts.11.down_proj", "model.layers.41.mlp.experts.12.down_proj", "model.layers.41.mlp.experts.13.down_proj", "model.layers.41.mlp.experts.14.down_proj", "model.layers.41.mlp.experts.15.down_proj", "model.layers.41.mlp.experts.16.down_proj", "model.layers.41.mlp.experts.17.down_proj", "model.layers.41.mlp.experts.18.down_proj", "model.layers.41.mlp.experts.19.down_proj", "model.layers.41.mlp.experts.20.down_proj", "model.layers.41.mlp.experts.21.down_proj", "model.layers.41.mlp.experts.22.down_proj", "model.layers.41.mlp.experts.23.down_proj", "model.layers.41.mlp.experts.24.down_proj", "model.layers.41.mlp.experts.25.down_proj", "model.layers.41.mlp.experts.26.down_proj", "model.layers.41.mlp.experts.27.down_proj", "model.layers.41.mlp.experts.28.down_proj", "model.layers.41.mlp.experts.29.down_proj", "model.layers.41.mlp.experts.30.down_proj", "model.layers.41.mlp.experts.31.down_proj", "model.layers.41.mlp.experts.32.down_proj", "model.layers.41.mlp.experts.33.down_proj", "model.layers.41.mlp.experts.34.down_proj", "model.layers.41.mlp.experts.35.down_proj", "model.layers.41.mlp.experts.36.down_proj", "model.layers.41.mlp.experts.37.down_proj", "model.layers.41.mlp.experts.38.down_proj", "model.layers.41.mlp.experts.39.down_proj", "model.layers.41.mlp.experts.40.down_proj", "model.layers.41.mlp.experts.41.down_proj", "model.layers.41.mlp.experts.42.down_proj", "model.layers.41.mlp.experts.43.down_proj", "model.layers.41.mlp.experts.44.down_proj", "model.layers.41.mlp.experts.45.down_proj", "model.layers.41.mlp.experts.46.down_proj", "model.layers.41.mlp.experts.47.down_proj", "model.layers.41.mlp.experts.48.down_proj", "model.layers.41.mlp.experts.49.down_proj", "model.layers.41.mlp.experts.50.down_proj", "model.layers.41.mlp.experts.51.down_proj", "model.layers.41.mlp.experts.52.down_proj", "model.layers.41.mlp.experts.53.down_proj", "model.layers.41.mlp.experts.54.down_proj", "model.layers.41.mlp.experts.55.down_proj", "model.layers.41.mlp.experts.56.down_proj", "model.layers.41.mlp.experts.57.down_proj", "model.layers.41.mlp.experts.58.down_proj", "model.layers.41.mlp.experts.59.down_proj", "model.layers.41.mlp.experts.60.down_proj", "model.layers.41.mlp.experts.61.down_proj", "model.layers.41.mlp.experts.62.down_proj", "model.layers.41.mlp.experts.63.down_proj", "model.layers.41.mlp.experts.64.down_proj", "model.layers.41.mlp.experts.65.down_proj", "model.layers.41.mlp.experts.66.down_proj", "model.layers.41.mlp.experts.67.down_proj", "model.layers.41.mlp.experts.68.down_proj", "model.layers.41.mlp.experts.69.down_proj", "model.layers.41.mlp.experts.70.down_proj", "model.layers.41.mlp.experts.71.down_proj", "model.layers.41.mlp.experts.72.down_proj", "model.layers.41.mlp.experts.73.down_proj", "model.layers.41.mlp.experts.74.down_proj", "model.layers.41.mlp.experts.75.down_proj", "model.layers.41.mlp.experts.76.down_proj", "model.layers.41.mlp.experts.77.down_proj", "model.layers.41.mlp.experts.78.down_proj", "model.layers.41.mlp.experts.79.down_proj", "model.layers.41.mlp.experts.80.down_proj", "model.layers.41.mlp.experts.81.down_proj", "model.layers.41.mlp.experts.82.down_proj", "model.layers.41.mlp.experts.83.down_proj", "model.layers.41.mlp.experts.84.down_proj", "model.layers.41.mlp.experts.85.down_proj", "model.layers.41.mlp.experts.86.down_proj", "model.layers.41.mlp.experts.87.down_proj", "model.layers.41.mlp.experts.88.down_proj", "model.layers.41.mlp.experts.89.down_proj", "model.layers.41.mlp.experts.90.down_proj", "model.layers.41.mlp.experts.91.down_proj", "model.layers.41.mlp.experts.92.down_proj", "model.layers.41.mlp.experts.93.down_proj", "model.layers.41.mlp.experts.94.down_proj", "model.layers.41.mlp.experts.95.down_proj", "model.layers.41.mlp.experts.96.down_proj", "model.layers.41.mlp.experts.97.down_proj", "model.layers.41.mlp.experts.98.down_proj", "model.layers.41.mlp.experts.99.down_proj", "model.layers.41.mlp.experts.100.down_proj", "model.layers.41.mlp.experts.101.down_proj", "model.layers.41.mlp.experts.102.down_proj", "model.layers.41.mlp.experts.103.down_proj", "model.layers.41.mlp.experts.104.down_proj", "model.layers.41.mlp.experts.105.down_proj", "model.layers.41.mlp.experts.106.down_proj", "model.layers.41.mlp.experts.107.down_proj", "model.layers.41.mlp.experts.108.down_proj", "model.layers.41.mlp.experts.109.down_proj", "model.layers.41.mlp.experts.110.down_proj", "model.layers.41.mlp.experts.111.down_proj", "model.layers.41.mlp.experts.112.down_proj", "model.layers.41.mlp.experts.113.down_proj", "model.layers.41.mlp.experts.114.down_proj", "model.layers.41.mlp.experts.115.down_proj", "model.layers.41.mlp.experts.116.down_proj", "model.layers.41.mlp.experts.117.down_proj", "model.layers.41.mlp.experts.118.down_proj", "model.layers.41.mlp.experts.119.down_proj", "model.layers.41.mlp.experts.120.down_proj", "model.layers.41.mlp.experts.121.down_proj", "model.layers.41.mlp.experts.122.down_proj", "model.layers.41.mlp.experts.123.down_proj", "model.layers.41.mlp.experts.124.down_proj", "model.layers.41.mlp.experts.125.down_proj", "model.layers.41.mlp.experts.126.down_proj", "model.layers.41.mlp.experts.127.down_proj", "model.layers.41.mlp.experts.128.down_proj", "model.layers.41.mlp.experts.129.down_proj", "model.layers.41.mlp.experts.130.down_proj", "model.layers.41.mlp.experts.131.down_proj", "model.layers.41.mlp.experts.132.down_proj", "model.layers.41.mlp.experts.133.down_proj", "model.layers.41.mlp.experts.134.down_proj", "model.layers.41.mlp.experts.135.down_proj", "model.layers.41.mlp.experts.136.down_proj", "model.layers.41.mlp.experts.137.down_proj", "model.layers.41.mlp.experts.138.down_proj", "model.layers.41.mlp.experts.139.down_proj", "model.layers.41.mlp.experts.140.down_proj", "model.layers.41.mlp.experts.141.down_proj", "model.layers.41.mlp.experts.142.down_proj", "model.layers.41.mlp.experts.143.down_proj", "model.layers.41.mlp.experts.144.down_proj", "model.layers.41.mlp.experts.145.down_proj", "model.layers.41.mlp.experts.146.down_proj", "model.layers.41.mlp.experts.147.down_proj", "model.layers.41.mlp.experts.148.down_proj", "model.layers.41.mlp.experts.149.down_proj", "model.layers.41.mlp.experts.150.down_proj", "model.layers.41.mlp.experts.151.down_proj", "model.layers.41.mlp.experts.152.down_proj", "model.layers.41.mlp.experts.153.down_proj", "model.layers.41.mlp.experts.154.down_proj", "model.layers.41.mlp.experts.155.down_proj", "model.layers.41.mlp.experts.156.down_proj", "model.layers.41.mlp.experts.157.down_proj", "model.layers.41.mlp.experts.158.down_proj", "model.layers.41.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0022965490818023793, "dbits": 1258291200 } ] }, { "idx": 249, "layers": [ "model.layers.42.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0028582409024238697, "dbits": 62914560 } ] }, { "idx": 250, "layers": [ "model.layers.42.self_attn.k_proj", "model.layers.42.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0008894287049770466, "dbits": 10485760 } ] }, { "idx": 251, "layers": [ "model.layers.42.self_attn.o_proj" ], "candidates": [ { "dkld": -0.002521768957376491, "dbits": 62914560 } ] }, { "idx": 252, "layers": [ "model.layers.42.mlp.shared_experts.gate_proj", "model.layers.42.mlp.shared_experts.up_proj", "model.layers.42.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00011478736996650696, "dbits": 23592960 } ] }, { "idx": 253, "layers": [ "model.layers.42.mlp.experts.0.gate_proj", "model.layers.42.mlp.experts.1.gate_proj", "model.layers.42.mlp.experts.2.gate_proj", "model.layers.42.mlp.experts.3.gate_proj", "model.layers.42.mlp.experts.4.gate_proj", "model.layers.42.mlp.experts.5.gate_proj", "model.layers.42.mlp.experts.6.gate_proj", "model.layers.42.mlp.experts.7.gate_proj", "model.layers.42.mlp.experts.8.gate_proj", "model.layers.42.mlp.experts.9.gate_proj", "model.layers.42.mlp.experts.10.gate_proj", "model.layers.42.mlp.experts.11.gate_proj", "model.layers.42.mlp.experts.12.gate_proj", "model.layers.42.mlp.experts.13.gate_proj", "model.layers.42.mlp.experts.14.gate_proj", "model.layers.42.mlp.experts.15.gate_proj", "model.layers.42.mlp.experts.16.gate_proj", "model.layers.42.mlp.experts.17.gate_proj", "model.layers.42.mlp.experts.18.gate_proj", "model.layers.42.mlp.experts.19.gate_proj", "model.layers.42.mlp.experts.20.gate_proj", "model.layers.42.mlp.experts.21.gate_proj", "model.layers.42.mlp.experts.22.gate_proj", "model.layers.42.mlp.experts.23.gate_proj", "model.layers.42.mlp.experts.24.gate_proj", "model.layers.42.mlp.experts.25.gate_proj", "model.layers.42.mlp.experts.26.gate_proj", "model.layers.42.mlp.experts.27.gate_proj", "model.layers.42.mlp.experts.28.gate_proj", "model.layers.42.mlp.experts.29.gate_proj", "model.layers.42.mlp.experts.30.gate_proj", "model.layers.42.mlp.experts.31.gate_proj", "model.layers.42.mlp.experts.32.gate_proj", "model.layers.42.mlp.experts.33.gate_proj", "model.layers.42.mlp.experts.34.gate_proj", "model.layers.42.mlp.experts.35.gate_proj", "model.layers.42.mlp.experts.36.gate_proj", "model.layers.42.mlp.experts.37.gate_proj", "model.layers.42.mlp.experts.38.gate_proj", "model.layers.42.mlp.experts.39.gate_proj", "model.layers.42.mlp.experts.40.gate_proj", "model.layers.42.mlp.experts.41.gate_proj", "model.layers.42.mlp.experts.42.gate_proj", "model.layers.42.mlp.experts.43.gate_proj", "model.layers.42.mlp.experts.44.gate_proj", "model.layers.42.mlp.experts.45.gate_proj", "model.layers.42.mlp.experts.46.gate_proj", "model.layers.42.mlp.experts.47.gate_proj", "model.layers.42.mlp.experts.48.gate_proj", "model.layers.42.mlp.experts.49.gate_proj", "model.layers.42.mlp.experts.50.gate_proj", "model.layers.42.mlp.experts.51.gate_proj", "model.layers.42.mlp.experts.52.gate_proj", "model.layers.42.mlp.experts.53.gate_proj", "model.layers.42.mlp.experts.54.gate_proj", "model.layers.42.mlp.experts.55.gate_proj", "model.layers.42.mlp.experts.56.gate_proj", "model.layers.42.mlp.experts.57.gate_proj", "model.layers.42.mlp.experts.58.gate_proj", "model.layers.42.mlp.experts.59.gate_proj", "model.layers.42.mlp.experts.60.gate_proj", "model.layers.42.mlp.experts.61.gate_proj", "model.layers.42.mlp.experts.62.gate_proj", "model.layers.42.mlp.experts.63.gate_proj", "model.layers.42.mlp.experts.64.gate_proj", "model.layers.42.mlp.experts.65.gate_proj", "model.layers.42.mlp.experts.66.gate_proj", "model.layers.42.mlp.experts.67.gate_proj", "model.layers.42.mlp.experts.68.gate_proj", "model.layers.42.mlp.experts.69.gate_proj", "model.layers.42.mlp.experts.70.gate_proj", "model.layers.42.mlp.experts.71.gate_proj", "model.layers.42.mlp.experts.72.gate_proj", "model.layers.42.mlp.experts.73.gate_proj", "model.layers.42.mlp.experts.74.gate_proj", "model.layers.42.mlp.experts.75.gate_proj", "model.layers.42.mlp.experts.76.gate_proj", "model.layers.42.mlp.experts.77.gate_proj", "model.layers.42.mlp.experts.78.gate_proj", "model.layers.42.mlp.experts.79.gate_proj", "model.layers.42.mlp.experts.80.gate_proj", "model.layers.42.mlp.experts.81.gate_proj", "model.layers.42.mlp.experts.82.gate_proj", "model.layers.42.mlp.experts.83.gate_proj", "model.layers.42.mlp.experts.84.gate_proj", "model.layers.42.mlp.experts.85.gate_proj", "model.layers.42.mlp.experts.86.gate_proj", "model.layers.42.mlp.experts.87.gate_proj", "model.layers.42.mlp.experts.88.gate_proj", "model.layers.42.mlp.experts.89.gate_proj", "model.layers.42.mlp.experts.90.gate_proj", "model.layers.42.mlp.experts.91.gate_proj", "model.layers.42.mlp.experts.92.gate_proj", "model.layers.42.mlp.experts.93.gate_proj", "model.layers.42.mlp.experts.94.gate_proj", "model.layers.42.mlp.experts.95.gate_proj", "model.layers.42.mlp.experts.96.gate_proj", "model.layers.42.mlp.experts.97.gate_proj", "model.layers.42.mlp.experts.98.gate_proj", "model.layers.42.mlp.experts.99.gate_proj", "model.layers.42.mlp.experts.100.gate_proj", "model.layers.42.mlp.experts.101.gate_proj", "model.layers.42.mlp.experts.102.gate_proj", "model.layers.42.mlp.experts.103.gate_proj", "model.layers.42.mlp.experts.104.gate_proj", "model.layers.42.mlp.experts.105.gate_proj", "model.layers.42.mlp.experts.106.gate_proj", "model.layers.42.mlp.experts.107.gate_proj", "model.layers.42.mlp.experts.108.gate_proj", "model.layers.42.mlp.experts.109.gate_proj", "model.layers.42.mlp.experts.110.gate_proj", "model.layers.42.mlp.experts.111.gate_proj", "model.layers.42.mlp.experts.112.gate_proj", "model.layers.42.mlp.experts.113.gate_proj", "model.layers.42.mlp.experts.114.gate_proj", "model.layers.42.mlp.experts.115.gate_proj", "model.layers.42.mlp.experts.116.gate_proj", "model.layers.42.mlp.experts.117.gate_proj", "model.layers.42.mlp.experts.118.gate_proj", "model.layers.42.mlp.experts.119.gate_proj", "model.layers.42.mlp.experts.120.gate_proj", "model.layers.42.mlp.experts.121.gate_proj", "model.layers.42.mlp.experts.122.gate_proj", "model.layers.42.mlp.experts.123.gate_proj", "model.layers.42.mlp.experts.124.gate_proj", "model.layers.42.mlp.experts.125.gate_proj", "model.layers.42.mlp.experts.126.gate_proj", "model.layers.42.mlp.experts.127.gate_proj", "model.layers.42.mlp.experts.128.gate_proj", "model.layers.42.mlp.experts.129.gate_proj", "model.layers.42.mlp.experts.130.gate_proj", "model.layers.42.mlp.experts.131.gate_proj", "model.layers.42.mlp.experts.132.gate_proj", "model.layers.42.mlp.experts.133.gate_proj", "model.layers.42.mlp.experts.134.gate_proj", "model.layers.42.mlp.experts.135.gate_proj", "model.layers.42.mlp.experts.136.gate_proj", "model.layers.42.mlp.experts.137.gate_proj", "model.layers.42.mlp.experts.138.gate_proj", "model.layers.42.mlp.experts.139.gate_proj", "model.layers.42.mlp.experts.140.gate_proj", "model.layers.42.mlp.experts.141.gate_proj", "model.layers.42.mlp.experts.142.gate_proj", "model.layers.42.mlp.experts.143.gate_proj", "model.layers.42.mlp.experts.144.gate_proj", "model.layers.42.mlp.experts.145.gate_proj", "model.layers.42.mlp.experts.146.gate_proj", "model.layers.42.mlp.experts.147.gate_proj", "model.layers.42.mlp.experts.148.gate_proj", "model.layers.42.mlp.experts.149.gate_proj", "model.layers.42.mlp.experts.150.gate_proj", "model.layers.42.mlp.experts.151.gate_proj", "model.layers.42.mlp.experts.152.gate_proj", "model.layers.42.mlp.experts.153.gate_proj", "model.layers.42.mlp.experts.154.gate_proj", "model.layers.42.mlp.experts.155.gate_proj", "model.layers.42.mlp.experts.156.gate_proj", "model.layers.42.mlp.experts.157.gate_proj", "model.layers.42.mlp.experts.158.gate_proj", "model.layers.42.mlp.experts.159.gate_proj", "model.layers.42.mlp.experts.0.up_proj", "model.layers.42.mlp.experts.1.up_proj", "model.layers.42.mlp.experts.2.up_proj", "model.layers.42.mlp.experts.3.up_proj", "model.layers.42.mlp.experts.4.up_proj", "model.layers.42.mlp.experts.5.up_proj", "model.layers.42.mlp.experts.6.up_proj", "model.layers.42.mlp.experts.7.up_proj", "model.layers.42.mlp.experts.8.up_proj", "model.layers.42.mlp.experts.9.up_proj", "model.layers.42.mlp.experts.10.up_proj", "model.layers.42.mlp.experts.11.up_proj", "model.layers.42.mlp.experts.12.up_proj", "model.layers.42.mlp.experts.13.up_proj", "model.layers.42.mlp.experts.14.up_proj", "model.layers.42.mlp.experts.15.up_proj", "model.layers.42.mlp.experts.16.up_proj", "model.layers.42.mlp.experts.17.up_proj", "model.layers.42.mlp.experts.18.up_proj", "model.layers.42.mlp.experts.19.up_proj", "model.layers.42.mlp.experts.20.up_proj", "model.layers.42.mlp.experts.21.up_proj", "model.layers.42.mlp.experts.22.up_proj", "model.layers.42.mlp.experts.23.up_proj", "model.layers.42.mlp.experts.24.up_proj", "model.layers.42.mlp.experts.25.up_proj", "model.layers.42.mlp.experts.26.up_proj", "model.layers.42.mlp.experts.27.up_proj", "model.layers.42.mlp.experts.28.up_proj", "model.layers.42.mlp.experts.29.up_proj", "model.layers.42.mlp.experts.30.up_proj", "model.layers.42.mlp.experts.31.up_proj", "model.layers.42.mlp.experts.32.up_proj", "model.layers.42.mlp.experts.33.up_proj", "model.layers.42.mlp.experts.34.up_proj", "model.layers.42.mlp.experts.35.up_proj", "model.layers.42.mlp.experts.36.up_proj", "model.layers.42.mlp.experts.37.up_proj", "model.layers.42.mlp.experts.38.up_proj", "model.layers.42.mlp.experts.39.up_proj", "model.layers.42.mlp.experts.40.up_proj", "model.layers.42.mlp.experts.41.up_proj", "model.layers.42.mlp.experts.42.up_proj", "model.layers.42.mlp.experts.43.up_proj", "model.layers.42.mlp.experts.44.up_proj", "model.layers.42.mlp.experts.45.up_proj", "model.layers.42.mlp.experts.46.up_proj", "model.layers.42.mlp.experts.47.up_proj", "model.layers.42.mlp.experts.48.up_proj", "model.layers.42.mlp.experts.49.up_proj", "model.layers.42.mlp.experts.50.up_proj", "model.layers.42.mlp.experts.51.up_proj", "model.layers.42.mlp.experts.52.up_proj", "model.layers.42.mlp.experts.53.up_proj", "model.layers.42.mlp.experts.54.up_proj", "model.layers.42.mlp.experts.55.up_proj", "model.layers.42.mlp.experts.56.up_proj", "model.layers.42.mlp.experts.57.up_proj", "model.layers.42.mlp.experts.58.up_proj", "model.layers.42.mlp.experts.59.up_proj", "model.layers.42.mlp.experts.60.up_proj", "model.layers.42.mlp.experts.61.up_proj", "model.layers.42.mlp.experts.62.up_proj", "model.layers.42.mlp.experts.63.up_proj", "model.layers.42.mlp.experts.64.up_proj", "model.layers.42.mlp.experts.65.up_proj", "model.layers.42.mlp.experts.66.up_proj", "model.layers.42.mlp.experts.67.up_proj", "model.layers.42.mlp.experts.68.up_proj", "model.layers.42.mlp.experts.69.up_proj", "model.layers.42.mlp.experts.70.up_proj", "model.layers.42.mlp.experts.71.up_proj", "model.layers.42.mlp.experts.72.up_proj", "model.layers.42.mlp.experts.73.up_proj", "model.layers.42.mlp.experts.74.up_proj", "model.layers.42.mlp.experts.75.up_proj", "model.layers.42.mlp.experts.76.up_proj", "model.layers.42.mlp.experts.77.up_proj", "model.layers.42.mlp.experts.78.up_proj", "model.layers.42.mlp.experts.79.up_proj", "model.layers.42.mlp.experts.80.up_proj", "model.layers.42.mlp.experts.81.up_proj", "model.layers.42.mlp.experts.82.up_proj", "model.layers.42.mlp.experts.83.up_proj", "model.layers.42.mlp.experts.84.up_proj", "model.layers.42.mlp.experts.85.up_proj", "model.layers.42.mlp.experts.86.up_proj", "model.layers.42.mlp.experts.87.up_proj", "model.layers.42.mlp.experts.88.up_proj", "model.layers.42.mlp.experts.89.up_proj", "model.layers.42.mlp.experts.90.up_proj", "model.layers.42.mlp.experts.91.up_proj", "model.layers.42.mlp.experts.92.up_proj", "model.layers.42.mlp.experts.93.up_proj", "model.layers.42.mlp.experts.94.up_proj", "model.layers.42.mlp.experts.95.up_proj", "model.layers.42.mlp.experts.96.up_proj", "model.layers.42.mlp.experts.97.up_proj", "model.layers.42.mlp.experts.98.up_proj", "model.layers.42.mlp.experts.99.up_proj", "model.layers.42.mlp.experts.100.up_proj", "model.layers.42.mlp.experts.101.up_proj", "model.layers.42.mlp.experts.102.up_proj", "model.layers.42.mlp.experts.103.up_proj", "model.layers.42.mlp.experts.104.up_proj", "model.layers.42.mlp.experts.105.up_proj", "model.layers.42.mlp.experts.106.up_proj", "model.layers.42.mlp.experts.107.up_proj", "model.layers.42.mlp.experts.108.up_proj", "model.layers.42.mlp.experts.109.up_proj", "model.layers.42.mlp.experts.110.up_proj", "model.layers.42.mlp.experts.111.up_proj", "model.layers.42.mlp.experts.112.up_proj", "model.layers.42.mlp.experts.113.up_proj", "model.layers.42.mlp.experts.114.up_proj", "model.layers.42.mlp.experts.115.up_proj", "model.layers.42.mlp.experts.116.up_proj", "model.layers.42.mlp.experts.117.up_proj", "model.layers.42.mlp.experts.118.up_proj", "model.layers.42.mlp.experts.119.up_proj", "model.layers.42.mlp.experts.120.up_proj", "model.layers.42.mlp.experts.121.up_proj", "model.layers.42.mlp.experts.122.up_proj", "model.layers.42.mlp.experts.123.up_proj", "model.layers.42.mlp.experts.124.up_proj", "model.layers.42.mlp.experts.125.up_proj", "model.layers.42.mlp.experts.126.up_proj", "model.layers.42.mlp.experts.127.up_proj", "model.layers.42.mlp.experts.128.up_proj", "model.layers.42.mlp.experts.129.up_proj", "model.layers.42.mlp.experts.130.up_proj", "model.layers.42.mlp.experts.131.up_proj", "model.layers.42.mlp.experts.132.up_proj", "model.layers.42.mlp.experts.133.up_proj", "model.layers.42.mlp.experts.134.up_proj", "model.layers.42.mlp.experts.135.up_proj", "model.layers.42.mlp.experts.136.up_proj", "model.layers.42.mlp.experts.137.up_proj", "model.layers.42.mlp.experts.138.up_proj", "model.layers.42.mlp.experts.139.up_proj", "model.layers.42.mlp.experts.140.up_proj", "model.layers.42.mlp.experts.141.up_proj", "model.layers.42.mlp.experts.142.up_proj", "model.layers.42.mlp.experts.143.up_proj", "model.layers.42.mlp.experts.144.up_proj", "model.layers.42.mlp.experts.145.up_proj", "model.layers.42.mlp.experts.146.up_proj", "model.layers.42.mlp.experts.147.up_proj", "model.layers.42.mlp.experts.148.up_proj", "model.layers.42.mlp.experts.149.up_proj", "model.layers.42.mlp.experts.150.up_proj", "model.layers.42.mlp.experts.151.up_proj", "model.layers.42.mlp.experts.152.up_proj", "model.layers.42.mlp.experts.153.up_proj", "model.layers.42.mlp.experts.154.up_proj", "model.layers.42.mlp.experts.155.up_proj", "model.layers.42.mlp.experts.156.up_proj", "model.layers.42.mlp.experts.157.up_proj", "model.layers.42.mlp.experts.158.up_proj", "model.layers.42.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.001696833223104488, "dbits": 2516582400 } ] }, { "idx": 254, "layers": [ "model.layers.42.mlp.experts.0.down_proj", "model.layers.42.mlp.experts.1.down_proj", "model.layers.42.mlp.experts.2.down_proj", "model.layers.42.mlp.experts.3.down_proj", "model.layers.42.mlp.experts.4.down_proj", "model.layers.42.mlp.experts.5.down_proj", "model.layers.42.mlp.experts.6.down_proj", "model.layers.42.mlp.experts.7.down_proj", "model.layers.42.mlp.experts.8.down_proj", "model.layers.42.mlp.experts.9.down_proj", "model.layers.42.mlp.experts.10.down_proj", "model.layers.42.mlp.experts.11.down_proj", "model.layers.42.mlp.experts.12.down_proj", "model.layers.42.mlp.experts.13.down_proj", "model.layers.42.mlp.experts.14.down_proj", "model.layers.42.mlp.experts.15.down_proj", "model.layers.42.mlp.experts.16.down_proj", "model.layers.42.mlp.experts.17.down_proj", "model.layers.42.mlp.experts.18.down_proj", "model.layers.42.mlp.experts.19.down_proj", "model.layers.42.mlp.experts.20.down_proj", "model.layers.42.mlp.experts.21.down_proj", "model.layers.42.mlp.experts.22.down_proj", "model.layers.42.mlp.experts.23.down_proj", "model.layers.42.mlp.experts.24.down_proj", "model.layers.42.mlp.experts.25.down_proj", "model.layers.42.mlp.experts.26.down_proj", "model.layers.42.mlp.experts.27.down_proj", "model.layers.42.mlp.experts.28.down_proj", "model.layers.42.mlp.experts.29.down_proj", "model.layers.42.mlp.experts.30.down_proj", "model.layers.42.mlp.experts.31.down_proj", "model.layers.42.mlp.experts.32.down_proj", "model.layers.42.mlp.experts.33.down_proj", "model.layers.42.mlp.experts.34.down_proj", "model.layers.42.mlp.experts.35.down_proj", "model.layers.42.mlp.experts.36.down_proj", "model.layers.42.mlp.experts.37.down_proj", "model.layers.42.mlp.experts.38.down_proj", "model.layers.42.mlp.experts.39.down_proj", "model.layers.42.mlp.experts.40.down_proj", "model.layers.42.mlp.experts.41.down_proj", "model.layers.42.mlp.experts.42.down_proj", "model.layers.42.mlp.experts.43.down_proj", "model.layers.42.mlp.experts.44.down_proj", "model.layers.42.mlp.experts.45.down_proj", "model.layers.42.mlp.experts.46.down_proj", "model.layers.42.mlp.experts.47.down_proj", "model.layers.42.mlp.experts.48.down_proj", "model.layers.42.mlp.experts.49.down_proj", "model.layers.42.mlp.experts.50.down_proj", "model.layers.42.mlp.experts.51.down_proj", "model.layers.42.mlp.experts.52.down_proj", "model.layers.42.mlp.experts.53.down_proj", "model.layers.42.mlp.experts.54.down_proj", "model.layers.42.mlp.experts.55.down_proj", "model.layers.42.mlp.experts.56.down_proj", "model.layers.42.mlp.experts.57.down_proj", "model.layers.42.mlp.experts.58.down_proj", "model.layers.42.mlp.experts.59.down_proj", "model.layers.42.mlp.experts.60.down_proj", "model.layers.42.mlp.experts.61.down_proj", "model.layers.42.mlp.experts.62.down_proj", "model.layers.42.mlp.experts.63.down_proj", "model.layers.42.mlp.experts.64.down_proj", "model.layers.42.mlp.experts.65.down_proj", "model.layers.42.mlp.experts.66.down_proj", "model.layers.42.mlp.experts.67.down_proj", "model.layers.42.mlp.experts.68.down_proj", "model.layers.42.mlp.experts.69.down_proj", "model.layers.42.mlp.experts.70.down_proj", "model.layers.42.mlp.experts.71.down_proj", "model.layers.42.mlp.experts.72.down_proj", "model.layers.42.mlp.experts.73.down_proj", "model.layers.42.mlp.experts.74.down_proj", "model.layers.42.mlp.experts.75.down_proj", "model.layers.42.mlp.experts.76.down_proj", "model.layers.42.mlp.experts.77.down_proj", "model.layers.42.mlp.experts.78.down_proj", "model.layers.42.mlp.experts.79.down_proj", "model.layers.42.mlp.experts.80.down_proj", "model.layers.42.mlp.experts.81.down_proj", "model.layers.42.mlp.experts.82.down_proj", "model.layers.42.mlp.experts.83.down_proj", "model.layers.42.mlp.experts.84.down_proj", "model.layers.42.mlp.experts.85.down_proj", "model.layers.42.mlp.experts.86.down_proj", "model.layers.42.mlp.experts.87.down_proj", "model.layers.42.mlp.experts.88.down_proj", "model.layers.42.mlp.experts.89.down_proj", "model.layers.42.mlp.experts.90.down_proj", "model.layers.42.mlp.experts.91.down_proj", "model.layers.42.mlp.experts.92.down_proj", "model.layers.42.mlp.experts.93.down_proj", "model.layers.42.mlp.experts.94.down_proj", "model.layers.42.mlp.experts.95.down_proj", "model.layers.42.mlp.experts.96.down_proj", "model.layers.42.mlp.experts.97.down_proj", "model.layers.42.mlp.experts.98.down_proj", "model.layers.42.mlp.experts.99.down_proj", "model.layers.42.mlp.experts.100.down_proj", "model.layers.42.mlp.experts.101.down_proj", "model.layers.42.mlp.experts.102.down_proj", "model.layers.42.mlp.experts.103.down_proj", "model.layers.42.mlp.experts.104.down_proj", "model.layers.42.mlp.experts.105.down_proj", "model.layers.42.mlp.experts.106.down_proj", "model.layers.42.mlp.experts.107.down_proj", "model.layers.42.mlp.experts.108.down_proj", "model.layers.42.mlp.experts.109.down_proj", "model.layers.42.mlp.experts.110.down_proj", "model.layers.42.mlp.experts.111.down_proj", "model.layers.42.mlp.experts.112.down_proj", "model.layers.42.mlp.experts.113.down_proj", "model.layers.42.mlp.experts.114.down_proj", "model.layers.42.mlp.experts.115.down_proj", "model.layers.42.mlp.experts.116.down_proj", "model.layers.42.mlp.experts.117.down_proj", "model.layers.42.mlp.experts.118.down_proj", "model.layers.42.mlp.experts.119.down_proj", "model.layers.42.mlp.experts.120.down_proj", "model.layers.42.mlp.experts.121.down_proj", "model.layers.42.mlp.experts.122.down_proj", "model.layers.42.mlp.experts.123.down_proj", "model.layers.42.mlp.experts.124.down_proj", "model.layers.42.mlp.experts.125.down_proj", "model.layers.42.mlp.experts.126.down_proj", "model.layers.42.mlp.experts.127.down_proj", "model.layers.42.mlp.experts.128.down_proj", "model.layers.42.mlp.experts.129.down_proj", "model.layers.42.mlp.experts.130.down_proj", "model.layers.42.mlp.experts.131.down_proj", "model.layers.42.mlp.experts.132.down_proj", "model.layers.42.mlp.experts.133.down_proj", "model.layers.42.mlp.experts.134.down_proj", "model.layers.42.mlp.experts.135.down_proj", "model.layers.42.mlp.experts.136.down_proj", "model.layers.42.mlp.experts.137.down_proj", "model.layers.42.mlp.experts.138.down_proj", "model.layers.42.mlp.experts.139.down_proj", "model.layers.42.mlp.experts.140.down_proj", "model.layers.42.mlp.experts.141.down_proj", "model.layers.42.mlp.experts.142.down_proj", "model.layers.42.mlp.experts.143.down_proj", "model.layers.42.mlp.experts.144.down_proj", "model.layers.42.mlp.experts.145.down_proj", "model.layers.42.mlp.experts.146.down_proj", "model.layers.42.mlp.experts.147.down_proj", "model.layers.42.mlp.experts.148.down_proj", "model.layers.42.mlp.experts.149.down_proj", "model.layers.42.mlp.experts.150.down_proj", "model.layers.42.mlp.experts.151.down_proj", "model.layers.42.mlp.experts.152.down_proj", "model.layers.42.mlp.experts.153.down_proj", "model.layers.42.mlp.experts.154.down_proj", "model.layers.42.mlp.experts.155.down_proj", "model.layers.42.mlp.experts.156.down_proj", "model.layers.42.mlp.experts.157.down_proj", "model.layers.42.mlp.experts.158.down_proj", "model.layers.42.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0019430026412010082, "dbits": 1258291200 } ] }, { "idx": 255, "layers": [ "model.layers.43.self_attn.q_proj" ], "candidates": [ { "dkld": -0.002842555195093188, "dbits": 62914560 } ] }, { "idx": 256, "layers": [ "model.layers.43.self_attn.k_proj", "model.layers.43.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0019427843391895405, "dbits": 10485760 } ] }, { "idx": 257, "layers": [ "model.layers.43.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0008782424032688363, "dbits": 62914560 } ] }, { "idx": 258, "layers": [ "model.layers.43.mlp.shared_experts.gate_proj", "model.layers.43.mlp.shared_experts.up_proj", "model.layers.43.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0018936388194560894, "dbits": 23592960 } ] }, { "idx": 259, "layers": [ "model.layers.43.mlp.experts.0.gate_proj", "model.layers.43.mlp.experts.1.gate_proj", "model.layers.43.mlp.experts.2.gate_proj", "model.layers.43.mlp.experts.3.gate_proj", "model.layers.43.mlp.experts.4.gate_proj", "model.layers.43.mlp.experts.5.gate_proj", "model.layers.43.mlp.experts.6.gate_proj", "model.layers.43.mlp.experts.7.gate_proj", "model.layers.43.mlp.experts.8.gate_proj", "model.layers.43.mlp.experts.9.gate_proj", "model.layers.43.mlp.experts.10.gate_proj", "model.layers.43.mlp.experts.11.gate_proj", "model.layers.43.mlp.experts.12.gate_proj", "model.layers.43.mlp.experts.13.gate_proj", "model.layers.43.mlp.experts.14.gate_proj", "model.layers.43.mlp.experts.15.gate_proj", "model.layers.43.mlp.experts.16.gate_proj", "model.layers.43.mlp.experts.17.gate_proj", "model.layers.43.mlp.experts.18.gate_proj", "model.layers.43.mlp.experts.19.gate_proj", "model.layers.43.mlp.experts.20.gate_proj", "model.layers.43.mlp.experts.21.gate_proj", "model.layers.43.mlp.experts.22.gate_proj", "model.layers.43.mlp.experts.23.gate_proj", "model.layers.43.mlp.experts.24.gate_proj", "model.layers.43.mlp.experts.25.gate_proj", "model.layers.43.mlp.experts.26.gate_proj", "model.layers.43.mlp.experts.27.gate_proj", "model.layers.43.mlp.experts.28.gate_proj", "model.layers.43.mlp.experts.29.gate_proj", "model.layers.43.mlp.experts.30.gate_proj", "model.layers.43.mlp.experts.31.gate_proj", "model.layers.43.mlp.experts.32.gate_proj", "model.layers.43.mlp.experts.33.gate_proj", "model.layers.43.mlp.experts.34.gate_proj", "model.layers.43.mlp.experts.35.gate_proj", "model.layers.43.mlp.experts.36.gate_proj", "model.layers.43.mlp.experts.37.gate_proj", "model.layers.43.mlp.experts.38.gate_proj", "model.layers.43.mlp.experts.39.gate_proj", "model.layers.43.mlp.experts.40.gate_proj", "model.layers.43.mlp.experts.41.gate_proj", "model.layers.43.mlp.experts.42.gate_proj", "model.layers.43.mlp.experts.43.gate_proj", "model.layers.43.mlp.experts.44.gate_proj", "model.layers.43.mlp.experts.45.gate_proj", "model.layers.43.mlp.experts.46.gate_proj", "model.layers.43.mlp.experts.47.gate_proj", "model.layers.43.mlp.experts.48.gate_proj", "model.layers.43.mlp.experts.49.gate_proj", "model.layers.43.mlp.experts.50.gate_proj", "model.layers.43.mlp.experts.51.gate_proj", "model.layers.43.mlp.experts.52.gate_proj", "model.layers.43.mlp.experts.53.gate_proj", "model.layers.43.mlp.experts.54.gate_proj", "model.layers.43.mlp.experts.55.gate_proj", "model.layers.43.mlp.experts.56.gate_proj", "model.layers.43.mlp.experts.57.gate_proj", "model.layers.43.mlp.experts.58.gate_proj", "model.layers.43.mlp.experts.59.gate_proj", "model.layers.43.mlp.experts.60.gate_proj", "model.layers.43.mlp.experts.61.gate_proj", "model.layers.43.mlp.experts.62.gate_proj", "model.layers.43.mlp.experts.63.gate_proj", "model.layers.43.mlp.experts.64.gate_proj", "model.layers.43.mlp.experts.65.gate_proj", "model.layers.43.mlp.experts.66.gate_proj", "model.layers.43.mlp.experts.67.gate_proj", "model.layers.43.mlp.experts.68.gate_proj", "model.layers.43.mlp.experts.69.gate_proj", "model.layers.43.mlp.experts.70.gate_proj", "model.layers.43.mlp.experts.71.gate_proj", "model.layers.43.mlp.experts.72.gate_proj", "model.layers.43.mlp.experts.73.gate_proj", "model.layers.43.mlp.experts.74.gate_proj", "model.layers.43.mlp.experts.75.gate_proj", "model.layers.43.mlp.experts.76.gate_proj", "model.layers.43.mlp.experts.77.gate_proj", "model.layers.43.mlp.experts.78.gate_proj", "model.layers.43.mlp.experts.79.gate_proj", "model.layers.43.mlp.experts.80.gate_proj", "model.layers.43.mlp.experts.81.gate_proj", "model.layers.43.mlp.experts.82.gate_proj", "model.layers.43.mlp.experts.83.gate_proj", "model.layers.43.mlp.experts.84.gate_proj", "model.layers.43.mlp.experts.85.gate_proj", "model.layers.43.mlp.experts.86.gate_proj", "model.layers.43.mlp.experts.87.gate_proj", "model.layers.43.mlp.experts.88.gate_proj", "model.layers.43.mlp.experts.89.gate_proj", "model.layers.43.mlp.experts.90.gate_proj", "model.layers.43.mlp.experts.91.gate_proj", "model.layers.43.mlp.experts.92.gate_proj", "model.layers.43.mlp.experts.93.gate_proj", "model.layers.43.mlp.experts.94.gate_proj", "model.layers.43.mlp.experts.95.gate_proj", "model.layers.43.mlp.experts.96.gate_proj", "model.layers.43.mlp.experts.97.gate_proj", "model.layers.43.mlp.experts.98.gate_proj", "model.layers.43.mlp.experts.99.gate_proj", "model.layers.43.mlp.experts.100.gate_proj", "model.layers.43.mlp.experts.101.gate_proj", "model.layers.43.mlp.experts.102.gate_proj", "model.layers.43.mlp.experts.103.gate_proj", "model.layers.43.mlp.experts.104.gate_proj", "model.layers.43.mlp.experts.105.gate_proj", "model.layers.43.mlp.experts.106.gate_proj", "model.layers.43.mlp.experts.107.gate_proj", "model.layers.43.mlp.experts.108.gate_proj", "model.layers.43.mlp.experts.109.gate_proj", "model.layers.43.mlp.experts.110.gate_proj", "model.layers.43.mlp.experts.111.gate_proj", "model.layers.43.mlp.experts.112.gate_proj", "model.layers.43.mlp.experts.113.gate_proj", "model.layers.43.mlp.experts.114.gate_proj", "model.layers.43.mlp.experts.115.gate_proj", "model.layers.43.mlp.experts.116.gate_proj", "model.layers.43.mlp.experts.117.gate_proj", "model.layers.43.mlp.experts.118.gate_proj", "model.layers.43.mlp.experts.119.gate_proj", "model.layers.43.mlp.experts.120.gate_proj", "model.layers.43.mlp.experts.121.gate_proj", "model.layers.43.mlp.experts.122.gate_proj", "model.layers.43.mlp.experts.123.gate_proj", "model.layers.43.mlp.experts.124.gate_proj", "model.layers.43.mlp.experts.125.gate_proj", "model.layers.43.mlp.experts.126.gate_proj", "model.layers.43.mlp.experts.127.gate_proj", "model.layers.43.mlp.experts.128.gate_proj", "model.layers.43.mlp.experts.129.gate_proj", "model.layers.43.mlp.experts.130.gate_proj", "model.layers.43.mlp.experts.131.gate_proj", "model.layers.43.mlp.experts.132.gate_proj", "model.layers.43.mlp.experts.133.gate_proj", "model.layers.43.mlp.experts.134.gate_proj", "model.layers.43.mlp.experts.135.gate_proj", "model.layers.43.mlp.experts.136.gate_proj", "model.layers.43.mlp.experts.137.gate_proj", "model.layers.43.mlp.experts.138.gate_proj", "model.layers.43.mlp.experts.139.gate_proj", "model.layers.43.mlp.experts.140.gate_proj", "model.layers.43.mlp.experts.141.gate_proj", "model.layers.43.mlp.experts.142.gate_proj", "model.layers.43.mlp.experts.143.gate_proj", "model.layers.43.mlp.experts.144.gate_proj", "model.layers.43.mlp.experts.145.gate_proj", "model.layers.43.mlp.experts.146.gate_proj", "model.layers.43.mlp.experts.147.gate_proj", "model.layers.43.mlp.experts.148.gate_proj", "model.layers.43.mlp.experts.149.gate_proj", "model.layers.43.mlp.experts.150.gate_proj", "model.layers.43.mlp.experts.151.gate_proj", "model.layers.43.mlp.experts.152.gate_proj", "model.layers.43.mlp.experts.153.gate_proj", "model.layers.43.mlp.experts.154.gate_proj", "model.layers.43.mlp.experts.155.gate_proj", "model.layers.43.mlp.experts.156.gate_proj", "model.layers.43.mlp.experts.157.gate_proj", "model.layers.43.mlp.experts.158.gate_proj", "model.layers.43.mlp.experts.159.gate_proj", "model.layers.43.mlp.experts.0.up_proj", "model.layers.43.mlp.experts.1.up_proj", "model.layers.43.mlp.experts.2.up_proj", "model.layers.43.mlp.experts.3.up_proj", "model.layers.43.mlp.experts.4.up_proj", "model.layers.43.mlp.experts.5.up_proj", "model.layers.43.mlp.experts.6.up_proj", "model.layers.43.mlp.experts.7.up_proj", "model.layers.43.mlp.experts.8.up_proj", "model.layers.43.mlp.experts.9.up_proj", "model.layers.43.mlp.experts.10.up_proj", "model.layers.43.mlp.experts.11.up_proj", "model.layers.43.mlp.experts.12.up_proj", "model.layers.43.mlp.experts.13.up_proj", "model.layers.43.mlp.experts.14.up_proj", "model.layers.43.mlp.experts.15.up_proj", "model.layers.43.mlp.experts.16.up_proj", "model.layers.43.mlp.experts.17.up_proj", "model.layers.43.mlp.experts.18.up_proj", "model.layers.43.mlp.experts.19.up_proj", "model.layers.43.mlp.experts.20.up_proj", "model.layers.43.mlp.experts.21.up_proj", "model.layers.43.mlp.experts.22.up_proj", "model.layers.43.mlp.experts.23.up_proj", "model.layers.43.mlp.experts.24.up_proj", "model.layers.43.mlp.experts.25.up_proj", "model.layers.43.mlp.experts.26.up_proj", "model.layers.43.mlp.experts.27.up_proj", "model.layers.43.mlp.experts.28.up_proj", "model.layers.43.mlp.experts.29.up_proj", "model.layers.43.mlp.experts.30.up_proj", "model.layers.43.mlp.experts.31.up_proj", "model.layers.43.mlp.experts.32.up_proj", "model.layers.43.mlp.experts.33.up_proj", "model.layers.43.mlp.experts.34.up_proj", "model.layers.43.mlp.experts.35.up_proj", "model.layers.43.mlp.experts.36.up_proj", "model.layers.43.mlp.experts.37.up_proj", "model.layers.43.mlp.experts.38.up_proj", "model.layers.43.mlp.experts.39.up_proj", "model.layers.43.mlp.experts.40.up_proj", "model.layers.43.mlp.experts.41.up_proj", "model.layers.43.mlp.experts.42.up_proj", "model.layers.43.mlp.experts.43.up_proj", "model.layers.43.mlp.experts.44.up_proj", "model.layers.43.mlp.experts.45.up_proj", "model.layers.43.mlp.experts.46.up_proj", "model.layers.43.mlp.experts.47.up_proj", "model.layers.43.mlp.experts.48.up_proj", "model.layers.43.mlp.experts.49.up_proj", "model.layers.43.mlp.experts.50.up_proj", "model.layers.43.mlp.experts.51.up_proj", "model.layers.43.mlp.experts.52.up_proj", "model.layers.43.mlp.experts.53.up_proj", "model.layers.43.mlp.experts.54.up_proj", "model.layers.43.mlp.experts.55.up_proj", "model.layers.43.mlp.experts.56.up_proj", "model.layers.43.mlp.experts.57.up_proj", "model.layers.43.mlp.experts.58.up_proj", "model.layers.43.mlp.experts.59.up_proj", "model.layers.43.mlp.experts.60.up_proj", "model.layers.43.mlp.experts.61.up_proj", "model.layers.43.mlp.experts.62.up_proj", "model.layers.43.mlp.experts.63.up_proj", "model.layers.43.mlp.experts.64.up_proj", "model.layers.43.mlp.experts.65.up_proj", "model.layers.43.mlp.experts.66.up_proj", "model.layers.43.mlp.experts.67.up_proj", "model.layers.43.mlp.experts.68.up_proj", "model.layers.43.mlp.experts.69.up_proj", "model.layers.43.mlp.experts.70.up_proj", "model.layers.43.mlp.experts.71.up_proj", "model.layers.43.mlp.experts.72.up_proj", "model.layers.43.mlp.experts.73.up_proj", "model.layers.43.mlp.experts.74.up_proj", "model.layers.43.mlp.experts.75.up_proj", "model.layers.43.mlp.experts.76.up_proj", "model.layers.43.mlp.experts.77.up_proj", "model.layers.43.mlp.experts.78.up_proj", "model.layers.43.mlp.experts.79.up_proj", "model.layers.43.mlp.experts.80.up_proj", "model.layers.43.mlp.experts.81.up_proj", "model.layers.43.mlp.experts.82.up_proj", "model.layers.43.mlp.experts.83.up_proj", "model.layers.43.mlp.experts.84.up_proj", "model.layers.43.mlp.experts.85.up_proj", "model.layers.43.mlp.experts.86.up_proj", "model.layers.43.mlp.experts.87.up_proj", "model.layers.43.mlp.experts.88.up_proj", "model.layers.43.mlp.experts.89.up_proj", "model.layers.43.mlp.experts.90.up_proj", "model.layers.43.mlp.experts.91.up_proj", "model.layers.43.mlp.experts.92.up_proj", "model.layers.43.mlp.experts.93.up_proj", "model.layers.43.mlp.experts.94.up_proj", "model.layers.43.mlp.experts.95.up_proj", "model.layers.43.mlp.experts.96.up_proj", "model.layers.43.mlp.experts.97.up_proj", "model.layers.43.mlp.experts.98.up_proj", "model.layers.43.mlp.experts.99.up_proj", "model.layers.43.mlp.experts.100.up_proj", "model.layers.43.mlp.experts.101.up_proj", "model.layers.43.mlp.experts.102.up_proj", "model.layers.43.mlp.experts.103.up_proj", "model.layers.43.mlp.experts.104.up_proj", "model.layers.43.mlp.experts.105.up_proj", "model.layers.43.mlp.experts.106.up_proj", "model.layers.43.mlp.experts.107.up_proj", "model.layers.43.mlp.experts.108.up_proj", "model.layers.43.mlp.experts.109.up_proj", "model.layers.43.mlp.experts.110.up_proj", "model.layers.43.mlp.experts.111.up_proj", "model.layers.43.mlp.experts.112.up_proj", "model.layers.43.mlp.experts.113.up_proj", "model.layers.43.mlp.experts.114.up_proj", "model.layers.43.mlp.experts.115.up_proj", "model.layers.43.mlp.experts.116.up_proj", "model.layers.43.mlp.experts.117.up_proj", "model.layers.43.mlp.experts.118.up_proj", "model.layers.43.mlp.experts.119.up_proj", "model.layers.43.mlp.experts.120.up_proj", "model.layers.43.mlp.experts.121.up_proj", "model.layers.43.mlp.experts.122.up_proj", "model.layers.43.mlp.experts.123.up_proj", "model.layers.43.mlp.experts.124.up_proj", "model.layers.43.mlp.experts.125.up_proj", "model.layers.43.mlp.experts.126.up_proj", "model.layers.43.mlp.experts.127.up_proj", "model.layers.43.mlp.experts.128.up_proj", "model.layers.43.mlp.experts.129.up_proj", "model.layers.43.mlp.experts.130.up_proj", "model.layers.43.mlp.experts.131.up_proj", "model.layers.43.mlp.experts.132.up_proj", "model.layers.43.mlp.experts.133.up_proj", "model.layers.43.mlp.experts.134.up_proj", "model.layers.43.mlp.experts.135.up_proj", "model.layers.43.mlp.experts.136.up_proj", "model.layers.43.mlp.experts.137.up_proj", "model.layers.43.mlp.experts.138.up_proj", "model.layers.43.mlp.experts.139.up_proj", "model.layers.43.mlp.experts.140.up_proj", "model.layers.43.mlp.experts.141.up_proj", "model.layers.43.mlp.experts.142.up_proj", "model.layers.43.mlp.experts.143.up_proj", "model.layers.43.mlp.experts.144.up_proj", "model.layers.43.mlp.experts.145.up_proj", "model.layers.43.mlp.experts.146.up_proj", "model.layers.43.mlp.experts.147.up_proj", "model.layers.43.mlp.experts.148.up_proj", "model.layers.43.mlp.experts.149.up_proj", "model.layers.43.mlp.experts.150.up_proj", "model.layers.43.mlp.experts.151.up_proj", "model.layers.43.mlp.experts.152.up_proj", "model.layers.43.mlp.experts.153.up_proj", "model.layers.43.mlp.experts.154.up_proj", "model.layers.43.mlp.experts.155.up_proj", "model.layers.43.mlp.experts.156.up_proj", "model.layers.43.mlp.experts.157.up_proj", "model.layers.43.mlp.experts.158.up_proj", "model.layers.43.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0015744641423225403, "dbits": 2516582400 } ] }, { "idx": 260, "layers": [ "model.layers.43.mlp.experts.0.down_proj", "model.layers.43.mlp.experts.1.down_proj", "model.layers.43.mlp.experts.2.down_proj", "model.layers.43.mlp.experts.3.down_proj", "model.layers.43.mlp.experts.4.down_proj", "model.layers.43.mlp.experts.5.down_proj", "model.layers.43.mlp.experts.6.down_proj", "model.layers.43.mlp.experts.7.down_proj", "model.layers.43.mlp.experts.8.down_proj", "model.layers.43.mlp.experts.9.down_proj", "model.layers.43.mlp.experts.10.down_proj", "model.layers.43.mlp.experts.11.down_proj", "model.layers.43.mlp.experts.12.down_proj", "model.layers.43.mlp.experts.13.down_proj", "model.layers.43.mlp.experts.14.down_proj", "model.layers.43.mlp.experts.15.down_proj", "model.layers.43.mlp.experts.16.down_proj", "model.layers.43.mlp.experts.17.down_proj", "model.layers.43.mlp.experts.18.down_proj", "model.layers.43.mlp.experts.19.down_proj", "model.layers.43.mlp.experts.20.down_proj", "model.layers.43.mlp.experts.21.down_proj", "model.layers.43.mlp.experts.22.down_proj", "model.layers.43.mlp.experts.23.down_proj", "model.layers.43.mlp.experts.24.down_proj", "model.layers.43.mlp.experts.25.down_proj", "model.layers.43.mlp.experts.26.down_proj", "model.layers.43.mlp.experts.27.down_proj", "model.layers.43.mlp.experts.28.down_proj", "model.layers.43.mlp.experts.29.down_proj", "model.layers.43.mlp.experts.30.down_proj", "model.layers.43.mlp.experts.31.down_proj", "model.layers.43.mlp.experts.32.down_proj", "model.layers.43.mlp.experts.33.down_proj", "model.layers.43.mlp.experts.34.down_proj", "model.layers.43.mlp.experts.35.down_proj", "model.layers.43.mlp.experts.36.down_proj", "model.layers.43.mlp.experts.37.down_proj", "model.layers.43.mlp.experts.38.down_proj", "model.layers.43.mlp.experts.39.down_proj", "model.layers.43.mlp.experts.40.down_proj", "model.layers.43.mlp.experts.41.down_proj", "model.layers.43.mlp.experts.42.down_proj", "model.layers.43.mlp.experts.43.down_proj", "model.layers.43.mlp.experts.44.down_proj", "model.layers.43.mlp.experts.45.down_proj", "model.layers.43.mlp.experts.46.down_proj", "model.layers.43.mlp.experts.47.down_proj", "model.layers.43.mlp.experts.48.down_proj", "model.layers.43.mlp.experts.49.down_proj", "model.layers.43.mlp.experts.50.down_proj", "model.layers.43.mlp.experts.51.down_proj", "model.layers.43.mlp.experts.52.down_proj", "model.layers.43.mlp.experts.53.down_proj", "model.layers.43.mlp.experts.54.down_proj", "model.layers.43.mlp.experts.55.down_proj", "model.layers.43.mlp.experts.56.down_proj", "model.layers.43.mlp.experts.57.down_proj", "model.layers.43.mlp.experts.58.down_proj", "model.layers.43.mlp.experts.59.down_proj", "model.layers.43.mlp.experts.60.down_proj", "model.layers.43.mlp.experts.61.down_proj", "model.layers.43.mlp.experts.62.down_proj", "model.layers.43.mlp.experts.63.down_proj", "model.layers.43.mlp.experts.64.down_proj", "model.layers.43.mlp.experts.65.down_proj", "model.layers.43.mlp.experts.66.down_proj", "model.layers.43.mlp.experts.67.down_proj", "model.layers.43.mlp.experts.68.down_proj", "model.layers.43.mlp.experts.69.down_proj", "model.layers.43.mlp.experts.70.down_proj", "model.layers.43.mlp.experts.71.down_proj", "model.layers.43.mlp.experts.72.down_proj", "model.layers.43.mlp.experts.73.down_proj", "model.layers.43.mlp.experts.74.down_proj", "model.layers.43.mlp.experts.75.down_proj", "model.layers.43.mlp.experts.76.down_proj", "model.layers.43.mlp.experts.77.down_proj", "model.layers.43.mlp.experts.78.down_proj", "model.layers.43.mlp.experts.79.down_proj", "model.layers.43.mlp.experts.80.down_proj", "model.layers.43.mlp.experts.81.down_proj", "model.layers.43.mlp.experts.82.down_proj", "model.layers.43.mlp.experts.83.down_proj", "model.layers.43.mlp.experts.84.down_proj", "model.layers.43.mlp.experts.85.down_proj", "model.layers.43.mlp.experts.86.down_proj", "model.layers.43.mlp.experts.87.down_proj", "model.layers.43.mlp.experts.88.down_proj", "model.layers.43.mlp.experts.89.down_proj", "model.layers.43.mlp.experts.90.down_proj", "model.layers.43.mlp.experts.91.down_proj", "model.layers.43.mlp.experts.92.down_proj", "model.layers.43.mlp.experts.93.down_proj", "model.layers.43.mlp.experts.94.down_proj", "model.layers.43.mlp.experts.95.down_proj", "model.layers.43.mlp.experts.96.down_proj", "model.layers.43.mlp.experts.97.down_proj", "model.layers.43.mlp.experts.98.down_proj", "model.layers.43.mlp.experts.99.down_proj", "model.layers.43.mlp.experts.100.down_proj", "model.layers.43.mlp.experts.101.down_proj", "model.layers.43.mlp.experts.102.down_proj", "model.layers.43.mlp.experts.103.down_proj", "model.layers.43.mlp.experts.104.down_proj", "model.layers.43.mlp.experts.105.down_proj", "model.layers.43.mlp.experts.106.down_proj", "model.layers.43.mlp.experts.107.down_proj", "model.layers.43.mlp.experts.108.down_proj", "model.layers.43.mlp.experts.109.down_proj", "model.layers.43.mlp.experts.110.down_proj", "model.layers.43.mlp.experts.111.down_proj", "model.layers.43.mlp.experts.112.down_proj", "model.layers.43.mlp.experts.113.down_proj", "model.layers.43.mlp.experts.114.down_proj", "model.layers.43.mlp.experts.115.down_proj", "model.layers.43.mlp.experts.116.down_proj", "model.layers.43.mlp.experts.117.down_proj", "model.layers.43.mlp.experts.118.down_proj", "model.layers.43.mlp.experts.119.down_proj", "model.layers.43.mlp.experts.120.down_proj", "model.layers.43.mlp.experts.121.down_proj", "model.layers.43.mlp.experts.122.down_proj", "model.layers.43.mlp.experts.123.down_proj", "model.layers.43.mlp.experts.124.down_proj", "model.layers.43.mlp.experts.125.down_proj", "model.layers.43.mlp.experts.126.down_proj", "model.layers.43.mlp.experts.127.down_proj", "model.layers.43.mlp.experts.128.down_proj", "model.layers.43.mlp.experts.129.down_proj", "model.layers.43.mlp.experts.130.down_proj", "model.layers.43.mlp.experts.131.down_proj", "model.layers.43.mlp.experts.132.down_proj", "model.layers.43.mlp.experts.133.down_proj", "model.layers.43.mlp.experts.134.down_proj", "model.layers.43.mlp.experts.135.down_proj", "model.layers.43.mlp.experts.136.down_proj", "model.layers.43.mlp.experts.137.down_proj", "model.layers.43.mlp.experts.138.down_proj", "model.layers.43.mlp.experts.139.down_proj", "model.layers.43.mlp.experts.140.down_proj", "model.layers.43.mlp.experts.141.down_proj", "model.layers.43.mlp.experts.142.down_proj", "model.layers.43.mlp.experts.143.down_proj", "model.layers.43.mlp.experts.144.down_proj", "model.layers.43.mlp.experts.145.down_proj", "model.layers.43.mlp.experts.146.down_proj", "model.layers.43.mlp.experts.147.down_proj", "model.layers.43.mlp.experts.148.down_proj", "model.layers.43.mlp.experts.149.down_proj", "model.layers.43.mlp.experts.150.down_proj", "model.layers.43.mlp.experts.151.down_proj", "model.layers.43.mlp.experts.152.down_proj", "model.layers.43.mlp.experts.153.down_proj", "model.layers.43.mlp.experts.154.down_proj", "model.layers.43.mlp.experts.155.down_proj", "model.layers.43.mlp.experts.156.down_proj", "model.layers.43.mlp.experts.157.down_proj", "model.layers.43.mlp.experts.158.down_proj", "model.layers.43.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0011717319488525613, "dbits": 1258291200 } ] }, { "idx": 261, "layers": [ "model.layers.44.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0004863366484642029, "dbits": 62914560 } ] }, { "idx": 262, "layers": [ "model.layers.44.self_attn.k_proj", "model.layers.44.self_attn.v_proj" ], "candidates": [ { "dkld": -0.00034119412302974217, "dbits": 10485760 } ] }, { "idx": 263, "layers": [ "model.layers.44.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0024054579436779133, "dbits": 62914560 } ] }, { "idx": 264, "layers": [ "model.layers.44.mlp.shared_experts.gate_proj", "model.layers.44.mlp.shared_experts.up_proj", "model.layers.44.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0005435399711132272, "dbits": 23592960 } ] }, { "idx": 265, "layers": [ "model.layers.44.mlp.experts.0.gate_proj", "model.layers.44.mlp.experts.1.gate_proj", "model.layers.44.mlp.experts.2.gate_proj", "model.layers.44.mlp.experts.3.gate_proj", "model.layers.44.mlp.experts.4.gate_proj", "model.layers.44.mlp.experts.5.gate_proj", "model.layers.44.mlp.experts.6.gate_proj", "model.layers.44.mlp.experts.7.gate_proj", "model.layers.44.mlp.experts.8.gate_proj", "model.layers.44.mlp.experts.9.gate_proj", "model.layers.44.mlp.experts.10.gate_proj", "model.layers.44.mlp.experts.11.gate_proj", "model.layers.44.mlp.experts.12.gate_proj", "model.layers.44.mlp.experts.13.gate_proj", "model.layers.44.mlp.experts.14.gate_proj", "model.layers.44.mlp.experts.15.gate_proj", "model.layers.44.mlp.experts.16.gate_proj", "model.layers.44.mlp.experts.17.gate_proj", "model.layers.44.mlp.experts.18.gate_proj", "model.layers.44.mlp.experts.19.gate_proj", "model.layers.44.mlp.experts.20.gate_proj", "model.layers.44.mlp.experts.21.gate_proj", "model.layers.44.mlp.experts.22.gate_proj", "model.layers.44.mlp.experts.23.gate_proj", "model.layers.44.mlp.experts.24.gate_proj", "model.layers.44.mlp.experts.25.gate_proj", "model.layers.44.mlp.experts.26.gate_proj", "model.layers.44.mlp.experts.27.gate_proj", "model.layers.44.mlp.experts.28.gate_proj", "model.layers.44.mlp.experts.29.gate_proj", "model.layers.44.mlp.experts.30.gate_proj", "model.layers.44.mlp.experts.31.gate_proj", "model.layers.44.mlp.experts.32.gate_proj", "model.layers.44.mlp.experts.33.gate_proj", "model.layers.44.mlp.experts.34.gate_proj", "model.layers.44.mlp.experts.35.gate_proj", "model.layers.44.mlp.experts.36.gate_proj", "model.layers.44.mlp.experts.37.gate_proj", "model.layers.44.mlp.experts.38.gate_proj", "model.layers.44.mlp.experts.39.gate_proj", "model.layers.44.mlp.experts.40.gate_proj", "model.layers.44.mlp.experts.41.gate_proj", "model.layers.44.mlp.experts.42.gate_proj", "model.layers.44.mlp.experts.43.gate_proj", "model.layers.44.mlp.experts.44.gate_proj", "model.layers.44.mlp.experts.45.gate_proj", "model.layers.44.mlp.experts.46.gate_proj", "model.layers.44.mlp.experts.47.gate_proj", "model.layers.44.mlp.experts.48.gate_proj", "model.layers.44.mlp.experts.49.gate_proj", "model.layers.44.mlp.experts.50.gate_proj", "model.layers.44.mlp.experts.51.gate_proj", "model.layers.44.mlp.experts.52.gate_proj", "model.layers.44.mlp.experts.53.gate_proj", "model.layers.44.mlp.experts.54.gate_proj", "model.layers.44.mlp.experts.55.gate_proj", "model.layers.44.mlp.experts.56.gate_proj", "model.layers.44.mlp.experts.57.gate_proj", "model.layers.44.mlp.experts.58.gate_proj", "model.layers.44.mlp.experts.59.gate_proj", "model.layers.44.mlp.experts.60.gate_proj", "model.layers.44.mlp.experts.61.gate_proj", "model.layers.44.mlp.experts.62.gate_proj", "model.layers.44.mlp.experts.63.gate_proj", "model.layers.44.mlp.experts.64.gate_proj", "model.layers.44.mlp.experts.65.gate_proj", "model.layers.44.mlp.experts.66.gate_proj", "model.layers.44.mlp.experts.67.gate_proj", "model.layers.44.mlp.experts.68.gate_proj", "model.layers.44.mlp.experts.69.gate_proj", "model.layers.44.mlp.experts.70.gate_proj", "model.layers.44.mlp.experts.71.gate_proj", "model.layers.44.mlp.experts.72.gate_proj", "model.layers.44.mlp.experts.73.gate_proj", "model.layers.44.mlp.experts.74.gate_proj", "model.layers.44.mlp.experts.75.gate_proj", "model.layers.44.mlp.experts.76.gate_proj", "model.layers.44.mlp.experts.77.gate_proj", "model.layers.44.mlp.experts.78.gate_proj", "model.layers.44.mlp.experts.79.gate_proj", "model.layers.44.mlp.experts.80.gate_proj", "model.layers.44.mlp.experts.81.gate_proj", "model.layers.44.mlp.experts.82.gate_proj", "model.layers.44.mlp.experts.83.gate_proj", "model.layers.44.mlp.experts.84.gate_proj", "model.layers.44.mlp.experts.85.gate_proj", "model.layers.44.mlp.experts.86.gate_proj", "model.layers.44.mlp.experts.87.gate_proj", "model.layers.44.mlp.experts.88.gate_proj", "model.layers.44.mlp.experts.89.gate_proj", "model.layers.44.mlp.experts.90.gate_proj", "model.layers.44.mlp.experts.91.gate_proj", "model.layers.44.mlp.experts.92.gate_proj", "model.layers.44.mlp.experts.93.gate_proj", "model.layers.44.mlp.experts.94.gate_proj", "model.layers.44.mlp.experts.95.gate_proj", "model.layers.44.mlp.experts.96.gate_proj", "model.layers.44.mlp.experts.97.gate_proj", "model.layers.44.mlp.experts.98.gate_proj", "model.layers.44.mlp.experts.99.gate_proj", "model.layers.44.mlp.experts.100.gate_proj", "model.layers.44.mlp.experts.101.gate_proj", "model.layers.44.mlp.experts.102.gate_proj", "model.layers.44.mlp.experts.103.gate_proj", "model.layers.44.mlp.experts.104.gate_proj", "model.layers.44.mlp.experts.105.gate_proj", "model.layers.44.mlp.experts.106.gate_proj", "model.layers.44.mlp.experts.107.gate_proj", "model.layers.44.mlp.experts.108.gate_proj", "model.layers.44.mlp.experts.109.gate_proj", "model.layers.44.mlp.experts.110.gate_proj", "model.layers.44.mlp.experts.111.gate_proj", "model.layers.44.mlp.experts.112.gate_proj", "model.layers.44.mlp.experts.113.gate_proj", "model.layers.44.mlp.experts.114.gate_proj", "model.layers.44.mlp.experts.115.gate_proj", "model.layers.44.mlp.experts.116.gate_proj", "model.layers.44.mlp.experts.117.gate_proj", "model.layers.44.mlp.experts.118.gate_proj", "model.layers.44.mlp.experts.119.gate_proj", "model.layers.44.mlp.experts.120.gate_proj", "model.layers.44.mlp.experts.121.gate_proj", "model.layers.44.mlp.experts.122.gate_proj", "model.layers.44.mlp.experts.123.gate_proj", "model.layers.44.mlp.experts.124.gate_proj", "model.layers.44.mlp.experts.125.gate_proj", "model.layers.44.mlp.experts.126.gate_proj", "model.layers.44.mlp.experts.127.gate_proj", "model.layers.44.mlp.experts.128.gate_proj", "model.layers.44.mlp.experts.129.gate_proj", "model.layers.44.mlp.experts.130.gate_proj", "model.layers.44.mlp.experts.131.gate_proj", "model.layers.44.mlp.experts.132.gate_proj", "model.layers.44.mlp.experts.133.gate_proj", "model.layers.44.mlp.experts.134.gate_proj", "model.layers.44.mlp.experts.135.gate_proj", "model.layers.44.mlp.experts.136.gate_proj", "model.layers.44.mlp.experts.137.gate_proj", "model.layers.44.mlp.experts.138.gate_proj", "model.layers.44.mlp.experts.139.gate_proj", "model.layers.44.mlp.experts.140.gate_proj", "model.layers.44.mlp.experts.141.gate_proj", "model.layers.44.mlp.experts.142.gate_proj", "model.layers.44.mlp.experts.143.gate_proj", "model.layers.44.mlp.experts.144.gate_proj", "model.layers.44.mlp.experts.145.gate_proj", "model.layers.44.mlp.experts.146.gate_proj", "model.layers.44.mlp.experts.147.gate_proj", "model.layers.44.mlp.experts.148.gate_proj", "model.layers.44.mlp.experts.149.gate_proj", "model.layers.44.mlp.experts.150.gate_proj", "model.layers.44.mlp.experts.151.gate_proj", "model.layers.44.mlp.experts.152.gate_proj", "model.layers.44.mlp.experts.153.gate_proj", "model.layers.44.mlp.experts.154.gate_proj", "model.layers.44.mlp.experts.155.gate_proj", "model.layers.44.mlp.experts.156.gate_proj", "model.layers.44.mlp.experts.157.gate_proj", "model.layers.44.mlp.experts.158.gate_proj", "model.layers.44.mlp.experts.159.gate_proj", "model.layers.44.mlp.experts.0.up_proj", "model.layers.44.mlp.experts.1.up_proj", "model.layers.44.mlp.experts.2.up_proj", "model.layers.44.mlp.experts.3.up_proj", "model.layers.44.mlp.experts.4.up_proj", "model.layers.44.mlp.experts.5.up_proj", "model.layers.44.mlp.experts.6.up_proj", "model.layers.44.mlp.experts.7.up_proj", "model.layers.44.mlp.experts.8.up_proj", "model.layers.44.mlp.experts.9.up_proj", "model.layers.44.mlp.experts.10.up_proj", "model.layers.44.mlp.experts.11.up_proj", "model.layers.44.mlp.experts.12.up_proj", "model.layers.44.mlp.experts.13.up_proj", "model.layers.44.mlp.experts.14.up_proj", "model.layers.44.mlp.experts.15.up_proj", "model.layers.44.mlp.experts.16.up_proj", "model.layers.44.mlp.experts.17.up_proj", "model.layers.44.mlp.experts.18.up_proj", "model.layers.44.mlp.experts.19.up_proj", "model.layers.44.mlp.experts.20.up_proj", "model.layers.44.mlp.experts.21.up_proj", "model.layers.44.mlp.experts.22.up_proj", "model.layers.44.mlp.experts.23.up_proj", "model.layers.44.mlp.experts.24.up_proj", "model.layers.44.mlp.experts.25.up_proj", "model.layers.44.mlp.experts.26.up_proj", "model.layers.44.mlp.experts.27.up_proj", "model.layers.44.mlp.experts.28.up_proj", "model.layers.44.mlp.experts.29.up_proj", "model.layers.44.mlp.experts.30.up_proj", "model.layers.44.mlp.experts.31.up_proj", "model.layers.44.mlp.experts.32.up_proj", "model.layers.44.mlp.experts.33.up_proj", "model.layers.44.mlp.experts.34.up_proj", "model.layers.44.mlp.experts.35.up_proj", "model.layers.44.mlp.experts.36.up_proj", "model.layers.44.mlp.experts.37.up_proj", "model.layers.44.mlp.experts.38.up_proj", "model.layers.44.mlp.experts.39.up_proj", "model.layers.44.mlp.experts.40.up_proj", "model.layers.44.mlp.experts.41.up_proj", "model.layers.44.mlp.experts.42.up_proj", "model.layers.44.mlp.experts.43.up_proj", "model.layers.44.mlp.experts.44.up_proj", "model.layers.44.mlp.experts.45.up_proj", "model.layers.44.mlp.experts.46.up_proj", "model.layers.44.mlp.experts.47.up_proj", "model.layers.44.mlp.experts.48.up_proj", "model.layers.44.mlp.experts.49.up_proj", "model.layers.44.mlp.experts.50.up_proj", "model.layers.44.mlp.experts.51.up_proj", "model.layers.44.mlp.experts.52.up_proj", "model.layers.44.mlp.experts.53.up_proj", "model.layers.44.mlp.experts.54.up_proj", "model.layers.44.mlp.experts.55.up_proj", "model.layers.44.mlp.experts.56.up_proj", "model.layers.44.mlp.experts.57.up_proj", "model.layers.44.mlp.experts.58.up_proj", "model.layers.44.mlp.experts.59.up_proj", "model.layers.44.mlp.experts.60.up_proj", "model.layers.44.mlp.experts.61.up_proj", "model.layers.44.mlp.experts.62.up_proj", "model.layers.44.mlp.experts.63.up_proj", "model.layers.44.mlp.experts.64.up_proj", "model.layers.44.mlp.experts.65.up_proj", "model.layers.44.mlp.experts.66.up_proj", "model.layers.44.mlp.experts.67.up_proj", "model.layers.44.mlp.experts.68.up_proj", "model.layers.44.mlp.experts.69.up_proj", "model.layers.44.mlp.experts.70.up_proj", "model.layers.44.mlp.experts.71.up_proj", "model.layers.44.mlp.experts.72.up_proj", "model.layers.44.mlp.experts.73.up_proj", "model.layers.44.mlp.experts.74.up_proj", "model.layers.44.mlp.experts.75.up_proj", "model.layers.44.mlp.experts.76.up_proj", "model.layers.44.mlp.experts.77.up_proj", "model.layers.44.mlp.experts.78.up_proj", "model.layers.44.mlp.experts.79.up_proj", "model.layers.44.mlp.experts.80.up_proj", "model.layers.44.mlp.experts.81.up_proj", "model.layers.44.mlp.experts.82.up_proj", "model.layers.44.mlp.experts.83.up_proj", "model.layers.44.mlp.experts.84.up_proj", "model.layers.44.mlp.experts.85.up_proj", "model.layers.44.mlp.experts.86.up_proj", "model.layers.44.mlp.experts.87.up_proj", "model.layers.44.mlp.experts.88.up_proj", "model.layers.44.mlp.experts.89.up_proj", "model.layers.44.mlp.experts.90.up_proj", "model.layers.44.mlp.experts.91.up_proj", "model.layers.44.mlp.experts.92.up_proj", "model.layers.44.mlp.experts.93.up_proj", "model.layers.44.mlp.experts.94.up_proj", "model.layers.44.mlp.experts.95.up_proj", "model.layers.44.mlp.experts.96.up_proj", "model.layers.44.mlp.experts.97.up_proj", "model.layers.44.mlp.experts.98.up_proj", "model.layers.44.mlp.experts.99.up_proj", "model.layers.44.mlp.experts.100.up_proj", "model.layers.44.mlp.experts.101.up_proj", "model.layers.44.mlp.experts.102.up_proj", "model.layers.44.mlp.experts.103.up_proj", "model.layers.44.mlp.experts.104.up_proj", "model.layers.44.mlp.experts.105.up_proj", "model.layers.44.mlp.experts.106.up_proj", "model.layers.44.mlp.experts.107.up_proj", "model.layers.44.mlp.experts.108.up_proj", "model.layers.44.mlp.experts.109.up_proj", "model.layers.44.mlp.experts.110.up_proj", "model.layers.44.mlp.experts.111.up_proj", "model.layers.44.mlp.experts.112.up_proj", "model.layers.44.mlp.experts.113.up_proj", "model.layers.44.mlp.experts.114.up_proj", "model.layers.44.mlp.experts.115.up_proj", "model.layers.44.mlp.experts.116.up_proj", "model.layers.44.mlp.experts.117.up_proj", "model.layers.44.mlp.experts.118.up_proj", "model.layers.44.mlp.experts.119.up_proj", "model.layers.44.mlp.experts.120.up_proj", "model.layers.44.mlp.experts.121.up_proj", "model.layers.44.mlp.experts.122.up_proj", "model.layers.44.mlp.experts.123.up_proj", "model.layers.44.mlp.experts.124.up_proj", "model.layers.44.mlp.experts.125.up_proj", "model.layers.44.mlp.experts.126.up_proj", "model.layers.44.mlp.experts.127.up_proj", "model.layers.44.mlp.experts.128.up_proj", "model.layers.44.mlp.experts.129.up_proj", "model.layers.44.mlp.experts.130.up_proj", "model.layers.44.mlp.experts.131.up_proj", "model.layers.44.mlp.experts.132.up_proj", "model.layers.44.mlp.experts.133.up_proj", "model.layers.44.mlp.experts.134.up_proj", "model.layers.44.mlp.experts.135.up_proj", "model.layers.44.mlp.experts.136.up_proj", "model.layers.44.mlp.experts.137.up_proj", "model.layers.44.mlp.experts.138.up_proj", "model.layers.44.mlp.experts.139.up_proj", "model.layers.44.mlp.experts.140.up_proj", "model.layers.44.mlp.experts.141.up_proj", "model.layers.44.mlp.experts.142.up_proj", "model.layers.44.mlp.experts.143.up_proj", "model.layers.44.mlp.experts.144.up_proj", "model.layers.44.mlp.experts.145.up_proj", "model.layers.44.mlp.experts.146.up_proj", "model.layers.44.mlp.experts.147.up_proj", "model.layers.44.mlp.experts.148.up_proj", "model.layers.44.mlp.experts.149.up_proj", "model.layers.44.mlp.experts.150.up_proj", "model.layers.44.mlp.experts.151.up_proj", "model.layers.44.mlp.experts.152.up_proj", "model.layers.44.mlp.experts.153.up_proj", "model.layers.44.mlp.experts.154.up_proj", "model.layers.44.mlp.experts.155.up_proj", "model.layers.44.mlp.experts.156.up_proj", "model.layers.44.mlp.experts.157.up_proj", "model.layers.44.mlp.experts.158.up_proj", "model.layers.44.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0010456815361976624, "dbits": 2516582400 } ] }, { "idx": 266, "layers": [ "model.layers.44.mlp.experts.0.down_proj", "model.layers.44.mlp.experts.1.down_proj", "model.layers.44.mlp.experts.2.down_proj", "model.layers.44.mlp.experts.3.down_proj", "model.layers.44.mlp.experts.4.down_proj", "model.layers.44.mlp.experts.5.down_proj", "model.layers.44.mlp.experts.6.down_proj", "model.layers.44.mlp.experts.7.down_proj", "model.layers.44.mlp.experts.8.down_proj", "model.layers.44.mlp.experts.9.down_proj", "model.layers.44.mlp.experts.10.down_proj", "model.layers.44.mlp.experts.11.down_proj", "model.layers.44.mlp.experts.12.down_proj", "model.layers.44.mlp.experts.13.down_proj", "model.layers.44.mlp.experts.14.down_proj", "model.layers.44.mlp.experts.15.down_proj", "model.layers.44.mlp.experts.16.down_proj", "model.layers.44.mlp.experts.17.down_proj", "model.layers.44.mlp.experts.18.down_proj", "model.layers.44.mlp.experts.19.down_proj", "model.layers.44.mlp.experts.20.down_proj", "model.layers.44.mlp.experts.21.down_proj", "model.layers.44.mlp.experts.22.down_proj", "model.layers.44.mlp.experts.23.down_proj", "model.layers.44.mlp.experts.24.down_proj", "model.layers.44.mlp.experts.25.down_proj", "model.layers.44.mlp.experts.26.down_proj", "model.layers.44.mlp.experts.27.down_proj", "model.layers.44.mlp.experts.28.down_proj", "model.layers.44.mlp.experts.29.down_proj", "model.layers.44.mlp.experts.30.down_proj", "model.layers.44.mlp.experts.31.down_proj", "model.layers.44.mlp.experts.32.down_proj", "model.layers.44.mlp.experts.33.down_proj", "model.layers.44.mlp.experts.34.down_proj", "model.layers.44.mlp.experts.35.down_proj", "model.layers.44.mlp.experts.36.down_proj", "model.layers.44.mlp.experts.37.down_proj", "model.layers.44.mlp.experts.38.down_proj", "model.layers.44.mlp.experts.39.down_proj", "model.layers.44.mlp.experts.40.down_proj", "model.layers.44.mlp.experts.41.down_proj", "model.layers.44.mlp.experts.42.down_proj", "model.layers.44.mlp.experts.43.down_proj", "model.layers.44.mlp.experts.44.down_proj", "model.layers.44.mlp.experts.45.down_proj", "model.layers.44.mlp.experts.46.down_proj", "model.layers.44.mlp.experts.47.down_proj", "model.layers.44.mlp.experts.48.down_proj", "model.layers.44.mlp.experts.49.down_proj", "model.layers.44.mlp.experts.50.down_proj", "model.layers.44.mlp.experts.51.down_proj", "model.layers.44.mlp.experts.52.down_proj", "model.layers.44.mlp.experts.53.down_proj", "model.layers.44.mlp.experts.54.down_proj", "model.layers.44.mlp.experts.55.down_proj", "model.layers.44.mlp.experts.56.down_proj", "model.layers.44.mlp.experts.57.down_proj", "model.layers.44.mlp.experts.58.down_proj", "model.layers.44.mlp.experts.59.down_proj", "model.layers.44.mlp.experts.60.down_proj", "model.layers.44.mlp.experts.61.down_proj", "model.layers.44.mlp.experts.62.down_proj", "model.layers.44.mlp.experts.63.down_proj", "model.layers.44.mlp.experts.64.down_proj", "model.layers.44.mlp.experts.65.down_proj", "model.layers.44.mlp.experts.66.down_proj", "model.layers.44.mlp.experts.67.down_proj", "model.layers.44.mlp.experts.68.down_proj", "model.layers.44.mlp.experts.69.down_proj", "model.layers.44.mlp.experts.70.down_proj", "model.layers.44.mlp.experts.71.down_proj", "model.layers.44.mlp.experts.72.down_proj", "model.layers.44.mlp.experts.73.down_proj", "model.layers.44.mlp.experts.74.down_proj", "model.layers.44.mlp.experts.75.down_proj", "model.layers.44.mlp.experts.76.down_proj", "model.layers.44.mlp.experts.77.down_proj", "model.layers.44.mlp.experts.78.down_proj", "model.layers.44.mlp.experts.79.down_proj", "model.layers.44.mlp.experts.80.down_proj", "model.layers.44.mlp.experts.81.down_proj", "model.layers.44.mlp.experts.82.down_proj", "model.layers.44.mlp.experts.83.down_proj", "model.layers.44.mlp.experts.84.down_proj", "model.layers.44.mlp.experts.85.down_proj", "model.layers.44.mlp.experts.86.down_proj", "model.layers.44.mlp.experts.87.down_proj", "model.layers.44.mlp.experts.88.down_proj", "model.layers.44.mlp.experts.89.down_proj", "model.layers.44.mlp.experts.90.down_proj", "model.layers.44.mlp.experts.91.down_proj", "model.layers.44.mlp.experts.92.down_proj", "model.layers.44.mlp.experts.93.down_proj", "model.layers.44.mlp.experts.94.down_proj", "model.layers.44.mlp.experts.95.down_proj", "model.layers.44.mlp.experts.96.down_proj", "model.layers.44.mlp.experts.97.down_proj", "model.layers.44.mlp.experts.98.down_proj", "model.layers.44.mlp.experts.99.down_proj", "model.layers.44.mlp.experts.100.down_proj", "model.layers.44.mlp.experts.101.down_proj", "model.layers.44.mlp.experts.102.down_proj", "model.layers.44.mlp.experts.103.down_proj", "model.layers.44.mlp.experts.104.down_proj", "model.layers.44.mlp.experts.105.down_proj", "model.layers.44.mlp.experts.106.down_proj", "model.layers.44.mlp.experts.107.down_proj", "model.layers.44.mlp.experts.108.down_proj", "model.layers.44.mlp.experts.109.down_proj", "model.layers.44.mlp.experts.110.down_proj", "model.layers.44.mlp.experts.111.down_proj", "model.layers.44.mlp.experts.112.down_proj", "model.layers.44.mlp.experts.113.down_proj", "model.layers.44.mlp.experts.114.down_proj", "model.layers.44.mlp.experts.115.down_proj", "model.layers.44.mlp.experts.116.down_proj", "model.layers.44.mlp.experts.117.down_proj", "model.layers.44.mlp.experts.118.down_proj", "model.layers.44.mlp.experts.119.down_proj", "model.layers.44.mlp.experts.120.down_proj", "model.layers.44.mlp.experts.121.down_proj", "model.layers.44.mlp.experts.122.down_proj", "model.layers.44.mlp.experts.123.down_proj", "model.layers.44.mlp.experts.124.down_proj", "model.layers.44.mlp.experts.125.down_proj", "model.layers.44.mlp.experts.126.down_proj", "model.layers.44.mlp.experts.127.down_proj", "model.layers.44.mlp.experts.128.down_proj", "model.layers.44.mlp.experts.129.down_proj", "model.layers.44.mlp.experts.130.down_proj", "model.layers.44.mlp.experts.131.down_proj", "model.layers.44.mlp.experts.132.down_proj", "model.layers.44.mlp.experts.133.down_proj", "model.layers.44.mlp.experts.134.down_proj", "model.layers.44.mlp.experts.135.down_proj", "model.layers.44.mlp.experts.136.down_proj", "model.layers.44.mlp.experts.137.down_proj", "model.layers.44.mlp.experts.138.down_proj", "model.layers.44.mlp.experts.139.down_proj", "model.layers.44.mlp.experts.140.down_proj", "model.layers.44.mlp.experts.141.down_proj", "model.layers.44.mlp.experts.142.down_proj", "model.layers.44.mlp.experts.143.down_proj", "model.layers.44.mlp.experts.144.down_proj", "model.layers.44.mlp.experts.145.down_proj", "model.layers.44.mlp.experts.146.down_proj", "model.layers.44.mlp.experts.147.down_proj", "model.layers.44.mlp.experts.148.down_proj", "model.layers.44.mlp.experts.149.down_proj", "model.layers.44.mlp.experts.150.down_proj", "model.layers.44.mlp.experts.151.down_proj", "model.layers.44.mlp.experts.152.down_proj", "model.layers.44.mlp.experts.153.down_proj", "model.layers.44.mlp.experts.154.down_proj", "model.layers.44.mlp.experts.155.down_proj", "model.layers.44.mlp.experts.156.down_proj", "model.layers.44.mlp.experts.157.down_proj", "model.layers.44.mlp.experts.158.down_proj", "model.layers.44.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006727680563926586, "dbits": 1258291200 } ] }, { "idx": 267, "layers": [ "model.layers.45.self_attn.q_proj" ], "candidates": [ { "dkld": 0.00011454075574873768, "dbits": 62914560 } ] }, { "idx": 268, "layers": [ "model.layers.45.self_attn.k_proj", "model.layers.45.self_attn.v_proj" ], "candidates": [ { "dkld": -0.002720272541046176, "dbits": 10485760 } ] }, { "idx": 269, "layers": [ "model.layers.45.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00013368725776669033, "dbits": 62914560 } ] }, { "idx": 270, "layers": [ "model.layers.45.mlp.shared_experts.gate_proj", "model.layers.45.mlp.shared_experts.up_proj", "model.layers.45.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0011164523661136294, "dbits": 23592960 } ] }, { "idx": 271, "layers": [ "model.layers.45.mlp.experts.0.gate_proj", "model.layers.45.mlp.experts.1.gate_proj", "model.layers.45.mlp.experts.2.gate_proj", "model.layers.45.mlp.experts.3.gate_proj", "model.layers.45.mlp.experts.4.gate_proj", "model.layers.45.mlp.experts.5.gate_proj", "model.layers.45.mlp.experts.6.gate_proj", "model.layers.45.mlp.experts.7.gate_proj", "model.layers.45.mlp.experts.8.gate_proj", "model.layers.45.mlp.experts.9.gate_proj", "model.layers.45.mlp.experts.10.gate_proj", "model.layers.45.mlp.experts.11.gate_proj", "model.layers.45.mlp.experts.12.gate_proj", "model.layers.45.mlp.experts.13.gate_proj", "model.layers.45.mlp.experts.14.gate_proj", "model.layers.45.mlp.experts.15.gate_proj", "model.layers.45.mlp.experts.16.gate_proj", "model.layers.45.mlp.experts.17.gate_proj", "model.layers.45.mlp.experts.18.gate_proj", "model.layers.45.mlp.experts.19.gate_proj", "model.layers.45.mlp.experts.20.gate_proj", "model.layers.45.mlp.experts.21.gate_proj", "model.layers.45.mlp.experts.22.gate_proj", "model.layers.45.mlp.experts.23.gate_proj", "model.layers.45.mlp.experts.24.gate_proj", "model.layers.45.mlp.experts.25.gate_proj", "model.layers.45.mlp.experts.26.gate_proj", "model.layers.45.mlp.experts.27.gate_proj", "model.layers.45.mlp.experts.28.gate_proj", "model.layers.45.mlp.experts.29.gate_proj", "model.layers.45.mlp.experts.30.gate_proj", "model.layers.45.mlp.experts.31.gate_proj", "model.layers.45.mlp.experts.32.gate_proj", "model.layers.45.mlp.experts.33.gate_proj", "model.layers.45.mlp.experts.34.gate_proj", "model.layers.45.mlp.experts.35.gate_proj", "model.layers.45.mlp.experts.36.gate_proj", "model.layers.45.mlp.experts.37.gate_proj", "model.layers.45.mlp.experts.38.gate_proj", "model.layers.45.mlp.experts.39.gate_proj", "model.layers.45.mlp.experts.40.gate_proj", "model.layers.45.mlp.experts.41.gate_proj", "model.layers.45.mlp.experts.42.gate_proj", "model.layers.45.mlp.experts.43.gate_proj", "model.layers.45.mlp.experts.44.gate_proj", "model.layers.45.mlp.experts.45.gate_proj", "model.layers.45.mlp.experts.46.gate_proj", "model.layers.45.mlp.experts.47.gate_proj", "model.layers.45.mlp.experts.48.gate_proj", "model.layers.45.mlp.experts.49.gate_proj", "model.layers.45.mlp.experts.50.gate_proj", "model.layers.45.mlp.experts.51.gate_proj", "model.layers.45.mlp.experts.52.gate_proj", "model.layers.45.mlp.experts.53.gate_proj", "model.layers.45.mlp.experts.54.gate_proj", "model.layers.45.mlp.experts.55.gate_proj", "model.layers.45.mlp.experts.56.gate_proj", "model.layers.45.mlp.experts.57.gate_proj", "model.layers.45.mlp.experts.58.gate_proj", "model.layers.45.mlp.experts.59.gate_proj", "model.layers.45.mlp.experts.60.gate_proj", "model.layers.45.mlp.experts.61.gate_proj", "model.layers.45.mlp.experts.62.gate_proj", "model.layers.45.mlp.experts.63.gate_proj", "model.layers.45.mlp.experts.64.gate_proj", "model.layers.45.mlp.experts.65.gate_proj", "model.layers.45.mlp.experts.66.gate_proj", "model.layers.45.mlp.experts.67.gate_proj", "model.layers.45.mlp.experts.68.gate_proj", "model.layers.45.mlp.experts.69.gate_proj", "model.layers.45.mlp.experts.70.gate_proj", "model.layers.45.mlp.experts.71.gate_proj", "model.layers.45.mlp.experts.72.gate_proj", "model.layers.45.mlp.experts.73.gate_proj", "model.layers.45.mlp.experts.74.gate_proj", "model.layers.45.mlp.experts.75.gate_proj", "model.layers.45.mlp.experts.76.gate_proj", "model.layers.45.mlp.experts.77.gate_proj", "model.layers.45.mlp.experts.78.gate_proj", "model.layers.45.mlp.experts.79.gate_proj", "model.layers.45.mlp.experts.80.gate_proj", "model.layers.45.mlp.experts.81.gate_proj", "model.layers.45.mlp.experts.82.gate_proj", "model.layers.45.mlp.experts.83.gate_proj", "model.layers.45.mlp.experts.84.gate_proj", "model.layers.45.mlp.experts.85.gate_proj", "model.layers.45.mlp.experts.86.gate_proj", "model.layers.45.mlp.experts.87.gate_proj", "model.layers.45.mlp.experts.88.gate_proj", "model.layers.45.mlp.experts.89.gate_proj", "model.layers.45.mlp.experts.90.gate_proj", "model.layers.45.mlp.experts.91.gate_proj", "model.layers.45.mlp.experts.92.gate_proj", "model.layers.45.mlp.experts.93.gate_proj", "model.layers.45.mlp.experts.94.gate_proj", "model.layers.45.mlp.experts.95.gate_proj", "model.layers.45.mlp.experts.96.gate_proj", "model.layers.45.mlp.experts.97.gate_proj", "model.layers.45.mlp.experts.98.gate_proj", "model.layers.45.mlp.experts.99.gate_proj", "model.layers.45.mlp.experts.100.gate_proj", "model.layers.45.mlp.experts.101.gate_proj", "model.layers.45.mlp.experts.102.gate_proj", "model.layers.45.mlp.experts.103.gate_proj", "model.layers.45.mlp.experts.104.gate_proj", "model.layers.45.mlp.experts.105.gate_proj", "model.layers.45.mlp.experts.106.gate_proj", "model.layers.45.mlp.experts.107.gate_proj", "model.layers.45.mlp.experts.108.gate_proj", "model.layers.45.mlp.experts.109.gate_proj", "model.layers.45.mlp.experts.110.gate_proj", "model.layers.45.mlp.experts.111.gate_proj", "model.layers.45.mlp.experts.112.gate_proj", "model.layers.45.mlp.experts.113.gate_proj", "model.layers.45.mlp.experts.114.gate_proj", "model.layers.45.mlp.experts.115.gate_proj", "model.layers.45.mlp.experts.116.gate_proj", "model.layers.45.mlp.experts.117.gate_proj", "model.layers.45.mlp.experts.118.gate_proj", "model.layers.45.mlp.experts.119.gate_proj", "model.layers.45.mlp.experts.120.gate_proj", "model.layers.45.mlp.experts.121.gate_proj", "model.layers.45.mlp.experts.122.gate_proj", "model.layers.45.mlp.experts.123.gate_proj", "model.layers.45.mlp.experts.124.gate_proj", "model.layers.45.mlp.experts.125.gate_proj", "model.layers.45.mlp.experts.126.gate_proj", "model.layers.45.mlp.experts.127.gate_proj", "model.layers.45.mlp.experts.128.gate_proj", "model.layers.45.mlp.experts.129.gate_proj", "model.layers.45.mlp.experts.130.gate_proj", "model.layers.45.mlp.experts.131.gate_proj", "model.layers.45.mlp.experts.132.gate_proj", "model.layers.45.mlp.experts.133.gate_proj", "model.layers.45.mlp.experts.134.gate_proj", "model.layers.45.mlp.experts.135.gate_proj", "model.layers.45.mlp.experts.136.gate_proj", "model.layers.45.mlp.experts.137.gate_proj", "model.layers.45.mlp.experts.138.gate_proj", "model.layers.45.mlp.experts.139.gate_proj", "model.layers.45.mlp.experts.140.gate_proj", "model.layers.45.mlp.experts.141.gate_proj", "model.layers.45.mlp.experts.142.gate_proj", "model.layers.45.mlp.experts.143.gate_proj", "model.layers.45.mlp.experts.144.gate_proj", "model.layers.45.mlp.experts.145.gate_proj", "model.layers.45.mlp.experts.146.gate_proj", "model.layers.45.mlp.experts.147.gate_proj", "model.layers.45.mlp.experts.148.gate_proj", "model.layers.45.mlp.experts.149.gate_proj", "model.layers.45.mlp.experts.150.gate_proj", "model.layers.45.mlp.experts.151.gate_proj", "model.layers.45.mlp.experts.152.gate_proj", "model.layers.45.mlp.experts.153.gate_proj", "model.layers.45.mlp.experts.154.gate_proj", "model.layers.45.mlp.experts.155.gate_proj", "model.layers.45.mlp.experts.156.gate_proj", "model.layers.45.mlp.experts.157.gate_proj", "model.layers.45.mlp.experts.158.gate_proj", "model.layers.45.mlp.experts.159.gate_proj", "model.layers.45.mlp.experts.0.up_proj", "model.layers.45.mlp.experts.1.up_proj", "model.layers.45.mlp.experts.2.up_proj", "model.layers.45.mlp.experts.3.up_proj", "model.layers.45.mlp.experts.4.up_proj", "model.layers.45.mlp.experts.5.up_proj", "model.layers.45.mlp.experts.6.up_proj", "model.layers.45.mlp.experts.7.up_proj", "model.layers.45.mlp.experts.8.up_proj", "model.layers.45.mlp.experts.9.up_proj", "model.layers.45.mlp.experts.10.up_proj", "model.layers.45.mlp.experts.11.up_proj", "model.layers.45.mlp.experts.12.up_proj", "model.layers.45.mlp.experts.13.up_proj", "model.layers.45.mlp.experts.14.up_proj", "model.layers.45.mlp.experts.15.up_proj", "model.layers.45.mlp.experts.16.up_proj", "model.layers.45.mlp.experts.17.up_proj", "model.layers.45.mlp.experts.18.up_proj", "model.layers.45.mlp.experts.19.up_proj", "model.layers.45.mlp.experts.20.up_proj", "model.layers.45.mlp.experts.21.up_proj", "model.layers.45.mlp.experts.22.up_proj", "model.layers.45.mlp.experts.23.up_proj", "model.layers.45.mlp.experts.24.up_proj", "model.layers.45.mlp.experts.25.up_proj", "model.layers.45.mlp.experts.26.up_proj", "model.layers.45.mlp.experts.27.up_proj", "model.layers.45.mlp.experts.28.up_proj", "model.layers.45.mlp.experts.29.up_proj", "model.layers.45.mlp.experts.30.up_proj", "model.layers.45.mlp.experts.31.up_proj", "model.layers.45.mlp.experts.32.up_proj", "model.layers.45.mlp.experts.33.up_proj", "model.layers.45.mlp.experts.34.up_proj", "model.layers.45.mlp.experts.35.up_proj", "model.layers.45.mlp.experts.36.up_proj", "model.layers.45.mlp.experts.37.up_proj", "model.layers.45.mlp.experts.38.up_proj", "model.layers.45.mlp.experts.39.up_proj", "model.layers.45.mlp.experts.40.up_proj", "model.layers.45.mlp.experts.41.up_proj", "model.layers.45.mlp.experts.42.up_proj", "model.layers.45.mlp.experts.43.up_proj", "model.layers.45.mlp.experts.44.up_proj", "model.layers.45.mlp.experts.45.up_proj", "model.layers.45.mlp.experts.46.up_proj", "model.layers.45.mlp.experts.47.up_proj", "model.layers.45.mlp.experts.48.up_proj", "model.layers.45.mlp.experts.49.up_proj", "model.layers.45.mlp.experts.50.up_proj", "model.layers.45.mlp.experts.51.up_proj", "model.layers.45.mlp.experts.52.up_proj", "model.layers.45.mlp.experts.53.up_proj", "model.layers.45.mlp.experts.54.up_proj", "model.layers.45.mlp.experts.55.up_proj", "model.layers.45.mlp.experts.56.up_proj", "model.layers.45.mlp.experts.57.up_proj", "model.layers.45.mlp.experts.58.up_proj", "model.layers.45.mlp.experts.59.up_proj", "model.layers.45.mlp.experts.60.up_proj", "model.layers.45.mlp.experts.61.up_proj", "model.layers.45.mlp.experts.62.up_proj", "model.layers.45.mlp.experts.63.up_proj", "model.layers.45.mlp.experts.64.up_proj", "model.layers.45.mlp.experts.65.up_proj", "model.layers.45.mlp.experts.66.up_proj", "model.layers.45.mlp.experts.67.up_proj", "model.layers.45.mlp.experts.68.up_proj", "model.layers.45.mlp.experts.69.up_proj", "model.layers.45.mlp.experts.70.up_proj", "model.layers.45.mlp.experts.71.up_proj", "model.layers.45.mlp.experts.72.up_proj", "model.layers.45.mlp.experts.73.up_proj", "model.layers.45.mlp.experts.74.up_proj", "model.layers.45.mlp.experts.75.up_proj", "model.layers.45.mlp.experts.76.up_proj", "model.layers.45.mlp.experts.77.up_proj", "model.layers.45.mlp.experts.78.up_proj", "model.layers.45.mlp.experts.79.up_proj", "model.layers.45.mlp.experts.80.up_proj", "model.layers.45.mlp.experts.81.up_proj", "model.layers.45.mlp.experts.82.up_proj", "model.layers.45.mlp.experts.83.up_proj", "model.layers.45.mlp.experts.84.up_proj", "model.layers.45.mlp.experts.85.up_proj", "model.layers.45.mlp.experts.86.up_proj", "model.layers.45.mlp.experts.87.up_proj", "model.layers.45.mlp.experts.88.up_proj", "model.layers.45.mlp.experts.89.up_proj", "model.layers.45.mlp.experts.90.up_proj", "model.layers.45.mlp.experts.91.up_proj", "model.layers.45.mlp.experts.92.up_proj", "model.layers.45.mlp.experts.93.up_proj", "model.layers.45.mlp.experts.94.up_proj", "model.layers.45.mlp.experts.95.up_proj", "model.layers.45.mlp.experts.96.up_proj", "model.layers.45.mlp.experts.97.up_proj", "model.layers.45.mlp.experts.98.up_proj", "model.layers.45.mlp.experts.99.up_proj", "model.layers.45.mlp.experts.100.up_proj", "model.layers.45.mlp.experts.101.up_proj", "model.layers.45.mlp.experts.102.up_proj", "model.layers.45.mlp.experts.103.up_proj", "model.layers.45.mlp.experts.104.up_proj", "model.layers.45.mlp.experts.105.up_proj", "model.layers.45.mlp.experts.106.up_proj", "model.layers.45.mlp.experts.107.up_proj", "model.layers.45.mlp.experts.108.up_proj", "model.layers.45.mlp.experts.109.up_proj", "model.layers.45.mlp.experts.110.up_proj", "model.layers.45.mlp.experts.111.up_proj", "model.layers.45.mlp.experts.112.up_proj", "model.layers.45.mlp.experts.113.up_proj", "model.layers.45.mlp.experts.114.up_proj", "model.layers.45.mlp.experts.115.up_proj", "model.layers.45.mlp.experts.116.up_proj", "model.layers.45.mlp.experts.117.up_proj", "model.layers.45.mlp.experts.118.up_proj", "model.layers.45.mlp.experts.119.up_proj", "model.layers.45.mlp.experts.120.up_proj", "model.layers.45.mlp.experts.121.up_proj", "model.layers.45.mlp.experts.122.up_proj", "model.layers.45.mlp.experts.123.up_proj", "model.layers.45.mlp.experts.124.up_proj", "model.layers.45.mlp.experts.125.up_proj", "model.layers.45.mlp.experts.126.up_proj", "model.layers.45.mlp.experts.127.up_proj", "model.layers.45.mlp.experts.128.up_proj", "model.layers.45.mlp.experts.129.up_proj", "model.layers.45.mlp.experts.130.up_proj", "model.layers.45.mlp.experts.131.up_proj", "model.layers.45.mlp.experts.132.up_proj", "model.layers.45.mlp.experts.133.up_proj", "model.layers.45.mlp.experts.134.up_proj", "model.layers.45.mlp.experts.135.up_proj", "model.layers.45.mlp.experts.136.up_proj", "model.layers.45.mlp.experts.137.up_proj", "model.layers.45.mlp.experts.138.up_proj", "model.layers.45.mlp.experts.139.up_proj", "model.layers.45.mlp.experts.140.up_proj", "model.layers.45.mlp.experts.141.up_proj", "model.layers.45.mlp.experts.142.up_proj", "model.layers.45.mlp.experts.143.up_proj", "model.layers.45.mlp.experts.144.up_proj", "model.layers.45.mlp.experts.145.up_proj", "model.layers.45.mlp.experts.146.up_proj", "model.layers.45.mlp.experts.147.up_proj", "model.layers.45.mlp.experts.148.up_proj", "model.layers.45.mlp.experts.149.up_proj", "model.layers.45.mlp.experts.150.up_proj", "model.layers.45.mlp.experts.151.up_proj", "model.layers.45.mlp.experts.152.up_proj", "model.layers.45.mlp.experts.153.up_proj", "model.layers.45.mlp.experts.154.up_proj", "model.layers.45.mlp.experts.155.up_proj", "model.layers.45.mlp.experts.156.up_proj", "model.layers.45.mlp.experts.157.up_proj", "model.layers.45.mlp.experts.158.up_proj", "model.layers.45.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0014736309647560009, "dbits": 2516582400 } ] }, { "idx": 272, "layers": [ "model.layers.45.mlp.experts.0.down_proj", "model.layers.45.mlp.experts.1.down_proj", "model.layers.45.mlp.experts.2.down_proj", "model.layers.45.mlp.experts.3.down_proj", "model.layers.45.mlp.experts.4.down_proj", "model.layers.45.mlp.experts.5.down_proj", "model.layers.45.mlp.experts.6.down_proj", "model.layers.45.mlp.experts.7.down_proj", "model.layers.45.mlp.experts.8.down_proj", "model.layers.45.mlp.experts.9.down_proj", "model.layers.45.mlp.experts.10.down_proj", "model.layers.45.mlp.experts.11.down_proj", "model.layers.45.mlp.experts.12.down_proj", "model.layers.45.mlp.experts.13.down_proj", "model.layers.45.mlp.experts.14.down_proj", "model.layers.45.mlp.experts.15.down_proj", "model.layers.45.mlp.experts.16.down_proj", "model.layers.45.mlp.experts.17.down_proj", "model.layers.45.mlp.experts.18.down_proj", "model.layers.45.mlp.experts.19.down_proj", "model.layers.45.mlp.experts.20.down_proj", "model.layers.45.mlp.experts.21.down_proj", "model.layers.45.mlp.experts.22.down_proj", "model.layers.45.mlp.experts.23.down_proj", "model.layers.45.mlp.experts.24.down_proj", "model.layers.45.mlp.experts.25.down_proj", "model.layers.45.mlp.experts.26.down_proj", "model.layers.45.mlp.experts.27.down_proj", "model.layers.45.mlp.experts.28.down_proj", "model.layers.45.mlp.experts.29.down_proj", "model.layers.45.mlp.experts.30.down_proj", "model.layers.45.mlp.experts.31.down_proj", "model.layers.45.mlp.experts.32.down_proj", "model.layers.45.mlp.experts.33.down_proj", "model.layers.45.mlp.experts.34.down_proj", "model.layers.45.mlp.experts.35.down_proj", "model.layers.45.mlp.experts.36.down_proj", "model.layers.45.mlp.experts.37.down_proj", "model.layers.45.mlp.experts.38.down_proj", "model.layers.45.mlp.experts.39.down_proj", "model.layers.45.mlp.experts.40.down_proj", "model.layers.45.mlp.experts.41.down_proj", "model.layers.45.mlp.experts.42.down_proj", "model.layers.45.mlp.experts.43.down_proj", "model.layers.45.mlp.experts.44.down_proj", "model.layers.45.mlp.experts.45.down_proj", "model.layers.45.mlp.experts.46.down_proj", "model.layers.45.mlp.experts.47.down_proj", "model.layers.45.mlp.experts.48.down_proj", "model.layers.45.mlp.experts.49.down_proj", "model.layers.45.mlp.experts.50.down_proj", "model.layers.45.mlp.experts.51.down_proj", "model.layers.45.mlp.experts.52.down_proj", "model.layers.45.mlp.experts.53.down_proj", "model.layers.45.mlp.experts.54.down_proj", "model.layers.45.mlp.experts.55.down_proj", "model.layers.45.mlp.experts.56.down_proj", "model.layers.45.mlp.experts.57.down_proj", "model.layers.45.mlp.experts.58.down_proj", "model.layers.45.mlp.experts.59.down_proj", "model.layers.45.mlp.experts.60.down_proj", "model.layers.45.mlp.experts.61.down_proj", "model.layers.45.mlp.experts.62.down_proj", "model.layers.45.mlp.experts.63.down_proj", "model.layers.45.mlp.experts.64.down_proj", "model.layers.45.mlp.experts.65.down_proj", "model.layers.45.mlp.experts.66.down_proj", "model.layers.45.mlp.experts.67.down_proj", "model.layers.45.mlp.experts.68.down_proj", "model.layers.45.mlp.experts.69.down_proj", "model.layers.45.mlp.experts.70.down_proj", "model.layers.45.mlp.experts.71.down_proj", "model.layers.45.mlp.experts.72.down_proj", "model.layers.45.mlp.experts.73.down_proj", "model.layers.45.mlp.experts.74.down_proj", "model.layers.45.mlp.experts.75.down_proj", "model.layers.45.mlp.experts.76.down_proj", "model.layers.45.mlp.experts.77.down_proj", "model.layers.45.mlp.experts.78.down_proj", "model.layers.45.mlp.experts.79.down_proj", "model.layers.45.mlp.experts.80.down_proj", "model.layers.45.mlp.experts.81.down_proj", "model.layers.45.mlp.experts.82.down_proj", "model.layers.45.mlp.experts.83.down_proj", "model.layers.45.mlp.experts.84.down_proj", "model.layers.45.mlp.experts.85.down_proj", "model.layers.45.mlp.experts.86.down_proj", "model.layers.45.mlp.experts.87.down_proj", "model.layers.45.mlp.experts.88.down_proj", "model.layers.45.mlp.experts.89.down_proj", "model.layers.45.mlp.experts.90.down_proj", "model.layers.45.mlp.experts.91.down_proj", "model.layers.45.mlp.experts.92.down_proj", "model.layers.45.mlp.experts.93.down_proj", "model.layers.45.mlp.experts.94.down_proj", "model.layers.45.mlp.experts.95.down_proj", "model.layers.45.mlp.experts.96.down_proj", "model.layers.45.mlp.experts.97.down_proj", "model.layers.45.mlp.experts.98.down_proj", "model.layers.45.mlp.experts.99.down_proj", "model.layers.45.mlp.experts.100.down_proj", "model.layers.45.mlp.experts.101.down_proj", "model.layers.45.mlp.experts.102.down_proj", "model.layers.45.mlp.experts.103.down_proj", "model.layers.45.mlp.experts.104.down_proj", "model.layers.45.mlp.experts.105.down_proj", "model.layers.45.mlp.experts.106.down_proj", "model.layers.45.mlp.experts.107.down_proj", "model.layers.45.mlp.experts.108.down_proj", "model.layers.45.mlp.experts.109.down_proj", "model.layers.45.mlp.experts.110.down_proj", "model.layers.45.mlp.experts.111.down_proj", "model.layers.45.mlp.experts.112.down_proj", "model.layers.45.mlp.experts.113.down_proj", "model.layers.45.mlp.experts.114.down_proj", "model.layers.45.mlp.experts.115.down_proj", "model.layers.45.mlp.experts.116.down_proj", "model.layers.45.mlp.experts.117.down_proj", "model.layers.45.mlp.experts.118.down_proj", "model.layers.45.mlp.experts.119.down_proj", "model.layers.45.mlp.experts.120.down_proj", "model.layers.45.mlp.experts.121.down_proj", "model.layers.45.mlp.experts.122.down_proj", "model.layers.45.mlp.experts.123.down_proj", "model.layers.45.mlp.experts.124.down_proj", "model.layers.45.mlp.experts.125.down_proj", "model.layers.45.mlp.experts.126.down_proj", "model.layers.45.mlp.experts.127.down_proj", "model.layers.45.mlp.experts.128.down_proj", "model.layers.45.mlp.experts.129.down_proj", "model.layers.45.mlp.experts.130.down_proj", "model.layers.45.mlp.experts.131.down_proj", "model.layers.45.mlp.experts.132.down_proj", "model.layers.45.mlp.experts.133.down_proj", "model.layers.45.mlp.experts.134.down_proj", "model.layers.45.mlp.experts.135.down_proj", "model.layers.45.mlp.experts.136.down_proj", "model.layers.45.mlp.experts.137.down_proj", "model.layers.45.mlp.experts.138.down_proj", "model.layers.45.mlp.experts.139.down_proj", "model.layers.45.mlp.experts.140.down_proj", "model.layers.45.mlp.experts.141.down_proj", "model.layers.45.mlp.experts.142.down_proj", "model.layers.45.mlp.experts.143.down_proj", "model.layers.45.mlp.experts.144.down_proj", "model.layers.45.mlp.experts.145.down_proj", "model.layers.45.mlp.experts.146.down_proj", "model.layers.45.mlp.experts.147.down_proj", "model.layers.45.mlp.experts.148.down_proj", "model.layers.45.mlp.experts.149.down_proj", "model.layers.45.mlp.experts.150.down_proj", "model.layers.45.mlp.experts.151.down_proj", "model.layers.45.mlp.experts.152.down_proj", "model.layers.45.mlp.experts.153.down_proj", "model.layers.45.mlp.experts.154.down_proj", "model.layers.45.mlp.experts.155.down_proj", "model.layers.45.mlp.experts.156.down_proj", "model.layers.45.mlp.experts.157.down_proj", "model.layers.45.mlp.experts.158.down_proj", "model.layers.45.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0007251881062984578, "dbits": 1258291200 } ] }, { "idx": 273, "layers": [ "model.layers.46.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00022909566760065392, "dbits": 62914560 } ] }, { "idx": 274, "layers": [ "model.layers.46.self_attn.k_proj", "model.layers.46.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0006470374763011821, "dbits": 10485760 } ] }, { "idx": 275, "layers": [ "model.layers.46.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0018897727131843567, "dbits": 62914560 } ] }, { "idx": 276, "layers": [ "model.layers.46.mlp.shared_experts.gate_proj", "model.layers.46.mlp.shared_experts.up_proj", "model.layers.46.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.5960803031910284e-05, "dbits": 23592960 } ] }, { "idx": 277, "layers": [ "model.layers.46.mlp.experts.0.gate_proj", "model.layers.46.mlp.experts.1.gate_proj", "model.layers.46.mlp.experts.2.gate_proj", "model.layers.46.mlp.experts.3.gate_proj", "model.layers.46.mlp.experts.4.gate_proj", "model.layers.46.mlp.experts.5.gate_proj", "model.layers.46.mlp.experts.6.gate_proj", "model.layers.46.mlp.experts.7.gate_proj", "model.layers.46.mlp.experts.8.gate_proj", "model.layers.46.mlp.experts.9.gate_proj", "model.layers.46.mlp.experts.10.gate_proj", "model.layers.46.mlp.experts.11.gate_proj", "model.layers.46.mlp.experts.12.gate_proj", "model.layers.46.mlp.experts.13.gate_proj", "model.layers.46.mlp.experts.14.gate_proj", "model.layers.46.mlp.experts.15.gate_proj", "model.layers.46.mlp.experts.16.gate_proj", "model.layers.46.mlp.experts.17.gate_proj", "model.layers.46.mlp.experts.18.gate_proj", "model.layers.46.mlp.experts.19.gate_proj", "model.layers.46.mlp.experts.20.gate_proj", "model.layers.46.mlp.experts.21.gate_proj", "model.layers.46.mlp.experts.22.gate_proj", "model.layers.46.mlp.experts.23.gate_proj", "model.layers.46.mlp.experts.24.gate_proj", "model.layers.46.mlp.experts.25.gate_proj", "model.layers.46.mlp.experts.26.gate_proj", "model.layers.46.mlp.experts.27.gate_proj", "model.layers.46.mlp.experts.28.gate_proj", "model.layers.46.mlp.experts.29.gate_proj", "model.layers.46.mlp.experts.30.gate_proj", "model.layers.46.mlp.experts.31.gate_proj", "model.layers.46.mlp.experts.32.gate_proj", "model.layers.46.mlp.experts.33.gate_proj", "model.layers.46.mlp.experts.34.gate_proj", "model.layers.46.mlp.experts.35.gate_proj", "model.layers.46.mlp.experts.36.gate_proj", "model.layers.46.mlp.experts.37.gate_proj", "model.layers.46.mlp.experts.38.gate_proj", "model.layers.46.mlp.experts.39.gate_proj", "model.layers.46.mlp.experts.40.gate_proj", "model.layers.46.mlp.experts.41.gate_proj", "model.layers.46.mlp.experts.42.gate_proj", "model.layers.46.mlp.experts.43.gate_proj", "model.layers.46.mlp.experts.44.gate_proj", "model.layers.46.mlp.experts.45.gate_proj", "model.layers.46.mlp.experts.46.gate_proj", "model.layers.46.mlp.experts.47.gate_proj", "model.layers.46.mlp.experts.48.gate_proj", "model.layers.46.mlp.experts.49.gate_proj", "model.layers.46.mlp.experts.50.gate_proj", "model.layers.46.mlp.experts.51.gate_proj", "model.layers.46.mlp.experts.52.gate_proj", "model.layers.46.mlp.experts.53.gate_proj", "model.layers.46.mlp.experts.54.gate_proj", "model.layers.46.mlp.experts.55.gate_proj", "model.layers.46.mlp.experts.56.gate_proj", "model.layers.46.mlp.experts.57.gate_proj", "model.layers.46.mlp.experts.58.gate_proj", "model.layers.46.mlp.experts.59.gate_proj", "model.layers.46.mlp.experts.60.gate_proj", "model.layers.46.mlp.experts.61.gate_proj", "model.layers.46.mlp.experts.62.gate_proj", "model.layers.46.mlp.experts.63.gate_proj", "model.layers.46.mlp.experts.64.gate_proj", "model.layers.46.mlp.experts.65.gate_proj", "model.layers.46.mlp.experts.66.gate_proj", "model.layers.46.mlp.experts.67.gate_proj", "model.layers.46.mlp.experts.68.gate_proj", "model.layers.46.mlp.experts.69.gate_proj", "model.layers.46.mlp.experts.70.gate_proj", "model.layers.46.mlp.experts.71.gate_proj", "model.layers.46.mlp.experts.72.gate_proj", "model.layers.46.mlp.experts.73.gate_proj", "model.layers.46.mlp.experts.74.gate_proj", "model.layers.46.mlp.experts.75.gate_proj", "model.layers.46.mlp.experts.76.gate_proj", "model.layers.46.mlp.experts.77.gate_proj", "model.layers.46.mlp.experts.78.gate_proj", "model.layers.46.mlp.experts.79.gate_proj", "model.layers.46.mlp.experts.80.gate_proj", "model.layers.46.mlp.experts.81.gate_proj", "model.layers.46.mlp.experts.82.gate_proj", "model.layers.46.mlp.experts.83.gate_proj", "model.layers.46.mlp.experts.84.gate_proj", "model.layers.46.mlp.experts.85.gate_proj", "model.layers.46.mlp.experts.86.gate_proj", "model.layers.46.mlp.experts.87.gate_proj", "model.layers.46.mlp.experts.88.gate_proj", "model.layers.46.mlp.experts.89.gate_proj", "model.layers.46.mlp.experts.90.gate_proj", "model.layers.46.mlp.experts.91.gate_proj", "model.layers.46.mlp.experts.92.gate_proj", "model.layers.46.mlp.experts.93.gate_proj", "model.layers.46.mlp.experts.94.gate_proj", "model.layers.46.mlp.experts.95.gate_proj", "model.layers.46.mlp.experts.96.gate_proj", "model.layers.46.mlp.experts.97.gate_proj", "model.layers.46.mlp.experts.98.gate_proj", "model.layers.46.mlp.experts.99.gate_proj", "model.layers.46.mlp.experts.100.gate_proj", "model.layers.46.mlp.experts.101.gate_proj", "model.layers.46.mlp.experts.102.gate_proj", "model.layers.46.mlp.experts.103.gate_proj", "model.layers.46.mlp.experts.104.gate_proj", "model.layers.46.mlp.experts.105.gate_proj", "model.layers.46.mlp.experts.106.gate_proj", "model.layers.46.mlp.experts.107.gate_proj", "model.layers.46.mlp.experts.108.gate_proj", "model.layers.46.mlp.experts.109.gate_proj", "model.layers.46.mlp.experts.110.gate_proj", "model.layers.46.mlp.experts.111.gate_proj", "model.layers.46.mlp.experts.112.gate_proj", "model.layers.46.mlp.experts.113.gate_proj", "model.layers.46.mlp.experts.114.gate_proj", "model.layers.46.mlp.experts.115.gate_proj", "model.layers.46.mlp.experts.116.gate_proj", "model.layers.46.mlp.experts.117.gate_proj", "model.layers.46.mlp.experts.118.gate_proj", "model.layers.46.mlp.experts.119.gate_proj", "model.layers.46.mlp.experts.120.gate_proj", "model.layers.46.mlp.experts.121.gate_proj", "model.layers.46.mlp.experts.122.gate_proj", "model.layers.46.mlp.experts.123.gate_proj", "model.layers.46.mlp.experts.124.gate_proj", "model.layers.46.mlp.experts.125.gate_proj", "model.layers.46.mlp.experts.126.gate_proj", "model.layers.46.mlp.experts.127.gate_proj", "model.layers.46.mlp.experts.128.gate_proj", "model.layers.46.mlp.experts.129.gate_proj", "model.layers.46.mlp.experts.130.gate_proj", "model.layers.46.mlp.experts.131.gate_proj", "model.layers.46.mlp.experts.132.gate_proj", "model.layers.46.mlp.experts.133.gate_proj", "model.layers.46.mlp.experts.134.gate_proj", "model.layers.46.mlp.experts.135.gate_proj", "model.layers.46.mlp.experts.136.gate_proj", "model.layers.46.mlp.experts.137.gate_proj", "model.layers.46.mlp.experts.138.gate_proj", "model.layers.46.mlp.experts.139.gate_proj", "model.layers.46.mlp.experts.140.gate_proj", "model.layers.46.mlp.experts.141.gate_proj", "model.layers.46.mlp.experts.142.gate_proj", "model.layers.46.mlp.experts.143.gate_proj", "model.layers.46.mlp.experts.144.gate_proj", "model.layers.46.mlp.experts.145.gate_proj", "model.layers.46.mlp.experts.146.gate_proj", "model.layers.46.mlp.experts.147.gate_proj", "model.layers.46.mlp.experts.148.gate_proj", "model.layers.46.mlp.experts.149.gate_proj", "model.layers.46.mlp.experts.150.gate_proj", "model.layers.46.mlp.experts.151.gate_proj", "model.layers.46.mlp.experts.152.gate_proj", "model.layers.46.mlp.experts.153.gate_proj", "model.layers.46.mlp.experts.154.gate_proj", "model.layers.46.mlp.experts.155.gate_proj", "model.layers.46.mlp.experts.156.gate_proj", "model.layers.46.mlp.experts.157.gate_proj", "model.layers.46.mlp.experts.158.gate_proj", "model.layers.46.mlp.experts.159.gate_proj", "model.layers.46.mlp.experts.0.up_proj", "model.layers.46.mlp.experts.1.up_proj", "model.layers.46.mlp.experts.2.up_proj", "model.layers.46.mlp.experts.3.up_proj", "model.layers.46.mlp.experts.4.up_proj", "model.layers.46.mlp.experts.5.up_proj", "model.layers.46.mlp.experts.6.up_proj", "model.layers.46.mlp.experts.7.up_proj", "model.layers.46.mlp.experts.8.up_proj", "model.layers.46.mlp.experts.9.up_proj", "model.layers.46.mlp.experts.10.up_proj", "model.layers.46.mlp.experts.11.up_proj", "model.layers.46.mlp.experts.12.up_proj", "model.layers.46.mlp.experts.13.up_proj", "model.layers.46.mlp.experts.14.up_proj", "model.layers.46.mlp.experts.15.up_proj", "model.layers.46.mlp.experts.16.up_proj", "model.layers.46.mlp.experts.17.up_proj", "model.layers.46.mlp.experts.18.up_proj", "model.layers.46.mlp.experts.19.up_proj", "model.layers.46.mlp.experts.20.up_proj", "model.layers.46.mlp.experts.21.up_proj", "model.layers.46.mlp.experts.22.up_proj", "model.layers.46.mlp.experts.23.up_proj", "model.layers.46.mlp.experts.24.up_proj", "model.layers.46.mlp.experts.25.up_proj", "model.layers.46.mlp.experts.26.up_proj", "model.layers.46.mlp.experts.27.up_proj", "model.layers.46.mlp.experts.28.up_proj", "model.layers.46.mlp.experts.29.up_proj", "model.layers.46.mlp.experts.30.up_proj", "model.layers.46.mlp.experts.31.up_proj", "model.layers.46.mlp.experts.32.up_proj", "model.layers.46.mlp.experts.33.up_proj", "model.layers.46.mlp.experts.34.up_proj", "model.layers.46.mlp.experts.35.up_proj", "model.layers.46.mlp.experts.36.up_proj", "model.layers.46.mlp.experts.37.up_proj", "model.layers.46.mlp.experts.38.up_proj", "model.layers.46.mlp.experts.39.up_proj", "model.layers.46.mlp.experts.40.up_proj", "model.layers.46.mlp.experts.41.up_proj", "model.layers.46.mlp.experts.42.up_proj", "model.layers.46.mlp.experts.43.up_proj", "model.layers.46.mlp.experts.44.up_proj", "model.layers.46.mlp.experts.45.up_proj", "model.layers.46.mlp.experts.46.up_proj", "model.layers.46.mlp.experts.47.up_proj", "model.layers.46.mlp.experts.48.up_proj", "model.layers.46.mlp.experts.49.up_proj", "model.layers.46.mlp.experts.50.up_proj", "model.layers.46.mlp.experts.51.up_proj", "model.layers.46.mlp.experts.52.up_proj", "model.layers.46.mlp.experts.53.up_proj", "model.layers.46.mlp.experts.54.up_proj", "model.layers.46.mlp.experts.55.up_proj", "model.layers.46.mlp.experts.56.up_proj", "model.layers.46.mlp.experts.57.up_proj", "model.layers.46.mlp.experts.58.up_proj", "model.layers.46.mlp.experts.59.up_proj", "model.layers.46.mlp.experts.60.up_proj", "model.layers.46.mlp.experts.61.up_proj", "model.layers.46.mlp.experts.62.up_proj", "model.layers.46.mlp.experts.63.up_proj", "model.layers.46.mlp.experts.64.up_proj", "model.layers.46.mlp.experts.65.up_proj", "model.layers.46.mlp.experts.66.up_proj", "model.layers.46.mlp.experts.67.up_proj", "model.layers.46.mlp.experts.68.up_proj", "model.layers.46.mlp.experts.69.up_proj", "model.layers.46.mlp.experts.70.up_proj", "model.layers.46.mlp.experts.71.up_proj", "model.layers.46.mlp.experts.72.up_proj", "model.layers.46.mlp.experts.73.up_proj", "model.layers.46.mlp.experts.74.up_proj", "model.layers.46.mlp.experts.75.up_proj", "model.layers.46.mlp.experts.76.up_proj", "model.layers.46.mlp.experts.77.up_proj", "model.layers.46.mlp.experts.78.up_proj", "model.layers.46.mlp.experts.79.up_proj", "model.layers.46.mlp.experts.80.up_proj", "model.layers.46.mlp.experts.81.up_proj", "model.layers.46.mlp.experts.82.up_proj", "model.layers.46.mlp.experts.83.up_proj", "model.layers.46.mlp.experts.84.up_proj", "model.layers.46.mlp.experts.85.up_proj", "model.layers.46.mlp.experts.86.up_proj", "model.layers.46.mlp.experts.87.up_proj", "model.layers.46.mlp.experts.88.up_proj", "model.layers.46.mlp.experts.89.up_proj", "model.layers.46.mlp.experts.90.up_proj", "model.layers.46.mlp.experts.91.up_proj", "model.layers.46.mlp.experts.92.up_proj", "model.layers.46.mlp.experts.93.up_proj", "model.layers.46.mlp.experts.94.up_proj", "model.layers.46.mlp.experts.95.up_proj", "model.layers.46.mlp.experts.96.up_proj", "model.layers.46.mlp.experts.97.up_proj", "model.layers.46.mlp.experts.98.up_proj", "model.layers.46.mlp.experts.99.up_proj", "model.layers.46.mlp.experts.100.up_proj", "model.layers.46.mlp.experts.101.up_proj", "model.layers.46.mlp.experts.102.up_proj", "model.layers.46.mlp.experts.103.up_proj", "model.layers.46.mlp.experts.104.up_proj", "model.layers.46.mlp.experts.105.up_proj", "model.layers.46.mlp.experts.106.up_proj", "model.layers.46.mlp.experts.107.up_proj", "model.layers.46.mlp.experts.108.up_proj", "model.layers.46.mlp.experts.109.up_proj", "model.layers.46.mlp.experts.110.up_proj", "model.layers.46.mlp.experts.111.up_proj", "model.layers.46.mlp.experts.112.up_proj", "model.layers.46.mlp.experts.113.up_proj", "model.layers.46.mlp.experts.114.up_proj", "model.layers.46.mlp.experts.115.up_proj", "model.layers.46.mlp.experts.116.up_proj", "model.layers.46.mlp.experts.117.up_proj", "model.layers.46.mlp.experts.118.up_proj", "model.layers.46.mlp.experts.119.up_proj", "model.layers.46.mlp.experts.120.up_proj", "model.layers.46.mlp.experts.121.up_proj", "model.layers.46.mlp.experts.122.up_proj", "model.layers.46.mlp.experts.123.up_proj", "model.layers.46.mlp.experts.124.up_proj", "model.layers.46.mlp.experts.125.up_proj", "model.layers.46.mlp.experts.126.up_proj", "model.layers.46.mlp.experts.127.up_proj", "model.layers.46.mlp.experts.128.up_proj", "model.layers.46.mlp.experts.129.up_proj", "model.layers.46.mlp.experts.130.up_proj", "model.layers.46.mlp.experts.131.up_proj", "model.layers.46.mlp.experts.132.up_proj", "model.layers.46.mlp.experts.133.up_proj", "model.layers.46.mlp.experts.134.up_proj", "model.layers.46.mlp.experts.135.up_proj", "model.layers.46.mlp.experts.136.up_proj", "model.layers.46.mlp.experts.137.up_proj", "model.layers.46.mlp.experts.138.up_proj", "model.layers.46.mlp.experts.139.up_proj", "model.layers.46.mlp.experts.140.up_proj", "model.layers.46.mlp.experts.141.up_proj", "model.layers.46.mlp.experts.142.up_proj", "model.layers.46.mlp.experts.143.up_proj", "model.layers.46.mlp.experts.144.up_proj", "model.layers.46.mlp.experts.145.up_proj", "model.layers.46.mlp.experts.146.up_proj", "model.layers.46.mlp.experts.147.up_proj", "model.layers.46.mlp.experts.148.up_proj", "model.layers.46.mlp.experts.149.up_proj", "model.layers.46.mlp.experts.150.up_proj", "model.layers.46.mlp.experts.151.up_proj", "model.layers.46.mlp.experts.152.up_proj", "model.layers.46.mlp.experts.153.up_proj", "model.layers.46.mlp.experts.154.up_proj", "model.layers.46.mlp.experts.155.up_proj", "model.layers.46.mlp.experts.156.up_proj", "model.layers.46.mlp.experts.157.up_proj", "model.layers.46.mlp.experts.158.up_proj", "model.layers.46.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0011013194918632618, "dbits": 2516582400 } ] }, { "idx": 278, "layers": [ "model.layers.46.mlp.experts.0.down_proj", "model.layers.46.mlp.experts.1.down_proj", "model.layers.46.mlp.experts.2.down_proj", "model.layers.46.mlp.experts.3.down_proj", "model.layers.46.mlp.experts.4.down_proj", "model.layers.46.mlp.experts.5.down_proj", "model.layers.46.mlp.experts.6.down_proj", "model.layers.46.mlp.experts.7.down_proj", "model.layers.46.mlp.experts.8.down_proj", "model.layers.46.mlp.experts.9.down_proj", "model.layers.46.mlp.experts.10.down_proj", "model.layers.46.mlp.experts.11.down_proj", "model.layers.46.mlp.experts.12.down_proj", "model.layers.46.mlp.experts.13.down_proj", "model.layers.46.mlp.experts.14.down_proj", "model.layers.46.mlp.experts.15.down_proj", "model.layers.46.mlp.experts.16.down_proj", "model.layers.46.mlp.experts.17.down_proj", "model.layers.46.mlp.experts.18.down_proj", "model.layers.46.mlp.experts.19.down_proj", "model.layers.46.mlp.experts.20.down_proj", "model.layers.46.mlp.experts.21.down_proj", "model.layers.46.mlp.experts.22.down_proj", "model.layers.46.mlp.experts.23.down_proj", "model.layers.46.mlp.experts.24.down_proj", "model.layers.46.mlp.experts.25.down_proj", "model.layers.46.mlp.experts.26.down_proj", "model.layers.46.mlp.experts.27.down_proj", "model.layers.46.mlp.experts.28.down_proj", "model.layers.46.mlp.experts.29.down_proj", "model.layers.46.mlp.experts.30.down_proj", "model.layers.46.mlp.experts.31.down_proj", "model.layers.46.mlp.experts.32.down_proj", "model.layers.46.mlp.experts.33.down_proj", "model.layers.46.mlp.experts.34.down_proj", "model.layers.46.mlp.experts.35.down_proj", "model.layers.46.mlp.experts.36.down_proj", "model.layers.46.mlp.experts.37.down_proj", "model.layers.46.mlp.experts.38.down_proj", "model.layers.46.mlp.experts.39.down_proj", "model.layers.46.mlp.experts.40.down_proj", "model.layers.46.mlp.experts.41.down_proj", "model.layers.46.mlp.experts.42.down_proj", "model.layers.46.mlp.experts.43.down_proj", "model.layers.46.mlp.experts.44.down_proj", "model.layers.46.mlp.experts.45.down_proj", "model.layers.46.mlp.experts.46.down_proj", "model.layers.46.mlp.experts.47.down_proj", "model.layers.46.mlp.experts.48.down_proj", "model.layers.46.mlp.experts.49.down_proj", "model.layers.46.mlp.experts.50.down_proj", "model.layers.46.mlp.experts.51.down_proj", "model.layers.46.mlp.experts.52.down_proj", "model.layers.46.mlp.experts.53.down_proj", "model.layers.46.mlp.experts.54.down_proj", "model.layers.46.mlp.experts.55.down_proj", "model.layers.46.mlp.experts.56.down_proj", "model.layers.46.mlp.experts.57.down_proj", "model.layers.46.mlp.experts.58.down_proj", "model.layers.46.mlp.experts.59.down_proj", "model.layers.46.mlp.experts.60.down_proj", "model.layers.46.mlp.experts.61.down_proj", "model.layers.46.mlp.experts.62.down_proj", "model.layers.46.mlp.experts.63.down_proj", "model.layers.46.mlp.experts.64.down_proj", "model.layers.46.mlp.experts.65.down_proj", "model.layers.46.mlp.experts.66.down_proj", "model.layers.46.mlp.experts.67.down_proj", "model.layers.46.mlp.experts.68.down_proj", "model.layers.46.mlp.experts.69.down_proj", "model.layers.46.mlp.experts.70.down_proj", "model.layers.46.mlp.experts.71.down_proj", "model.layers.46.mlp.experts.72.down_proj", "model.layers.46.mlp.experts.73.down_proj", "model.layers.46.mlp.experts.74.down_proj", "model.layers.46.mlp.experts.75.down_proj", "model.layers.46.mlp.experts.76.down_proj", "model.layers.46.mlp.experts.77.down_proj", "model.layers.46.mlp.experts.78.down_proj", "model.layers.46.mlp.experts.79.down_proj", "model.layers.46.mlp.experts.80.down_proj", "model.layers.46.mlp.experts.81.down_proj", "model.layers.46.mlp.experts.82.down_proj", "model.layers.46.mlp.experts.83.down_proj", "model.layers.46.mlp.experts.84.down_proj", "model.layers.46.mlp.experts.85.down_proj", "model.layers.46.mlp.experts.86.down_proj", "model.layers.46.mlp.experts.87.down_proj", "model.layers.46.mlp.experts.88.down_proj", "model.layers.46.mlp.experts.89.down_proj", "model.layers.46.mlp.experts.90.down_proj", "model.layers.46.mlp.experts.91.down_proj", "model.layers.46.mlp.experts.92.down_proj", "model.layers.46.mlp.experts.93.down_proj", "model.layers.46.mlp.experts.94.down_proj", "model.layers.46.mlp.experts.95.down_proj", "model.layers.46.mlp.experts.96.down_proj", "model.layers.46.mlp.experts.97.down_proj", "model.layers.46.mlp.experts.98.down_proj", "model.layers.46.mlp.experts.99.down_proj", "model.layers.46.mlp.experts.100.down_proj", "model.layers.46.mlp.experts.101.down_proj", "model.layers.46.mlp.experts.102.down_proj", "model.layers.46.mlp.experts.103.down_proj", "model.layers.46.mlp.experts.104.down_proj", "model.layers.46.mlp.experts.105.down_proj", "model.layers.46.mlp.experts.106.down_proj", "model.layers.46.mlp.experts.107.down_proj", "model.layers.46.mlp.experts.108.down_proj", "model.layers.46.mlp.experts.109.down_proj", "model.layers.46.mlp.experts.110.down_proj", "model.layers.46.mlp.experts.111.down_proj", "model.layers.46.mlp.experts.112.down_proj", "model.layers.46.mlp.experts.113.down_proj", "model.layers.46.mlp.experts.114.down_proj", "model.layers.46.mlp.experts.115.down_proj", "model.layers.46.mlp.experts.116.down_proj", "model.layers.46.mlp.experts.117.down_proj", "model.layers.46.mlp.experts.118.down_proj", "model.layers.46.mlp.experts.119.down_proj", "model.layers.46.mlp.experts.120.down_proj", "model.layers.46.mlp.experts.121.down_proj", "model.layers.46.mlp.experts.122.down_proj", "model.layers.46.mlp.experts.123.down_proj", "model.layers.46.mlp.experts.124.down_proj", "model.layers.46.mlp.experts.125.down_proj", "model.layers.46.mlp.experts.126.down_proj", "model.layers.46.mlp.experts.127.down_proj", "model.layers.46.mlp.experts.128.down_proj", "model.layers.46.mlp.experts.129.down_proj", "model.layers.46.mlp.experts.130.down_proj", "model.layers.46.mlp.experts.131.down_proj", "model.layers.46.mlp.experts.132.down_proj", "model.layers.46.mlp.experts.133.down_proj", "model.layers.46.mlp.experts.134.down_proj", "model.layers.46.mlp.experts.135.down_proj", "model.layers.46.mlp.experts.136.down_proj", "model.layers.46.mlp.experts.137.down_proj", "model.layers.46.mlp.experts.138.down_proj", "model.layers.46.mlp.experts.139.down_proj", "model.layers.46.mlp.experts.140.down_proj", "model.layers.46.mlp.experts.141.down_proj", "model.layers.46.mlp.experts.142.down_proj", "model.layers.46.mlp.experts.143.down_proj", "model.layers.46.mlp.experts.144.down_proj", "model.layers.46.mlp.experts.145.down_proj", "model.layers.46.mlp.experts.146.down_proj", "model.layers.46.mlp.experts.147.down_proj", "model.layers.46.mlp.experts.148.down_proj", "model.layers.46.mlp.experts.149.down_proj", "model.layers.46.mlp.experts.150.down_proj", "model.layers.46.mlp.experts.151.down_proj", "model.layers.46.mlp.experts.152.down_proj", "model.layers.46.mlp.experts.153.down_proj", "model.layers.46.mlp.experts.154.down_proj", "model.layers.46.mlp.experts.155.down_proj", "model.layers.46.mlp.experts.156.down_proj", "model.layers.46.mlp.experts.157.down_proj", "model.layers.46.mlp.experts.158.down_proj", "model.layers.46.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0010587736964225991, "dbits": 1258291200 } ] }, { "idx": 279, "layers": [ "model.layers.47.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0015959963202476501, "dbits": 62914560 } ] }, { "idx": 280, "layers": [ "model.layers.47.self_attn.k_proj", "model.layers.47.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0001504071056843137, "dbits": 10485760 } ] }, { "idx": 281, "layers": [ "model.layers.47.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00021029934287070118, "dbits": 62914560 } ] }, { "idx": 282, "layers": [ "model.layers.47.mlp.shared_experts.gate_proj", "model.layers.47.mlp.shared_experts.up_proj", "model.layers.47.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0007121577858924533, "dbits": 23592960 } ] }, { "idx": 283, "layers": [ "model.layers.47.mlp.experts.0.gate_proj", "model.layers.47.mlp.experts.1.gate_proj", "model.layers.47.mlp.experts.2.gate_proj", "model.layers.47.mlp.experts.3.gate_proj", "model.layers.47.mlp.experts.4.gate_proj", "model.layers.47.mlp.experts.5.gate_proj", "model.layers.47.mlp.experts.6.gate_proj", "model.layers.47.mlp.experts.7.gate_proj", "model.layers.47.mlp.experts.8.gate_proj", "model.layers.47.mlp.experts.9.gate_proj", "model.layers.47.mlp.experts.10.gate_proj", "model.layers.47.mlp.experts.11.gate_proj", "model.layers.47.mlp.experts.12.gate_proj", "model.layers.47.mlp.experts.13.gate_proj", "model.layers.47.mlp.experts.14.gate_proj", "model.layers.47.mlp.experts.15.gate_proj", "model.layers.47.mlp.experts.16.gate_proj", "model.layers.47.mlp.experts.17.gate_proj", "model.layers.47.mlp.experts.18.gate_proj", "model.layers.47.mlp.experts.19.gate_proj", "model.layers.47.mlp.experts.20.gate_proj", "model.layers.47.mlp.experts.21.gate_proj", "model.layers.47.mlp.experts.22.gate_proj", "model.layers.47.mlp.experts.23.gate_proj", "model.layers.47.mlp.experts.24.gate_proj", "model.layers.47.mlp.experts.25.gate_proj", "model.layers.47.mlp.experts.26.gate_proj", "model.layers.47.mlp.experts.27.gate_proj", "model.layers.47.mlp.experts.28.gate_proj", "model.layers.47.mlp.experts.29.gate_proj", "model.layers.47.mlp.experts.30.gate_proj", "model.layers.47.mlp.experts.31.gate_proj", "model.layers.47.mlp.experts.32.gate_proj", "model.layers.47.mlp.experts.33.gate_proj", "model.layers.47.mlp.experts.34.gate_proj", "model.layers.47.mlp.experts.35.gate_proj", "model.layers.47.mlp.experts.36.gate_proj", "model.layers.47.mlp.experts.37.gate_proj", "model.layers.47.mlp.experts.38.gate_proj", "model.layers.47.mlp.experts.39.gate_proj", "model.layers.47.mlp.experts.40.gate_proj", "model.layers.47.mlp.experts.41.gate_proj", "model.layers.47.mlp.experts.42.gate_proj", "model.layers.47.mlp.experts.43.gate_proj", "model.layers.47.mlp.experts.44.gate_proj", "model.layers.47.mlp.experts.45.gate_proj", "model.layers.47.mlp.experts.46.gate_proj", "model.layers.47.mlp.experts.47.gate_proj", "model.layers.47.mlp.experts.48.gate_proj", "model.layers.47.mlp.experts.49.gate_proj", "model.layers.47.mlp.experts.50.gate_proj", "model.layers.47.mlp.experts.51.gate_proj", "model.layers.47.mlp.experts.52.gate_proj", "model.layers.47.mlp.experts.53.gate_proj", "model.layers.47.mlp.experts.54.gate_proj", "model.layers.47.mlp.experts.55.gate_proj", "model.layers.47.mlp.experts.56.gate_proj", "model.layers.47.mlp.experts.57.gate_proj", "model.layers.47.mlp.experts.58.gate_proj", "model.layers.47.mlp.experts.59.gate_proj", "model.layers.47.mlp.experts.60.gate_proj", "model.layers.47.mlp.experts.61.gate_proj", "model.layers.47.mlp.experts.62.gate_proj", "model.layers.47.mlp.experts.63.gate_proj", "model.layers.47.mlp.experts.64.gate_proj", "model.layers.47.mlp.experts.65.gate_proj", "model.layers.47.mlp.experts.66.gate_proj", "model.layers.47.mlp.experts.67.gate_proj", "model.layers.47.mlp.experts.68.gate_proj", "model.layers.47.mlp.experts.69.gate_proj", "model.layers.47.mlp.experts.70.gate_proj", "model.layers.47.mlp.experts.71.gate_proj", "model.layers.47.mlp.experts.72.gate_proj", "model.layers.47.mlp.experts.73.gate_proj", "model.layers.47.mlp.experts.74.gate_proj", "model.layers.47.mlp.experts.75.gate_proj", "model.layers.47.mlp.experts.76.gate_proj", "model.layers.47.mlp.experts.77.gate_proj", "model.layers.47.mlp.experts.78.gate_proj", "model.layers.47.mlp.experts.79.gate_proj", "model.layers.47.mlp.experts.80.gate_proj", "model.layers.47.mlp.experts.81.gate_proj", "model.layers.47.mlp.experts.82.gate_proj", "model.layers.47.mlp.experts.83.gate_proj", "model.layers.47.mlp.experts.84.gate_proj", "model.layers.47.mlp.experts.85.gate_proj", "model.layers.47.mlp.experts.86.gate_proj", "model.layers.47.mlp.experts.87.gate_proj", "model.layers.47.mlp.experts.88.gate_proj", "model.layers.47.mlp.experts.89.gate_proj", "model.layers.47.mlp.experts.90.gate_proj", "model.layers.47.mlp.experts.91.gate_proj", "model.layers.47.mlp.experts.92.gate_proj", "model.layers.47.mlp.experts.93.gate_proj", "model.layers.47.mlp.experts.94.gate_proj", "model.layers.47.mlp.experts.95.gate_proj", "model.layers.47.mlp.experts.96.gate_proj", "model.layers.47.mlp.experts.97.gate_proj", "model.layers.47.mlp.experts.98.gate_proj", "model.layers.47.mlp.experts.99.gate_proj", "model.layers.47.mlp.experts.100.gate_proj", "model.layers.47.mlp.experts.101.gate_proj", "model.layers.47.mlp.experts.102.gate_proj", "model.layers.47.mlp.experts.103.gate_proj", "model.layers.47.mlp.experts.104.gate_proj", "model.layers.47.mlp.experts.105.gate_proj", "model.layers.47.mlp.experts.106.gate_proj", "model.layers.47.mlp.experts.107.gate_proj", "model.layers.47.mlp.experts.108.gate_proj", "model.layers.47.mlp.experts.109.gate_proj", "model.layers.47.mlp.experts.110.gate_proj", "model.layers.47.mlp.experts.111.gate_proj", "model.layers.47.mlp.experts.112.gate_proj", "model.layers.47.mlp.experts.113.gate_proj", "model.layers.47.mlp.experts.114.gate_proj", "model.layers.47.mlp.experts.115.gate_proj", "model.layers.47.mlp.experts.116.gate_proj", "model.layers.47.mlp.experts.117.gate_proj", "model.layers.47.mlp.experts.118.gate_proj", "model.layers.47.mlp.experts.119.gate_proj", "model.layers.47.mlp.experts.120.gate_proj", "model.layers.47.mlp.experts.121.gate_proj", "model.layers.47.mlp.experts.122.gate_proj", "model.layers.47.mlp.experts.123.gate_proj", "model.layers.47.mlp.experts.124.gate_proj", "model.layers.47.mlp.experts.125.gate_proj", "model.layers.47.mlp.experts.126.gate_proj", "model.layers.47.mlp.experts.127.gate_proj", "model.layers.47.mlp.experts.128.gate_proj", "model.layers.47.mlp.experts.129.gate_proj", "model.layers.47.mlp.experts.130.gate_proj", "model.layers.47.mlp.experts.131.gate_proj", "model.layers.47.mlp.experts.132.gate_proj", "model.layers.47.mlp.experts.133.gate_proj", "model.layers.47.mlp.experts.134.gate_proj", "model.layers.47.mlp.experts.135.gate_proj", "model.layers.47.mlp.experts.136.gate_proj", "model.layers.47.mlp.experts.137.gate_proj", "model.layers.47.mlp.experts.138.gate_proj", "model.layers.47.mlp.experts.139.gate_proj", "model.layers.47.mlp.experts.140.gate_proj", "model.layers.47.mlp.experts.141.gate_proj", "model.layers.47.mlp.experts.142.gate_proj", "model.layers.47.mlp.experts.143.gate_proj", "model.layers.47.mlp.experts.144.gate_proj", "model.layers.47.mlp.experts.145.gate_proj", "model.layers.47.mlp.experts.146.gate_proj", "model.layers.47.mlp.experts.147.gate_proj", "model.layers.47.mlp.experts.148.gate_proj", "model.layers.47.mlp.experts.149.gate_proj", "model.layers.47.mlp.experts.150.gate_proj", "model.layers.47.mlp.experts.151.gate_proj", "model.layers.47.mlp.experts.152.gate_proj", "model.layers.47.mlp.experts.153.gate_proj", "model.layers.47.mlp.experts.154.gate_proj", "model.layers.47.mlp.experts.155.gate_proj", "model.layers.47.mlp.experts.156.gate_proj", "model.layers.47.mlp.experts.157.gate_proj", "model.layers.47.mlp.experts.158.gate_proj", "model.layers.47.mlp.experts.159.gate_proj", "model.layers.47.mlp.experts.0.up_proj", "model.layers.47.mlp.experts.1.up_proj", "model.layers.47.mlp.experts.2.up_proj", "model.layers.47.mlp.experts.3.up_proj", "model.layers.47.mlp.experts.4.up_proj", "model.layers.47.mlp.experts.5.up_proj", "model.layers.47.mlp.experts.6.up_proj", "model.layers.47.mlp.experts.7.up_proj", "model.layers.47.mlp.experts.8.up_proj", "model.layers.47.mlp.experts.9.up_proj", "model.layers.47.mlp.experts.10.up_proj", "model.layers.47.mlp.experts.11.up_proj", "model.layers.47.mlp.experts.12.up_proj", "model.layers.47.mlp.experts.13.up_proj", "model.layers.47.mlp.experts.14.up_proj", "model.layers.47.mlp.experts.15.up_proj", "model.layers.47.mlp.experts.16.up_proj", "model.layers.47.mlp.experts.17.up_proj", "model.layers.47.mlp.experts.18.up_proj", "model.layers.47.mlp.experts.19.up_proj", "model.layers.47.mlp.experts.20.up_proj", "model.layers.47.mlp.experts.21.up_proj", "model.layers.47.mlp.experts.22.up_proj", "model.layers.47.mlp.experts.23.up_proj", "model.layers.47.mlp.experts.24.up_proj", "model.layers.47.mlp.experts.25.up_proj", "model.layers.47.mlp.experts.26.up_proj", "model.layers.47.mlp.experts.27.up_proj", "model.layers.47.mlp.experts.28.up_proj", "model.layers.47.mlp.experts.29.up_proj", "model.layers.47.mlp.experts.30.up_proj", "model.layers.47.mlp.experts.31.up_proj", "model.layers.47.mlp.experts.32.up_proj", "model.layers.47.mlp.experts.33.up_proj", "model.layers.47.mlp.experts.34.up_proj", "model.layers.47.mlp.experts.35.up_proj", "model.layers.47.mlp.experts.36.up_proj", "model.layers.47.mlp.experts.37.up_proj", "model.layers.47.mlp.experts.38.up_proj", "model.layers.47.mlp.experts.39.up_proj", "model.layers.47.mlp.experts.40.up_proj", "model.layers.47.mlp.experts.41.up_proj", "model.layers.47.mlp.experts.42.up_proj", "model.layers.47.mlp.experts.43.up_proj", "model.layers.47.mlp.experts.44.up_proj", "model.layers.47.mlp.experts.45.up_proj", "model.layers.47.mlp.experts.46.up_proj", "model.layers.47.mlp.experts.47.up_proj", "model.layers.47.mlp.experts.48.up_proj", "model.layers.47.mlp.experts.49.up_proj", "model.layers.47.mlp.experts.50.up_proj", "model.layers.47.mlp.experts.51.up_proj", "model.layers.47.mlp.experts.52.up_proj", "model.layers.47.mlp.experts.53.up_proj", "model.layers.47.mlp.experts.54.up_proj", "model.layers.47.mlp.experts.55.up_proj", "model.layers.47.mlp.experts.56.up_proj", "model.layers.47.mlp.experts.57.up_proj", "model.layers.47.mlp.experts.58.up_proj", "model.layers.47.mlp.experts.59.up_proj", "model.layers.47.mlp.experts.60.up_proj", "model.layers.47.mlp.experts.61.up_proj", "model.layers.47.mlp.experts.62.up_proj", "model.layers.47.mlp.experts.63.up_proj", "model.layers.47.mlp.experts.64.up_proj", "model.layers.47.mlp.experts.65.up_proj", "model.layers.47.mlp.experts.66.up_proj", "model.layers.47.mlp.experts.67.up_proj", "model.layers.47.mlp.experts.68.up_proj", "model.layers.47.mlp.experts.69.up_proj", "model.layers.47.mlp.experts.70.up_proj", "model.layers.47.mlp.experts.71.up_proj", "model.layers.47.mlp.experts.72.up_proj", "model.layers.47.mlp.experts.73.up_proj", "model.layers.47.mlp.experts.74.up_proj", "model.layers.47.mlp.experts.75.up_proj", "model.layers.47.mlp.experts.76.up_proj", "model.layers.47.mlp.experts.77.up_proj", "model.layers.47.mlp.experts.78.up_proj", "model.layers.47.mlp.experts.79.up_proj", "model.layers.47.mlp.experts.80.up_proj", "model.layers.47.mlp.experts.81.up_proj", "model.layers.47.mlp.experts.82.up_proj", "model.layers.47.mlp.experts.83.up_proj", "model.layers.47.mlp.experts.84.up_proj", "model.layers.47.mlp.experts.85.up_proj", "model.layers.47.mlp.experts.86.up_proj", "model.layers.47.mlp.experts.87.up_proj", "model.layers.47.mlp.experts.88.up_proj", "model.layers.47.mlp.experts.89.up_proj", "model.layers.47.mlp.experts.90.up_proj", "model.layers.47.mlp.experts.91.up_proj", "model.layers.47.mlp.experts.92.up_proj", "model.layers.47.mlp.experts.93.up_proj", "model.layers.47.mlp.experts.94.up_proj", "model.layers.47.mlp.experts.95.up_proj", "model.layers.47.mlp.experts.96.up_proj", "model.layers.47.mlp.experts.97.up_proj", "model.layers.47.mlp.experts.98.up_proj", "model.layers.47.mlp.experts.99.up_proj", "model.layers.47.mlp.experts.100.up_proj", "model.layers.47.mlp.experts.101.up_proj", "model.layers.47.mlp.experts.102.up_proj", "model.layers.47.mlp.experts.103.up_proj", "model.layers.47.mlp.experts.104.up_proj", "model.layers.47.mlp.experts.105.up_proj", "model.layers.47.mlp.experts.106.up_proj", "model.layers.47.mlp.experts.107.up_proj", "model.layers.47.mlp.experts.108.up_proj", "model.layers.47.mlp.experts.109.up_proj", "model.layers.47.mlp.experts.110.up_proj", "model.layers.47.mlp.experts.111.up_proj", "model.layers.47.mlp.experts.112.up_proj", "model.layers.47.mlp.experts.113.up_proj", "model.layers.47.mlp.experts.114.up_proj", "model.layers.47.mlp.experts.115.up_proj", "model.layers.47.mlp.experts.116.up_proj", "model.layers.47.mlp.experts.117.up_proj", "model.layers.47.mlp.experts.118.up_proj", "model.layers.47.mlp.experts.119.up_proj", "model.layers.47.mlp.experts.120.up_proj", "model.layers.47.mlp.experts.121.up_proj", "model.layers.47.mlp.experts.122.up_proj", "model.layers.47.mlp.experts.123.up_proj", "model.layers.47.mlp.experts.124.up_proj", "model.layers.47.mlp.experts.125.up_proj", "model.layers.47.mlp.experts.126.up_proj", "model.layers.47.mlp.experts.127.up_proj", "model.layers.47.mlp.experts.128.up_proj", "model.layers.47.mlp.experts.129.up_proj", "model.layers.47.mlp.experts.130.up_proj", "model.layers.47.mlp.experts.131.up_proj", "model.layers.47.mlp.experts.132.up_proj", "model.layers.47.mlp.experts.133.up_proj", "model.layers.47.mlp.experts.134.up_proj", "model.layers.47.mlp.experts.135.up_proj", "model.layers.47.mlp.experts.136.up_proj", "model.layers.47.mlp.experts.137.up_proj", "model.layers.47.mlp.experts.138.up_proj", "model.layers.47.mlp.experts.139.up_proj", "model.layers.47.mlp.experts.140.up_proj", "model.layers.47.mlp.experts.141.up_proj", "model.layers.47.mlp.experts.142.up_proj", "model.layers.47.mlp.experts.143.up_proj", "model.layers.47.mlp.experts.144.up_proj", "model.layers.47.mlp.experts.145.up_proj", "model.layers.47.mlp.experts.146.up_proj", "model.layers.47.mlp.experts.147.up_proj", "model.layers.47.mlp.experts.148.up_proj", "model.layers.47.mlp.experts.149.up_proj", "model.layers.47.mlp.experts.150.up_proj", "model.layers.47.mlp.experts.151.up_proj", "model.layers.47.mlp.experts.152.up_proj", "model.layers.47.mlp.experts.153.up_proj", "model.layers.47.mlp.experts.154.up_proj", "model.layers.47.mlp.experts.155.up_proj", "model.layers.47.mlp.experts.156.up_proj", "model.layers.47.mlp.experts.157.up_proj", "model.layers.47.mlp.experts.158.up_proj", "model.layers.47.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0008656226098537334, "dbits": 2516582400 } ] }, { "idx": 284, "layers": [ "model.layers.47.mlp.experts.0.down_proj", "model.layers.47.mlp.experts.1.down_proj", "model.layers.47.mlp.experts.2.down_proj", "model.layers.47.mlp.experts.3.down_proj", "model.layers.47.mlp.experts.4.down_proj", "model.layers.47.mlp.experts.5.down_proj", "model.layers.47.mlp.experts.6.down_proj", "model.layers.47.mlp.experts.7.down_proj", "model.layers.47.mlp.experts.8.down_proj", "model.layers.47.mlp.experts.9.down_proj", "model.layers.47.mlp.experts.10.down_proj", "model.layers.47.mlp.experts.11.down_proj", "model.layers.47.mlp.experts.12.down_proj", "model.layers.47.mlp.experts.13.down_proj", "model.layers.47.mlp.experts.14.down_proj", "model.layers.47.mlp.experts.15.down_proj", "model.layers.47.mlp.experts.16.down_proj", "model.layers.47.mlp.experts.17.down_proj", "model.layers.47.mlp.experts.18.down_proj", "model.layers.47.mlp.experts.19.down_proj", "model.layers.47.mlp.experts.20.down_proj", "model.layers.47.mlp.experts.21.down_proj", "model.layers.47.mlp.experts.22.down_proj", "model.layers.47.mlp.experts.23.down_proj", "model.layers.47.mlp.experts.24.down_proj", "model.layers.47.mlp.experts.25.down_proj", "model.layers.47.mlp.experts.26.down_proj", "model.layers.47.mlp.experts.27.down_proj", "model.layers.47.mlp.experts.28.down_proj", "model.layers.47.mlp.experts.29.down_proj", "model.layers.47.mlp.experts.30.down_proj", "model.layers.47.mlp.experts.31.down_proj", "model.layers.47.mlp.experts.32.down_proj", "model.layers.47.mlp.experts.33.down_proj", "model.layers.47.mlp.experts.34.down_proj", "model.layers.47.mlp.experts.35.down_proj", "model.layers.47.mlp.experts.36.down_proj", "model.layers.47.mlp.experts.37.down_proj", "model.layers.47.mlp.experts.38.down_proj", "model.layers.47.mlp.experts.39.down_proj", "model.layers.47.mlp.experts.40.down_proj", "model.layers.47.mlp.experts.41.down_proj", "model.layers.47.mlp.experts.42.down_proj", "model.layers.47.mlp.experts.43.down_proj", "model.layers.47.mlp.experts.44.down_proj", "model.layers.47.mlp.experts.45.down_proj", "model.layers.47.mlp.experts.46.down_proj", "model.layers.47.mlp.experts.47.down_proj", "model.layers.47.mlp.experts.48.down_proj", "model.layers.47.mlp.experts.49.down_proj", "model.layers.47.mlp.experts.50.down_proj", "model.layers.47.mlp.experts.51.down_proj", "model.layers.47.mlp.experts.52.down_proj", "model.layers.47.mlp.experts.53.down_proj", "model.layers.47.mlp.experts.54.down_proj", "model.layers.47.mlp.experts.55.down_proj", "model.layers.47.mlp.experts.56.down_proj", "model.layers.47.mlp.experts.57.down_proj", "model.layers.47.mlp.experts.58.down_proj", "model.layers.47.mlp.experts.59.down_proj", "model.layers.47.mlp.experts.60.down_proj", "model.layers.47.mlp.experts.61.down_proj", "model.layers.47.mlp.experts.62.down_proj", "model.layers.47.mlp.experts.63.down_proj", "model.layers.47.mlp.experts.64.down_proj", "model.layers.47.mlp.experts.65.down_proj", "model.layers.47.mlp.experts.66.down_proj", "model.layers.47.mlp.experts.67.down_proj", "model.layers.47.mlp.experts.68.down_proj", "model.layers.47.mlp.experts.69.down_proj", "model.layers.47.mlp.experts.70.down_proj", "model.layers.47.mlp.experts.71.down_proj", "model.layers.47.mlp.experts.72.down_proj", "model.layers.47.mlp.experts.73.down_proj", "model.layers.47.mlp.experts.74.down_proj", "model.layers.47.mlp.experts.75.down_proj", "model.layers.47.mlp.experts.76.down_proj", "model.layers.47.mlp.experts.77.down_proj", "model.layers.47.mlp.experts.78.down_proj", "model.layers.47.mlp.experts.79.down_proj", "model.layers.47.mlp.experts.80.down_proj", "model.layers.47.mlp.experts.81.down_proj", "model.layers.47.mlp.experts.82.down_proj", "model.layers.47.mlp.experts.83.down_proj", "model.layers.47.mlp.experts.84.down_proj", "model.layers.47.mlp.experts.85.down_proj", "model.layers.47.mlp.experts.86.down_proj", "model.layers.47.mlp.experts.87.down_proj", "model.layers.47.mlp.experts.88.down_proj", "model.layers.47.mlp.experts.89.down_proj", "model.layers.47.mlp.experts.90.down_proj", "model.layers.47.mlp.experts.91.down_proj", "model.layers.47.mlp.experts.92.down_proj", "model.layers.47.mlp.experts.93.down_proj", "model.layers.47.mlp.experts.94.down_proj", "model.layers.47.mlp.experts.95.down_proj", "model.layers.47.mlp.experts.96.down_proj", "model.layers.47.mlp.experts.97.down_proj", "model.layers.47.mlp.experts.98.down_proj", "model.layers.47.mlp.experts.99.down_proj", "model.layers.47.mlp.experts.100.down_proj", "model.layers.47.mlp.experts.101.down_proj", "model.layers.47.mlp.experts.102.down_proj", "model.layers.47.mlp.experts.103.down_proj", "model.layers.47.mlp.experts.104.down_proj", "model.layers.47.mlp.experts.105.down_proj", "model.layers.47.mlp.experts.106.down_proj", "model.layers.47.mlp.experts.107.down_proj", "model.layers.47.mlp.experts.108.down_proj", "model.layers.47.mlp.experts.109.down_proj", "model.layers.47.mlp.experts.110.down_proj", "model.layers.47.mlp.experts.111.down_proj", "model.layers.47.mlp.experts.112.down_proj", "model.layers.47.mlp.experts.113.down_proj", "model.layers.47.mlp.experts.114.down_proj", "model.layers.47.mlp.experts.115.down_proj", "model.layers.47.mlp.experts.116.down_proj", "model.layers.47.mlp.experts.117.down_proj", "model.layers.47.mlp.experts.118.down_proj", "model.layers.47.mlp.experts.119.down_proj", "model.layers.47.mlp.experts.120.down_proj", "model.layers.47.mlp.experts.121.down_proj", "model.layers.47.mlp.experts.122.down_proj", "model.layers.47.mlp.experts.123.down_proj", "model.layers.47.mlp.experts.124.down_proj", "model.layers.47.mlp.experts.125.down_proj", "model.layers.47.mlp.experts.126.down_proj", "model.layers.47.mlp.experts.127.down_proj", "model.layers.47.mlp.experts.128.down_proj", "model.layers.47.mlp.experts.129.down_proj", "model.layers.47.mlp.experts.130.down_proj", "model.layers.47.mlp.experts.131.down_proj", "model.layers.47.mlp.experts.132.down_proj", "model.layers.47.mlp.experts.133.down_proj", "model.layers.47.mlp.experts.134.down_proj", "model.layers.47.mlp.experts.135.down_proj", "model.layers.47.mlp.experts.136.down_proj", "model.layers.47.mlp.experts.137.down_proj", "model.layers.47.mlp.experts.138.down_proj", "model.layers.47.mlp.experts.139.down_proj", "model.layers.47.mlp.experts.140.down_proj", "model.layers.47.mlp.experts.141.down_proj", "model.layers.47.mlp.experts.142.down_proj", "model.layers.47.mlp.experts.143.down_proj", "model.layers.47.mlp.experts.144.down_proj", "model.layers.47.mlp.experts.145.down_proj", "model.layers.47.mlp.experts.146.down_proj", "model.layers.47.mlp.experts.147.down_proj", "model.layers.47.mlp.experts.148.down_proj", "model.layers.47.mlp.experts.149.down_proj", "model.layers.47.mlp.experts.150.down_proj", "model.layers.47.mlp.experts.151.down_proj", "model.layers.47.mlp.experts.152.down_proj", "model.layers.47.mlp.experts.153.down_proj", "model.layers.47.mlp.experts.154.down_proj", "model.layers.47.mlp.experts.155.down_proj", "model.layers.47.mlp.experts.156.down_proj", "model.layers.47.mlp.experts.157.down_proj", "model.layers.47.mlp.experts.158.down_proj", "model.layers.47.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0008842833340167999, "dbits": 1258291200 } ] }, { "idx": 285, "layers": [ "model.layers.48.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0006745606660842896, "dbits": 62914560 } ] }, { "idx": 286, "layers": [ "model.layers.48.self_attn.k_proj", "model.layers.48.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0031440526247024647, "dbits": 10485760 } ] }, { "idx": 287, "layers": [ "model.layers.48.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00016749277710914612, "dbits": 62914560 } ] }, { "idx": 288, "layers": [ "model.layers.48.mlp.shared_experts.gate_proj", "model.layers.48.mlp.shared_experts.up_proj", "model.layers.48.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0008768290281295776, "dbits": 23592960 } ] }, { "idx": 289, "layers": [ "model.layers.48.mlp.experts.0.gate_proj", "model.layers.48.mlp.experts.1.gate_proj", "model.layers.48.mlp.experts.2.gate_proj", "model.layers.48.mlp.experts.3.gate_proj", "model.layers.48.mlp.experts.4.gate_proj", "model.layers.48.mlp.experts.5.gate_proj", "model.layers.48.mlp.experts.6.gate_proj", "model.layers.48.mlp.experts.7.gate_proj", "model.layers.48.mlp.experts.8.gate_proj", "model.layers.48.mlp.experts.9.gate_proj", "model.layers.48.mlp.experts.10.gate_proj", "model.layers.48.mlp.experts.11.gate_proj", "model.layers.48.mlp.experts.12.gate_proj", "model.layers.48.mlp.experts.13.gate_proj", "model.layers.48.mlp.experts.14.gate_proj", "model.layers.48.mlp.experts.15.gate_proj", "model.layers.48.mlp.experts.16.gate_proj", "model.layers.48.mlp.experts.17.gate_proj", "model.layers.48.mlp.experts.18.gate_proj", "model.layers.48.mlp.experts.19.gate_proj", "model.layers.48.mlp.experts.20.gate_proj", "model.layers.48.mlp.experts.21.gate_proj", "model.layers.48.mlp.experts.22.gate_proj", "model.layers.48.mlp.experts.23.gate_proj", "model.layers.48.mlp.experts.24.gate_proj", "model.layers.48.mlp.experts.25.gate_proj", "model.layers.48.mlp.experts.26.gate_proj", "model.layers.48.mlp.experts.27.gate_proj", "model.layers.48.mlp.experts.28.gate_proj", "model.layers.48.mlp.experts.29.gate_proj", "model.layers.48.mlp.experts.30.gate_proj", "model.layers.48.mlp.experts.31.gate_proj", "model.layers.48.mlp.experts.32.gate_proj", "model.layers.48.mlp.experts.33.gate_proj", "model.layers.48.mlp.experts.34.gate_proj", "model.layers.48.mlp.experts.35.gate_proj", "model.layers.48.mlp.experts.36.gate_proj", "model.layers.48.mlp.experts.37.gate_proj", "model.layers.48.mlp.experts.38.gate_proj", "model.layers.48.mlp.experts.39.gate_proj", "model.layers.48.mlp.experts.40.gate_proj", "model.layers.48.mlp.experts.41.gate_proj", "model.layers.48.mlp.experts.42.gate_proj", "model.layers.48.mlp.experts.43.gate_proj", "model.layers.48.mlp.experts.44.gate_proj", "model.layers.48.mlp.experts.45.gate_proj", "model.layers.48.mlp.experts.46.gate_proj", "model.layers.48.mlp.experts.47.gate_proj", "model.layers.48.mlp.experts.48.gate_proj", "model.layers.48.mlp.experts.49.gate_proj", "model.layers.48.mlp.experts.50.gate_proj", "model.layers.48.mlp.experts.51.gate_proj", "model.layers.48.mlp.experts.52.gate_proj", "model.layers.48.mlp.experts.53.gate_proj", "model.layers.48.mlp.experts.54.gate_proj", "model.layers.48.mlp.experts.55.gate_proj", "model.layers.48.mlp.experts.56.gate_proj", "model.layers.48.mlp.experts.57.gate_proj", "model.layers.48.mlp.experts.58.gate_proj", "model.layers.48.mlp.experts.59.gate_proj", "model.layers.48.mlp.experts.60.gate_proj", "model.layers.48.mlp.experts.61.gate_proj", "model.layers.48.mlp.experts.62.gate_proj", "model.layers.48.mlp.experts.63.gate_proj", "model.layers.48.mlp.experts.64.gate_proj", "model.layers.48.mlp.experts.65.gate_proj", "model.layers.48.mlp.experts.66.gate_proj", "model.layers.48.mlp.experts.67.gate_proj", "model.layers.48.mlp.experts.68.gate_proj", "model.layers.48.mlp.experts.69.gate_proj", "model.layers.48.mlp.experts.70.gate_proj", "model.layers.48.mlp.experts.71.gate_proj", "model.layers.48.mlp.experts.72.gate_proj", "model.layers.48.mlp.experts.73.gate_proj", "model.layers.48.mlp.experts.74.gate_proj", "model.layers.48.mlp.experts.75.gate_proj", "model.layers.48.mlp.experts.76.gate_proj", "model.layers.48.mlp.experts.77.gate_proj", "model.layers.48.mlp.experts.78.gate_proj", "model.layers.48.mlp.experts.79.gate_proj", "model.layers.48.mlp.experts.80.gate_proj", "model.layers.48.mlp.experts.81.gate_proj", "model.layers.48.mlp.experts.82.gate_proj", "model.layers.48.mlp.experts.83.gate_proj", "model.layers.48.mlp.experts.84.gate_proj", "model.layers.48.mlp.experts.85.gate_proj", "model.layers.48.mlp.experts.86.gate_proj", "model.layers.48.mlp.experts.87.gate_proj", "model.layers.48.mlp.experts.88.gate_proj", "model.layers.48.mlp.experts.89.gate_proj", "model.layers.48.mlp.experts.90.gate_proj", "model.layers.48.mlp.experts.91.gate_proj", "model.layers.48.mlp.experts.92.gate_proj", "model.layers.48.mlp.experts.93.gate_proj", "model.layers.48.mlp.experts.94.gate_proj", "model.layers.48.mlp.experts.95.gate_proj", "model.layers.48.mlp.experts.96.gate_proj", "model.layers.48.mlp.experts.97.gate_proj", "model.layers.48.mlp.experts.98.gate_proj", "model.layers.48.mlp.experts.99.gate_proj", "model.layers.48.mlp.experts.100.gate_proj", "model.layers.48.mlp.experts.101.gate_proj", "model.layers.48.mlp.experts.102.gate_proj", "model.layers.48.mlp.experts.103.gate_proj", "model.layers.48.mlp.experts.104.gate_proj", "model.layers.48.mlp.experts.105.gate_proj", "model.layers.48.mlp.experts.106.gate_proj", "model.layers.48.mlp.experts.107.gate_proj", "model.layers.48.mlp.experts.108.gate_proj", "model.layers.48.mlp.experts.109.gate_proj", "model.layers.48.mlp.experts.110.gate_proj", "model.layers.48.mlp.experts.111.gate_proj", "model.layers.48.mlp.experts.112.gate_proj", "model.layers.48.mlp.experts.113.gate_proj", "model.layers.48.mlp.experts.114.gate_proj", "model.layers.48.mlp.experts.115.gate_proj", "model.layers.48.mlp.experts.116.gate_proj", "model.layers.48.mlp.experts.117.gate_proj", "model.layers.48.mlp.experts.118.gate_proj", "model.layers.48.mlp.experts.119.gate_proj", "model.layers.48.mlp.experts.120.gate_proj", "model.layers.48.mlp.experts.121.gate_proj", "model.layers.48.mlp.experts.122.gate_proj", "model.layers.48.mlp.experts.123.gate_proj", "model.layers.48.mlp.experts.124.gate_proj", "model.layers.48.mlp.experts.125.gate_proj", "model.layers.48.mlp.experts.126.gate_proj", "model.layers.48.mlp.experts.127.gate_proj", "model.layers.48.mlp.experts.128.gate_proj", "model.layers.48.mlp.experts.129.gate_proj", "model.layers.48.mlp.experts.130.gate_proj", "model.layers.48.mlp.experts.131.gate_proj", "model.layers.48.mlp.experts.132.gate_proj", "model.layers.48.mlp.experts.133.gate_proj", "model.layers.48.mlp.experts.134.gate_proj", "model.layers.48.mlp.experts.135.gate_proj", "model.layers.48.mlp.experts.136.gate_proj", "model.layers.48.mlp.experts.137.gate_proj", "model.layers.48.mlp.experts.138.gate_proj", "model.layers.48.mlp.experts.139.gate_proj", "model.layers.48.mlp.experts.140.gate_proj", "model.layers.48.mlp.experts.141.gate_proj", "model.layers.48.mlp.experts.142.gate_proj", "model.layers.48.mlp.experts.143.gate_proj", "model.layers.48.mlp.experts.144.gate_proj", "model.layers.48.mlp.experts.145.gate_proj", "model.layers.48.mlp.experts.146.gate_proj", "model.layers.48.mlp.experts.147.gate_proj", "model.layers.48.mlp.experts.148.gate_proj", "model.layers.48.mlp.experts.149.gate_proj", "model.layers.48.mlp.experts.150.gate_proj", "model.layers.48.mlp.experts.151.gate_proj", "model.layers.48.mlp.experts.152.gate_proj", "model.layers.48.mlp.experts.153.gate_proj", "model.layers.48.mlp.experts.154.gate_proj", "model.layers.48.mlp.experts.155.gate_proj", "model.layers.48.mlp.experts.156.gate_proj", "model.layers.48.mlp.experts.157.gate_proj", "model.layers.48.mlp.experts.158.gate_proj", "model.layers.48.mlp.experts.159.gate_proj", "model.layers.48.mlp.experts.0.up_proj", "model.layers.48.mlp.experts.1.up_proj", "model.layers.48.mlp.experts.2.up_proj", "model.layers.48.mlp.experts.3.up_proj", "model.layers.48.mlp.experts.4.up_proj", "model.layers.48.mlp.experts.5.up_proj", "model.layers.48.mlp.experts.6.up_proj", "model.layers.48.mlp.experts.7.up_proj", "model.layers.48.mlp.experts.8.up_proj", "model.layers.48.mlp.experts.9.up_proj", "model.layers.48.mlp.experts.10.up_proj", "model.layers.48.mlp.experts.11.up_proj", "model.layers.48.mlp.experts.12.up_proj", "model.layers.48.mlp.experts.13.up_proj", "model.layers.48.mlp.experts.14.up_proj", "model.layers.48.mlp.experts.15.up_proj", "model.layers.48.mlp.experts.16.up_proj", "model.layers.48.mlp.experts.17.up_proj", "model.layers.48.mlp.experts.18.up_proj", "model.layers.48.mlp.experts.19.up_proj", "model.layers.48.mlp.experts.20.up_proj", "model.layers.48.mlp.experts.21.up_proj", "model.layers.48.mlp.experts.22.up_proj", "model.layers.48.mlp.experts.23.up_proj", "model.layers.48.mlp.experts.24.up_proj", "model.layers.48.mlp.experts.25.up_proj", "model.layers.48.mlp.experts.26.up_proj", "model.layers.48.mlp.experts.27.up_proj", "model.layers.48.mlp.experts.28.up_proj", "model.layers.48.mlp.experts.29.up_proj", "model.layers.48.mlp.experts.30.up_proj", "model.layers.48.mlp.experts.31.up_proj", "model.layers.48.mlp.experts.32.up_proj", "model.layers.48.mlp.experts.33.up_proj", "model.layers.48.mlp.experts.34.up_proj", "model.layers.48.mlp.experts.35.up_proj", "model.layers.48.mlp.experts.36.up_proj", "model.layers.48.mlp.experts.37.up_proj", "model.layers.48.mlp.experts.38.up_proj", "model.layers.48.mlp.experts.39.up_proj", "model.layers.48.mlp.experts.40.up_proj", "model.layers.48.mlp.experts.41.up_proj", "model.layers.48.mlp.experts.42.up_proj", "model.layers.48.mlp.experts.43.up_proj", "model.layers.48.mlp.experts.44.up_proj", "model.layers.48.mlp.experts.45.up_proj", "model.layers.48.mlp.experts.46.up_proj", "model.layers.48.mlp.experts.47.up_proj", "model.layers.48.mlp.experts.48.up_proj", "model.layers.48.mlp.experts.49.up_proj", "model.layers.48.mlp.experts.50.up_proj", "model.layers.48.mlp.experts.51.up_proj", "model.layers.48.mlp.experts.52.up_proj", "model.layers.48.mlp.experts.53.up_proj", "model.layers.48.mlp.experts.54.up_proj", "model.layers.48.mlp.experts.55.up_proj", "model.layers.48.mlp.experts.56.up_proj", "model.layers.48.mlp.experts.57.up_proj", "model.layers.48.mlp.experts.58.up_proj", "model.layers.48.mlp.experts.59.up_proj", "model.layers.48.mlp.experts.60.up_proj", "model.layers.48.mlp.experts.61.up_proj", "model.layers.48.mlp.experts.62.up_proj", "model.layers.48.mlp.experts.63.up_proj", "model.layers.48.mlp.experts.64.up_proj", "model.layers.48.mlp.experts.65.up_proj", "model.layers.48.mlp.experts.66.up_proj", "model.layers.48.mlp.experts.67.up_proj", "model.layers.48.mlp.experts.68.up_proj", "model.layers.48.mlp.experts.69.up_proj", "model.layers.48.mlp.experts.70.up_proj", "model.layers.48.mlp.experts.71.up_proj", "model.layers.48.mlp.experts.72.up_proj", "model.layers.48.mlp.experts.73.up_proj", "model.layers.48.mlp.experts.74.up_proj", "model.layers.48.mlp.experts.75.up_proj", "model.layers.48.mlp.experts.76.up_proj", "model.layers.48.mlp.experts.77.up_proj", "model.layers.48.mlp.experts.78.up_proj", "model.layers.48.mlp.experts.79.up_proj", "model.layers.48.mlp.experts.80.up_proj", "model.layers.48.mlp.experts.81.up_proj", "model.layers.48.mlp.experts.82.up_proj", "model.layers.48.mlp.experts.83.up_proj", "model.layers.48.mlp.experts.84.up_proj", "model.layers.48.mlp.experts.85.up_proj", "model.layers.48.mlp.experts.86.up_proj", "model.layers.48.mlp.experts.87.up_proj", "model.layers.48.mlp.experts.88.up_proj", "model.layers.48.mlp.experts.89.up_proj", "model.layers.48.mlp.experts.90.up_proj", "model.layers.48.mlp.experts.91.up_proj", "model.layers.48.mlp.experts.92.up_proj", "model.layers.48.mlp.experts.93.up_proj", "model.layers.48.mlp.experts.94.up_proj", "model.layers.48.mlp.experts.95.up_proj", "model.layers.48.mlp.experts.96.up_proj", "model.layers.48.mlp.experts.97.up_proj", "model.layers.48.mlp.experts.98.up_proj", "model.layers.48.mlp.experts.99.up_proj", "model.layers.48.mlp.experts.100.up_proj", "model.layers.48.mlp.experts.101.up_proj", "model.layers.48.mlp.experts.102.up_proj", "model.layers.48.mlp.experts.103.up_proj", "model.layers.48.mlp.experts.104.up_proj", "model.layers.48.mlp.experts.105.up_proj", "model.layers.48.mlp.experts.106.up_proj", "model.layers.48.mlp.experts.107.up_proj", "model.layers.48.mlp.experts.108.up_proj", "model.layers.48.mlp.experts.109.up_proj", "model.layers.48.mlp.experts.110.up_proj", "model.layers.48.mlp.experts.111.up_proj", "model.layers.48.mlp.experts.112.up_proj", "model.layers.48.mlp.experts.113.up_proj", "model.layers.48.mlp.experts.114.up_proj", "model.layers.48.mlp.experts.115.up_proj", "model.layers.48.mlp.experts.116.up_proj", "model.layers.48.mlp.experts.117.up_proj", "model.layers.48.mlp.experts.118.up_proj", "model.layers.48.mlp.experts.119.up_proj", "model.layers.48.mlp.experts.120.up_proj", "model.layers.48.mlp.experts.121.up_proj", "model.layers.48.mlp.experts.122.up_proj", "model.layers.48.mlp.experts.123.up_proj", "model.layers.48.mlp.experts.124.up_proj", "model.layers.48.mlp.experts.125.up_proj", "model.layers.48.mlp.experts.126.up_proj", "model.layers.48.mlp.experts.127.up_proj", "model.layers.48.mlp.experts.128.up_proj", "model.layers.48.mlp.experts.129.up_proj", "model.layers.48.mlp.experts.130.up_proj", "model.layers.48.mlp.experts.131.up_proj", "model.layers.48.mlp.experts.132.up_proj", "model.layers.48.mlp.experts.133.up_proj", "model.layers.48.mlp.experts.134.up_proj", "model.layers.48.mlp.experts.135.up_proj", "model.layers.48.mlp.experts.136.up_proj", "model.layers.48.mlp.experts.137.up_proj", "model.layers.48.mlp.experts.138.up_proj", "model.layers.48.mlp.experts.139.up_proj", "model.layers.48.mlp.experts.140.up_proj", "model.layers.48.mlp.experts.141.up_proj", "model.layers.48.mlp.experts.142.up_proj", "model.layers.48.mlp.experts.143.up_proj", "model.layers.48.mlp.experts.144.up_proj", "model.layers.48.mlp.experts.145.up_proj", "model.layers.48.mlp.experts.146.up_proj", "model.layers.48.mlp.experts.147.up_proj", "model.layers.48.mlp.experts.148.up_proj", "model.layers.48.mlp.experts.149.up_proj", "model.layers.48.mlp.experts.150.up_proj", "model.layers.48.mlp.experts.151.up_proj", "model.layers.48.mlp.experts.152.up_proj", "model.layers.48.mlp.experts.153.up_proj", "model.layers.48.mlp.experts.154.up_proj", "model.layers.48.mlp.experts.155.up_proj", "model.layers.48.mlp.experts.156.up_proj", "model.layers.48.mlp.experts.157.up_proj", "model.layers.48.mlp.experts.158.up_proj", "model.layers.48.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0009322300553322171, "dbits": 2516582400 } ] }, { "idx": 290, "layers": [ "model.layers.48.mlp.experts.0.down_proj", "model.layers.48.mlp.experts.1.down_proj", "model.layers.48.mlp.experts.2.down_proj", "model.layers.48.mlp.experts.3.down_proj", "model.layers.48.mlp.experts.4.down_proj", "model.layers.48.mlp.experts.5.down_proj", "model.layers.48.mlp.experts.6.down_proj", "model.layers.48.mlp.experts.7.down_proj", "model.layers.48.mlp.experts.8.down_proj", "model.layers.48.mlp.experts.9.down_proj", "model.layers.48.mlp.experts.10.down_proj", "model.layers.48.mlp.experts.11.down_proj", "model.layers.48.mlp.experts.12.down_proj", "model.layers.48.mlp.experts.13.down_proj", "model.layers.48.mlp.experts.14.down_proj", "model.layers.48.mlp.experts.15.down_proj", "model.layers.48.mlp.experts.16.down_proj", "model.layers.48.mlp.experts.17.down_proj", "model.layers.48.mlp.experts.18.down_proj", "model.layers.48.mlp.experts.19.down_proj", "model.layers.48.mlp.experts.20.down_proj", "model.layers.48.mlp.experts.21.down_proj", "model.layers.48.mlp.experts.22.down_proj", "model.layers.48.mlp.experts.23.down_proj", "model.layers.48.mlp.experts.24.down_proj", "model.layers.48.mlp.experts.25.down_proj", "model.layers.48.mlp.experts.26.down_proj", "model.layers.48.mlp.experts.27.down_proj", "model.layers.48.mlp.experts.28.down_proj", "model.layers.48.mlp.experts.29.down_proj", "model.layers.48.mlp.experts.30.down_proj", "model.layers.48.mlp.experts.31.down_proj", "model.layers.48.mlp.experts.32.down_proj", "model.layers.48.mlp.experts.33.down_proj", "model.layers.48.mlp.experts.34.down_proj", "model.layers.48.mlp.experts.35.down_proj", "model.layers.48.mlp.experts.36.down_proj", "model.layers.48.mlp.experts.37.down_proj", "model.layers.48.mlp.experts.38.down_proj", "model.layers.48.mlp.experts.39.down_proj", "model.layers.48.mlp.experts.40.down_proj", "model.layers.48.mlp.experts.41.down_proj", "model.layers.48.mlp.experts.42.down_proj", "model.layers.48.mlp.experts.43.down_proj", "model.layers.48.mlp.experts.44.down_proj", "model.layers.48.mlp.experts.45.down_proj", "model.layers.48.mlp.experts.46.down_proj", "model.layers.48.mlp.experts.47.down_proj", "model.layers.48.mlp.experts.48.down_proj", "model.layers.48.mlp.experts.49.down_proj", "model.layers.48.mlp.experts.50.down_proj", "model.layers.48.mlp.experts.51.down_proj", "model.layers.48.mlp.experts.52.down_proj", "model.layers.48.mlp.experts.53.down_proj", "model.layers.48.mlp.experts.54.down_proj", "model.layers.48.mlp.experts.55.down_proj", "model.layers.48.mlp.experts.56.down_proj", "model.layers.48.mlp.experts.57.down_proj", "model.layers.48.mlp.experts.58.down_proj", "model.layers.48.mlp.experts.59.down_proj", "model.layers.48.mlp.experts.60.down_proj", "model.layers.48.mlp.experts.61.down_proj", "model.layers.48.mlp.experts.62.down_proj", "model.layers.48.mlp.experts.63.down_proj", "model.layers.48.mlp.experts.64.down_proj", "model.layers.48.mlp.experts.65.down_proj", "model.layers.48.mlp.experts.66.down_proj", "model.layers.48.mlp.experts.67.down_proj", "model.layers.48.mlp.experts.68.down_proj", "model.layers.48.mlp.experts.69.down_proj", "model.layers.48.mlp.experts.70.down_proj", "model.layers.48.mlp.experts.71.down_proj", "model.layers.48.mlp.experts.72.down_proj", "model.layers.48.mlp.experts.73.down_proj", "model.layers.48.mlp.experts.74.down_proj", "model.layers.48.mlp.experts.75.down_proj", "model.layers.48.mlp.experts.76.down_proj", "model.layers.48.mlp.experts.77.down_proj", "model.layers.48.mlp.experts.78.down_proj", "model.layers.48.mlp.experts.79.down_proj", "model.layers.48.mlp.experts.80.down_proj", "model.layers.48.mlp.experts.81.down_proj", "model.layers.48.mlp.experts.82.down_proj", "model.layers.48.mlp.experts.83.down_proj", "model.layers.48.mlp.experts.84.down_proj", "model.layers.48.mlp.experts.85.down_proj", "model.layers.48.mlp.experts.86.down_proj", "model.layers.48.mlp.experts.87.down_proj", "model.layers.48.mlp.experts.88.down_proj", "model.layers.48.mlp.experts.89.down_proj", "model.layers.48.mlp.experts.90.down_proj", "model.layers.48.mlp.experts.91.down_proj", "model.layers.48.mlp.experts.92.down_proj", "model.layers.48.mlp.experts.93.down_proj", "model.layers.48.mlp.experts.94.down_proj", "model.layers.48.mlp.experts.95.down_proj", "model.layers.48.mlp.experts.96.down_proj", "model.layers.48.mlp.experts.97.down_proj", "model.layers.48.mlp.experts.98.down_proj", "model.layers.48.mlp.experts.99.down_proj", "model.layers.48.mlp.experts.100.down_proj", "model.layers.48.mlp.experts.101.down_proj", "model.layers.48.mlp.experts.102.down_proj", "model.layers.48.mlp.experts.103.down_proj", "model.layers.48.mlp.experts.104.down_proj", "model.layers.48.mlp.experts.105.down_proj", "model.layers.48.mlp.experts.106.down_proj", "model.layers.48.mlp.experts.107.down_proj", "model.layers.48.mlp.experts.108.down_proj", "model.layers.48.mlp.experts.109.down_proj", "model.layers.48.mlp.experts.110.down_proj", "model.layers.48.mlp.experts.111.down_proj", "model.layers.48.mlp.experts.112.down_proj", "model.layers.48.mlp.experts.113.down_proj", "model.layers.48.mlp.experts.114.down_proj", "model.layers.48.mlp.experts.115.down_proj", "model.layers.48.mlp.experts.116.down_proj", "model.layers.48.mlp.experts.117.down_proj", "model.layers.48.mlp.experts.118.down_proj", "model.layers.48.mlp.experts.119.down_proj", "model.layers.48.mlp.experts.120.down_proj", "model.layers.48.mlp.experts.121.down_proj", "model.layers.48.mlp.experts.122.down_proj", "model.layers.48.mlp.experts.123.down_proj", "model.layers.48.mlp.experts.124.down_proj", "model.layers.48.mlp.experts.125.down_proj", "model.layers.48.mlp.experts.126.down_proj", "model.layers.48.mlp.experts.127.down_proj", "model.layers.48.mlp.experts.128.down_proj", "model.layers.48.mlp.experts.129.down_proj", "model.layers.48.mlp.experts.130.down_proj", "model.layers.48.mlp.experts.131.down_proj", "model.layers.48.mlp.experts.132.down_proj", "model.layers.48.mlp.experts.133.down_proj", "model.layers.48.mlp.experts.134.down_proj", "model.layers.48.mlp.experts.135.down_proj", "model.layers.48.mlp.experts.136.down_proj", "model.layers.48.mlp.experts.137.down_proj", "model.layers.48.mlp.experts.138.down_proj", "model.layers.48.mlp.experts.139.down_proj", "model.layers.48.mlp.experts.140.down_proj", "model.layers.48.mlp.experts.141.down_proj", "model.layers.48.mlp.experts.142.down_proj", "model.layers.48.mlp.experts.143.down_proj", "model.layers.48.mlp.experts.144.down_proj", "model.layers.48.mlp.experts.145.down_proj", "model.layers.48.mlp.experts.146.down_proj", "model.layers.48.mlp.experts.147.down_proj", "model.layers.48.mlp.experts.148.down_proj", "model.layers.48.mlp.experts.149.down_proj", "model.layers.48.mlp.experts.150.down_proj", "model.layers.48.mlp.experts.151.down_proj", "model.layers.48.mlp.experts.152.down_proj", "model.layers.48.mlp.experts.153.down_proj", "model.layers.48.mlp.experts.154.down_proj", "model.layers.48.mlp.experts.155.down_proj", "model.layers.48.mlp.experts.156.down_proj", "model.layers.48.mlp.experts.157.down_proj", "model.layers.48.mlp.experts.158.down_proj", "model.layers.48.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0009577952325344197, "dbits": 1258291200 } ] }, { "idx": 291, "layers": [ "model.layers.49.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0027435824275016674, "dbits": 62914560 } ] }, { "idx": 292, "layers": [ "model.layers.49.self_attn.k_proj", "model.layers.49.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0008031368255615456, "dbits": 10485760 } ] }, { "idx": 293, "layers": [ "model.layers.49.self_attn.o_proj" ], "candidates": [ { "dkld": -3.419816493988037e-05, "dbits": 62914560 } ] }, { "idx": 294, "layers": [ "model.layers.49.mlp.shared_experts.gate_proj", "model.layers.49.mlp.shared_experts.up_proj", "model.layers.49.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.002722470462322213, "dbits": 23592960 } ] }, { "idx": 295, "layers": [ "model.layers.49.mlp.experts.0.gate_proj", "model.layers.49.mlp.experts.1.gate_proj", "model.layers.49.mlp.experts.2.gate_proj", "model.layers.49.mlp.experts.3.gate_proj", "model.layers.49.mlp.experts.4.gate_proj", "model.layers.49.mlp.experts.5.gate_proj", "model.layers.49.mlp.experts.6.gate_proj", "model.layers.49.mlp.experts.7.gate_proj", "model.layers.49.mlp.experts.8.gate_proj", "model.layers.49.mlp.experts.9.gate_proj", "model.layers.49.mlp.experts.10.gate_proj", "model.layers.49.mlp.experts.11.gate_proj", "model.layers.49.mlp.experts.12.gate_proj", "model.layers.49.mlp.experts.13.gate_proj", "model.layers.49.mlp.experts.14.gate_proj", "model.layers.49.mlp.experts.15.gate_proj", "model.layers.49.mlp.experts.16.gate_proj", "model.layers.49.mlp.experts.17.gate_proj", "model.layers.49.mlp.experts.18.gate_proj", "model.layers.49.mlp.experts.19.gate_proj", "model.layers.49.mlp.experts.20.gate_proj", "model.layers.49.mlp.experts.21.gate_proj", "model.layers.49.mlp.experts.22.gate_proj", "model.layers.49.mlp.experts.23.gate_proj", "model.layers.49.mlp.experts.24.gate_proj", "model.layers.49.mlp.experts.25.gate_proj", "model.layers.49.mlp.experts.26.gate_proj", "model.layers.49.mlp.experts.27.gate_proj", "model.layers.49.mlp.experts.28.gate_proj", "model.layers.49.mlp.experts.29.gate_proj", "model.layers.49.mlp.experts.30.gate_proj", "model.layers.49.mlp.experts.31.gate_proj", "model.layers.49.mlp.experts.32.gate_proj", "model.layers.49.mlp.experts.33.gate_proj", "model.layers.49.mlp.experts.34.gate_proj", "model.layers.49.mlp.experts.35.gate_proj", "model.layers.49.mlp.experts.36.gate_proj", "model.layers.49.mlp.experts.37.gate_proj", "model.layers.49.mlp.experts.38.gate_proj", "model.layers.49.mlp.experts.39.gate_proj", "model.layers.49.mlp.experts.40.gate_proj", "model.layers.49.mlp.experts.41.gate_proj", "model.layers.49.mlp.experts.42.gate_proj", "model.layers.49.mlp.experts.43.gate_proj", "model.layers.49.mlp.experts.44.gate_proj", "model.layers.49.mlp.experts.45.gate_proj", "model.layers.49.mlp.experts.46.gate_proj", "model.layers.49.mlp.experts.47.gate_proj", "model.layers.49.mlp.experts.48.gate_proj", "model.layers.49.mlp.experts.49.gate_proj", "model.layers.49.mlp.experts.50.gate_proj", "model.layers.49.mlp.experts.51.gate_proj", "model.layers.49.mlp.experts.52.gate_proj", "model.layers.49.mlp.experts.53.gate_proj", "model.layers.49.mlp.experts.54.gate_proj", "model.layers.49.mlp.experts.55.gate_proj", "model.layers.49.mlp.experts.56.gate_proj", "model.layers.49.mlp.experts.57.gate_proj", "model.layers.49.mlp.experts.58.gate_proj", "model.layers.49.mlp.experts.59.gate_proj", "model.layers.49.mlp.experts.60.gate_proj", "model.layers.49.mlp.experts.61.gate_proj", "model.layers.49.mlp.experts.62.gate_proj", "model.layers.49.mlp.experts.63.gate_proj", "model.layers.49.mlp.experts.64.gate_proj", "model.layers.49.mlp.experts.65.gate_proj", "model.layers.49.mlp.experts.66.gate_proj", "model.layers.49.mlp.experts.67.gate_proj", "model.layers.49.mlp.experts.68.gate_proj", "model.layers.49.mlp.experts.69.gate_proj", "model.layers.49.mlp.experts.70.gate_proj", "model.layers.49.mlp.experts.71.gate_proj", "model.layers.49.mlp.experts.72.gate_proj", "model.layers.49.mlp.experts.73.gate_proj", "model.layers.49.mlp.experts.74.gate_proj", "model.layers.49.mlp.experts.75.gate_proj", "model.layers.49.mlp.experts.76.gate_proj", "model.layers.49.mlp.experts.77.gate_proj", "model.layers.49.mlp.experts.78.gate_proj", "model.layers.49.mlp.experts.79.gate_proj", "model.layers.49.mlp.experts.80.gate_proj", "model.layers.49.mlp.experts.81.gate_proj", "model.layers.49.mlp.experts.82.gate_proj", "model.layers.49.mlp.experts.83.gate_proj", "model.layers.49.mlp.experts.84.gate_proj", "model.layers.49.mlp.experts.85.gate_proj", "model.layers.49.mlp.experts.86.gate_proj", "model.layers.49.mlp.experts.87.gate_proj", "model.layers.49.mlp.experts.88.gate_proj", "model.layers.49.mlp.experts.89.gate_proj", "model.layers.49.mlp.experts.90.gate_proj", "model.layers.49.mlp.experts.91.gate_proj", "model.layers.49.mlp.experts.92.gate_proj", "model.layers.49.mlp.experts.93.gate_proj", "model.layers.49.mlp.experts.94.gate_proj", "model.layers.49.mlp.experts.95.gate_proj", "model.layers.49.mlp.experts.96.gate_proj", "model.layers.49.mlp.experts.97.gate_proj", "model.layers.49.mlp.experts.98.gate_proj", "model.layers.49.mlp.experts.99.gate_proj", "model.layers.49.mlp.experts.100.gate_proj", "model.layers.49.mlp.experts.101.gate_proj", "model.layers.49.mlp.experts.102.gate_proj", "model.layers.49.mlp.experts.103.gate_proj", "model.layers.49.mlp.experts.104.gate_proj", "model.layers.49.mlp.experts.105.gate_proj", "model.layers.49.mlp.experts.106.gate_proj", "model.layers.49.mlp.experts.107.gate_proj", "model.layers.49.mlp.experts.108.gate_proj", "model.layers.49.mlp.experts.109.gate_proj", "model.layers.49.mlp.experts.110.gate_proj", "model.layers.49.mlp.experts.111.gate_proj", "model.layers.49.mlp.experts.112.gate_proj", "model.layers.49.mlp.experts.113.gate_proj", "model.layers.49.mlp.experts.114.gate_proj", "model.layers.49.mlp.experts.115.gate_proj", "model.layers.49.mlp.experts.116.gate_proj", "model.layers.49.mlp.experts.117.gate_proj", "model.layers.49.mlp.experts.118.gate_proj", "model.layers.49.mlp.experts.119.gate_proj", "model.layers.49.mlp.experts.120.gate_proj", "model.layers.49.mlp.experts.121.gate_proj", "model.layers.49.mlp.experts.122.gate_proj", "model.layers.49.mlp.experts.123.gate_proj", "model.layers.49.mlp.experts.124.gate_proj", "model.layers.49.mlp.experts.125.gate_proj", "model.layers.49.mlp.experts.126.gate_proj", "model.layers.49.mlp.experts.127.gate_proj", "model.layers.49.mlp.experts.128.gate_proj", "model.layers.49.mlp.experts.129.gate_proj", "model.layers.49.mlp.experts.130.gate_proj", "model.layers.49.mlp.experts.131.gate_proj", "model.layers.49.mlp.experts.132.gate_proj", "model.layers.49.mlp.experts.133.gate_proj", "model.layers.49.mlp.experts.134.gate_proj", "model.layers.49.mlp.experts.135.gate_proj", "model.layers.49.mlp.experts.136.gate_proj", "model.layers.49.mlp.experts.137.gate_proj", "model.layers.49.mlp.experts.138.gate_proj", "model.layers.49.mlp.experts.139.gate_proj", "model.layers.49.mlp.experts.140.gate_proj", "model.layers.49.mlp.experts.141.gate_proj", "model.layers.49.mlp.experts.142.gate_proj", "model.layers.49.mlp.experts.143.gate_proj", "model.layers.49.mlp.experts.144.gate_proj", "model.layers.49.mlp.experts.145.gate_proj", "model.layers.49.mlp.experts.146.gate_proj", "model.layers.49.mlp.experts.147.gate_proj", "model.layers.49.mlp.experts.148.gate_proj", "model.layers.49.mlp.experts.149.gate_proj", "model.layers.49.mlp.experts.150.gate_proj", "model.layers.49.mlp.experts.151.gate_proj", "model.layers.49.mlp.experts.152.gate_proj", "model.layers.49.mlp.experts.153.gate_proj", "model.layers.49.mlp.experts.154.gate_proj", "model.layers.49.mlp.experts.155.gate_proj", "model.layers.49.mlp.experts.156.gate_proj", "model.layers.49.mlp.experts.157.gate_proj", "model.layers.49.mlp.experts.158.gate_proj", "model.layers.49.mlp.experts.159.gate_proj", "model.layers.49.mlp.experts.0.up_proj", "model.layers.49.mlp.experts.1.up_proj", "model.layers.49.mlp.experts.2.up_proj", "model.layers.49.mlp.experts.3.up_proj", "model.layers.49.mlp.experts.4.up_proj", "model.layers.49.mlp.experts.5.up_proj", "model.layers.49.mlp.experts.6.up_proj", "model.layers.49.mlp.experts.7.up_proj", "model.layers.49.mlp.experts.8.up_proj", "model.layers.49.mlp.experts.9.up_proj", "model.layers.49.mlp.experts.10.up_proj", "model.layers.49.mlp.experts.11.up_proj", "model.layers.49.mlp.experts.12.up_proj", "model.layers.49.mlp.experts.13.up_proj", "model.layers.49.mlp.experts.14.up_proj", "model.layers.49.mlp.experts.15.up_proj", "model.layers.49.mlp.experts.16.up_proj", "model.layers.49.mlp.experts.17.up_proj", "model.layers.49.mlp.experts.18.up_proj", "model.layers.49.mlp.experts.19.up_proj", "model.layers.49.mlp.experts.20.up_proj", "model.layers.49.mlp.experts.21.up_proj", "model.layers.49.mlp.experts.22.up_proj", "model.layers.49.mlp.experts.23.up_proj", "model.layers.49.mlp.experts.24.up_proj", "model.layers.49.mlp.experts.25.up_proj", "model.layers.49.mlp.experts.26.up_proj", "model.layers.49.mlp.experts.27.up_proj", "model.layers.49.mlp.experts.28.up_proj", "model.layers.49.mlp.experts.29.up_proj", "model.layers.49.mlp.experts.30.up_proj", "model.layers.49.mlp.experts.31.up_proj", "model.layers.49.mlp.experts.32.up_proj", "model.layers.49.mlp.experts.33.up_proj", "model.layers.49.mlp.experts.34.up_proj", "model.layers.49.mlp.experts.35.up_proj", "model.layers.49.mlp.experts.36.up_proj", "model.layers.49.mlp.experts.37.up_proj", "model.layers.49.mlp.experts.38.up_proj", "model.layers.49.mlp.experts.39.up_proj", "model.layers.49.mlp.experts.40.up_proj", "model.layers.49.mlp.experts.41.up_proj", "model.layers.49.mlp.experts.42.up_proj", "model.layers.49.mlp.experts.43.up_proj", "model.layers.49.mlp.experts.44.up_proj", "model.layers.49.mlp.experts.45.up_proj", "model.layers.49.mlp.experts.46.up_proj", "model.layers.49.mlp.experts.47.up_proj", "model.layers.49.mlp.experts.48.up_proj", "model.layers.49.mlp.experts.49.up_proj", "model.layers.49.mlp.experts.50.up_proj", "model.layers.49.mlp.experts.51.up_proj", "model.layers.49.mlp.experts.52.up_proj", "model.layers.49.mlp.experts.53.up_proj", "model.layers.49.mlp.experts.54.up_proj", "model.layers.49.mlp.experts.55.up_proj", "model.layers.49.mlp.experts.56.up_proj", "model.layers.49.mlp.experts.57.up_proj", "model.layers.49.mlp.experts.58.up_proj", "model.layers.49.mlp.experts.59.up_proj", "model.layers.49.mlp.experts.60.up_proj", "model.layers.49.mlp.experts.61.up_proj", "model.layers.49.mlp.experts.62.up_proj", "model.layers.49.mlp.experts.63.up_proj", "model.layers.49.mlp.experts.64.up_proj", "model.layers.49.mlp.experts.65.up_proj", "model.layers.49.mlp.experts.66.up_proj", "model.layers.49.mlp.experts.67.up_proj", "model.layers.49.mlp.experts.68.up_proj", "model.layers.49.mlp.experts.69.up_proj", "model.layers.49.mlp.experts.70.up_proj", "model.layers.49.mlp.experts.71.up_proj", "model.layers.49.mlp.experts.72.up_proj", "model.layers.49.mlp.experts.73.up_proj", "model.layers.49.mlp.experts.74.up_proj", "model.layers.49.mlp.experts.75.up_proj", "model.layers.49.mlp.experts.76.up_proj", "model.layers.49.mlp.experts.77.up_proj", "model.layers.49.mlp.experts.78.up_proj", "model.layers.49.mlp.experts.79.up_proj", "model.layers.49.mlp.experts.80.up_proj", "model.layers.49.mlp.experts.81.up_proj", "model.layers.49.mlp.experts.82.up_proj", "model.layers.49.mlp.experts.83.up_proj", "model.layers.49.mlp.experts.84.up_proj", "model.layers.49.mlp.experts.85.up_proj", "model.layers.49.mlp.experts.86.up_proj", "model.layers.49.mlp.experts.87.up_proj", "model.layers.49.mlp.experts.88.up_proj", "model.layers.49.mlp.experts.89.up_proj", "model.layers.49.mlp.experts.90.up_proj", "model.layers.49.mlp.experts.91.up_proj", "model.layers.49.mlp.experts.92.up_proj", "model.layers.49.mlp.experts.93.up_proj", "model.layers.49.mlp.experts.94.up_proj", "model.layers.49.mlp.experts.95.up_proj", "model.layers.49.mlp.experts.96.up_proj", "model.layers.49.mlp.experts.97.up_proj", "model.layers.49.mlp.experts.98.up_proj", "model.layers.49.mlp.experts.99.up_proj", "model.layers.49.mlp.experts.100.up_proj", "model.layers.49.mlp.experts.101.up_proj", "model.layers.49.mlp.experts.102.up_proj", "model.layers.49.mlp.experts.103.up_proj", "model.layers.49.mlp.experts.104.up_proj", "model.layers.49.mlp.experts.105.up_proj", "model.layers.49.mlp.experts.106.up_proj", "model.layers.49.mlp.experts.107.up_proj", "model.layers.49.mlp.experts.108.up_proj", "model.layers.49.mlp.experts.109.up_proj", "model.layers.49.mlp.experts.110.up_proj", "model.layers.49.mlp.experts.111.up_proj", "model.layers.49.mlp.experts.112.up_proj", "model.layers.49.mlp.experts.113.up_proj", "model.layers.49.mlp.experts.114.up_proj", "model.layers.49.mlp.experts.115.up_proj", "model.layers.49.mlp.experts.116.up_proj", "model.layers.49.mlp.experts.117.up_proj", "model.layers.49.mlp.experts.118.up_proj", "model.layers.49.mlp.experts.119.up_proj", "model.layers.49.mlp.experts.120.up_proj", "model.layers.49.mlp.experts.121.up_proj", "model.layers.49.mlp.experts.122.up_proj", "model.layers.49.mlp.experts.123.up_proj", "model.layers.49.mlp.experts.124.up_proj", "model.layers.49.mlp.experts.125.up_proj", "model.layers.49.mlp.experts.126.up_proj", "model.layers.49.mlp.experts.127.up_proj", "model.layers.49.mlp.experts.128.up_proj", "model.layers.49.mlp.experts.129.up_proj", "model.layers.49.mlp.experts.130.up_proj", "model.layers.49.mlp.experts.131.up_proj", "model.layers.49.mlp.experts.132.up_proj", "model.layers.49.mlp.experts.133.up_proj", "model.layers.49.mlp.experts.134.up_proj", "model.layers.49.mlp.experts.135.up_proj", "model.layers.49.mlp.experts.136.up_proj", "model.layers.49.mlp.experts.137.up_proj", "model.layers.49.mlp.experts.138.up_proj", "model.layers.49.mlp.experts.139.up_proj", "model.layers.49.mlp.experts.140.up_proj", "model.layers.49.mlp.experts.141.up_proj", "model.layers.49.mlp.experts.142.up_proj", "model.layers.49.mlp.experts.143.up_proj", "model.layers.49.mlp.experts.144.up_proj", "model.layers.49.mlp.experts.145.up_proj", "model.layers.49.mlp.experts.146.up_proj", "model.layers.49.mlp.experts.147.up_proj", "model.layers.49.mlp.experts.148.up_proj", "model.layers.49.mlp.experts.149.up_proj", "model.layers.49.mlp.experts.150.up_proj", "model.layers.49.mlp.experts.151.up_proj", "model.layers.49.mlp.experts.152.up_proj", "model.layers.49.mlp.experts.153.up_proj", "model.layers.49.mlp.experts.154.up_proj", "model.layers.49.mlp.experts.155.up_proj", "model.layers.49.mlp.experts.156.up_proj", "model.layers.49.mlp.experts.157.up_proj", "model.layers.49.mlp.experts.158.up_proj", "model.layers.49.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0009696319699287748, "dbits": 2516582400 } ] }, { "idx": 296, "layers": [ "model.layers.49.mlp.experts.0.down_proj", "model.layers.49.mlp.experts.1.down_proj", "model.layers.49.mlp.experts.2.down_proj", "model.layers.49.mlp.experts.3.down_proj", "model.layers.49.mlp.experts.4.down_proj", "model.layers.49.mlp.experts.5.down_proj", "model.layers.49.mlp.experts.6.down_proj", "model.layers.49.mlp.experts.7.down_proj", "model.layers.49.mlp.experts.8.down_proj", "model.layers.49.mlp.experts.9.down_proj", "model.layers.49.mlp.experts.10.down_proj", "model.layers.49.mlp.experts.11.down_proj", "model.layers.49.mlp.experts.12.down_proj", "model.layers.49.mlp.experts.13.down_proj", "model.layers.49.mlp.experts.14.down_proj", "model.layers.49.mlp.experts.15.down_proj", "model.layers.49.mlp.experts.16.down_proj", "model.layers.49.mlp.experts.17.down_proj", "model.layers.49.mlp.experts.18.down_proj", "model.layers.49.mlp.experts.19.down_proj", "model.layers.49.mlp.experts.20.down_proj", "model.layers.49.mlp.experts.21.down_proj", "model.layers.49.mlp.experts.22.down_proj", "model.layers.49.mlp.experts.23.down_proj", "model.layers.49.mlp.experts.24.down_proj", "model.layers.49.mlp.experts.25.down_proj", "model.layers.49.mlp.experts.26.down_proj", "model.layers.49.mlp.experts.27.down_proj", "model.layers.49.mlp.experts.28.down_proj", "model.layers.49.mlp.experts.29.down_proj", "model.layers.49.mlp.experts.30.down_proj", "model.layers.49.mlp.experts.31.down_proj", "model.layers.49.mlp.experts.32.down_proj", "model.layers.49.mlp.experts.33.down_proj", "model.layers.49.mlp.experts.34.down_proj", "model.layers.49.mlp.experts.35.down_proj", "model.layers.49.mlp.experts.36.down_proj", "model.layers.49.mlp.experts.37.down_proj", "model.layers.49.mlp.experts.38.down_proj", "model.layers.49.mlp.experts.39.down_proj", "model.layers.49.mlp.experts.40.down_proj", "model.layers.49.mlp.experts.41.down_proj", "model.layers.49.mlp.experts.42.down_proj", "model.layers.49.mlp.experts.43.down_proj", "model.layers.49.mlp.experts.44.down_proj", "model.layers.49.mlp.experts.45.down_proj", "model.layers.49.mlp.experts.46.down_proj", "model.layers.49.mlp.experts.47.down_proj", "model.layers.49.mlp.experts.48.down_proj", "model.layers.49.mlp.experts.49.down_proj", "model.layers.49.mlp.experts.50.down_proj", "model.layers.49.mlp.experts.51.down_proj", "model.layers.49.mlp.experts.52.down_proj", "model.layers.49.mlp.experts.53.down_proj", "model.layers.49.mlp.experts.54.down_proj", "model.layers.49.mlp.experts.55.down_proj", "model.layers.49.mlp.experts.56.down_proj", "model.layers.49.mlp.experts.57.down_proj", "model.layers.49.mlp.experts.58.down_proj", "model.layers.49.mlp.experts.59.down_proj", "model.layers.49.mlp.experts.60.down_proj", "model.layers.49.mlp.experts.61.down_proj", "model.layers.49.mlp.experts.62.down_proj", "model.layers.49.mlp.experts.63.down_proj", "model.layers.49.mlp.experts.64.down_proj", "model.layers.49.mlp.experts.65.down_proj", "model.layers.49.mlp.experts.66.down_proj", "model.layers.49.mlp.experts.67.down_proj", "model.layers.49.mlp.experts.68.down_proj", "model.layers.49.mlp.experts.69.down_proj", "model.layers.49.mlp.experts.70.down_proj", "model.layers.49.mlp.experts.71.down_proj", "model.layers.49.mlp.experts.72.down_proj", "model.layers.49.mlp.experts.73.down_proj", "model.layers.49.mlp.experts.74.down_proj", "model.layers.49.mlp.experts.75.down_proj", "model.layers.49.mlp.experts.76.down_proj", "model.layers.49.mlp.experts.77.down_proj", "model.layers.49.mlp.experts.78.down_proj", "model.layers.49.mlp.experts.79.down_proj", "model.layers.49.mlp.experts.80.down_proj", "model.layers.49.mlp.experts.81.down_proj", "model.layers.49.mlp.experts.82.down_proj", "model.layers.49.mlp.experts.83.down_proj", "model.layers.49.mlp.experts.84.down_proj", "model.layers.49.mlp.experts.85.down_proj", "model.layers.49.mlp.experts.86.down_proj", "model.layers.49.mlp.experts.87.down_proj", "model.layers.49.mlp.experts.88.down_proj", "model.layers.49.mlp.experts.89.down_proj", "model.layers.49.mlp.experts.90.down_proj", "model.layers.49.mlp.experts.91.down_proj", "model.layers.49.mlp.experts.92.down_proj", "model.layers.49.mlp.experts.93.down_proj", "model.layers.49.mlp.experts.94.down_proj", "model.layers.49.mlp.experts.95.down_proj", "model.layers.49.mlp.experts.96.down_proj", "model.layers.49.mlp.experts.97.down_proj", "model.layers.49.mlp.experts.98.down_proj", "model.layers.49.mlp.experts.99.down_proj", "model.layers.49.mlp.experts.100.down_proj", "model.layers.49.mlp.experts.101.down_proj", "model.layers.49.mlp.experts.102.down_proj", "model.layers.49.mlp.experts.103.down_proj", "model.layers.49.mlp.experts.104.down_proj", "model.layers.49.mlp.experts.105.down_proj", "model.layers.49.mlp.experts.106.down_proj", "model.layers.49.mlp.experts.107.down_proj", "model.layers.49.mlp.experts.108.down_proj", "model.layers.49.mlp.experts.109.down_proj", "model.layers.49.mlp.experts.110.down_proj", "model.layers.49.mlp.experts.111.down_proj", "model.layers.49.mlp.experts.112.down_proj", "model.layers.49.mlp.experts.113.down_proj", "model.layers.49.mlp.experts.114.down_proj", "model.layers.49.mlp.experts.115.down_proj", "model.layers.49.mlp.experts.116.down_proj", "model.layers.49.mlp.experts.117.down_proj", "model.layers.49.mlp.experts.118.down_proj", "model.layers.49.mlp.experts.119.down_proj", "model.layers.49.mlp.experts.120.down_proj", "model.layers.49.mlp.experts.121.down_proj", "model.layers.49.mlp.experts.122.down_proj", "model.layers.49.mlp.experts.123.down_proj", "model.layers.49.mlp.experts.124.down_proj", "model.layers.49.mlp.experts.125.down_proj", "model.layers.49.mlp.experts.126.down_proj", "model.layers.49.mlp.experts.127.down_proj", "model.layers.49.mlp.experts.128.down_proj", "model.layers.49.mlp.experts.129.down_proj", "model.layers.49.mlp.experts.130.down_proj", "model.layers.49.mlp.experts.131.down_proj", "model.layers.49.mlp.experts.132.down_proj", "model.layers.49.mlp.experts.133.down_proj", "model.layers.49.mlp.experts.134.down_proj", "model.layers.49.mlp.experts.135.down_proj", "model.layers.49.mlp.experts.136.down_proj", "model.layers.49.mlp.experts.137.down_proj", "model.layers.49.mlp.experts.138.down_proj", "model.layers.49.mlp.experts.139.down_proj", "model.layers.49.mlp.experts.140.down_proj", "model.layers.49.mlp.experts.141.down_proj", "model.layers.49.mlp.experts.142.down_proj", "model.layers.49.mlp.experts.143.down_proj", "model.layers.49.mlp.experts.144.down_proj", "model.layers.49.mlp.experts.145.down_proj", "model.layers.49.mlp.experts.146.down_proj", "model.layers.49.mlp.experts.147.down_proj", "model.layers.49.mlp.experts.148.down_proj", "model.layers.49.mlp.experts.149.down_proj", "model.layers.49.mlp.experts.150.down_proj", "model.layers.49.mlp.experts.151.down_proj", "model.layers.49.mlp.experts.152.down_proj", "model.layers.49.mlp.experts.153.down_proj", "model.layers.49.mlp.experts.154.down_proj", "model.layers.49.mlp.experts.155.down_proj", "model.layers.49.mlp.experts.156.down_proj", "model.layers.49.mlp.experts.157.down_proj", "model.layers.49.mlp.experts.158.down_proj", "model.layers.49.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.001005881279706966, "dbits": 1258291200 } ] }, { "idx": 297, "layers": [ "model.layers.50.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00023608207702635609, "dbits": 62914560 } ] }, { "idx": 298, "layers": [ "model.layers.50.self_attn.k_proj", "model.layers.50.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0010021418333053256, "dbits": 10485760 } ] }, { "idx": 299, "layers": [ "model.layers.50.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0006976887583732272, "dbits": 62914560 } ] }, { "idx": 300, "layers": [ "model.layers.50.mlp.shared_experts.gate_proj", "model.layers.50.mlp.shared_experts.up_proj", "model.layers.50.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0014011815190315136, "dbits": 23592960 } ] }, { "idx": 301, "layers": [ "model.layers.50.mlp.experts.0.gate_proj", "model.layers.50.mlp.experts.1.gate_proj", "model.layers.50.mlp.experts.2.gate_proj", "model.layers.50.mlp.experts.3.gate_proj", "model.layers.50.mlp.experts.4.gate_proj", "model.layers.50.mlp.experts.5.gate_proj", "model.layers.50.mlp.experts.6.gate_proj", "model.layers.50.mlp.experts.7.gate_proj", "model.layers.50.mlp.experts.8.gate_proj", "model.layers.50.mlp.experts.9.gate_proj", "model.layers.50.mlp.experts.10.gate_proj", "model.layers.50.mlp.experts.11.gate_proj", "model.layers.50.mlp.experts.12.gate_proj", "model.layers.50.mlp.experts.13.gate_proj", "model.layers.50.mlp.experts.14.gate_proj", "model.layers.50.mlp.experts.15.gate_proj", "model.layers.50.mlp.experts.16.gate_proj", "model.layers.50.mlp.experts.17.gate_proj", "model.layers.50.mlp.experts.18.gate_proj", "model.layers.50.mlp.experts.19.gate_proj", "model.layers.50.mlp.experts.20.gate_proj", "model.layers.50.mlp.experts.21.gate_proj", "model.layers.50.mlp.experts.22.gate_proj", "model.layers.50.mlp.experts.23.gate_proj", "model.layers.50.mlp.experts.24.gate_proj", "model.layers.50.mlp.experts.25.gate_proj", "model.layers.50.mlp.experts.26.gate_proj", "model.layers.50.mlp.experts.27.gate_proj", "model.layers.50.mlp.experts.28.gate_proj", "model.layers.50.mlp.experts.29.gate_proj", "model.layers.50.mlp.experts.30.gate_proj", "model.layers.50.mlp.experts.31.gate_proj", "model.layers.50.mlp.experts.32.gate_proj", "model.layers.50.mlp.experts.33.gate_proj", "model.layers.50.mlp.experts.34.gate_proj", "model.layers.50.mlp.experts.35.gate_proj", "model.layers.50.mlp.experts.36.gate_proj", "model.layers.50.mlp.experts.37.gate_proj", "model.layers.50.mlp.experts.38.gate_proj", "model.layers.50.mlp.experts.39.gate_proj", "model.layers.50.mlp.experts.40.gate_proj", "model.layers.50.mlp.experts.41.gate_proj", "model.layers.50.mlp.experts.42.gate_proj", "model.layers.50.mlp.experts.43.gate_proj", "model.layers.50.mlp.experts.44.gate_proj", "model.layers.50.mlp.experts.45.gate_proj", "model.layers.50.mlp.experts.46.gate_proj", "model.layers.50.mlp.experts.47.gate_proj", "model.layers.50.mlp.experts.48.gate_proj", "model.layers.50.mlp.experts.49.gate_proj", "model.layers.50.mlp.experts.50.gate_proj", "model.layers.50.mlp.experts.51.gate_proj", "model.layers.50.mlp.experts.52.gate_proj", "model.layers.50.mlp.experts.53.gate_proj", "model.layers.50.mlp.experts.54.gate_proj", "model.layers.50.mlp.experts.55.gate_proj", "model.layers.50.mlp.experts.56.gate_proj", "model.layers.50.mlp.experts.57.gate_proj", "model.layers.50.mlp.experts.58.gate_proj", "model.layers.50.mlp.experts.59.gate_proj", "model.layers.50.mlp.experts.60.gate_proj", "model.layers.50.mlp.experts.61.gate_proj", "model.layers.50.mlp.experts.62.gate_proj", "model.layers.50.mlp.experts.63.gate_proj", "model.layers.50.mlp.experts.64.gate_proj", "model.layers.50.mlp.experts.65.gate_proj", "model.layers.50.mlp.experts.66.gate_proj", "model.layers.50.mlp.experts.67.gate_proj", "model.layers.50.mlp.experts.68.gate_proj", "model.layers.50.mlp.experts.69.gate_proj", "model.layers.50.mlp.experts.70.gate_proj", "model.layers.50.mlp.experts.71.gate_proj", "model.layers.50.mlp.experts.72.gate_proj", "model.layers.50.mlp.experts.73.gate_proj", "model.layers.50.mlp.experts.74.gate_proj", "model.layers.50.mlp.experts.75.gate_proj", "model.layers.50.mlp.experts.76.gate_proj", "model.layers.50.mlp.experts.77.gate_proj", "model.layers.50.mlp.experts.78.gate_proj", "model.layers.50.mlp.experts.79.gate_proj", "model.layers.50.mlp.experts.80.gate_proj", "model.layers.50.mlp.experts.81.gate_proj", "model.layers.50.mlp.experts.82.gate_proj", "model.layers.50.mlp.experts.83.gate_proj", "model.layers.50.mlp.experts.84.gate_proj", "model.layers.50.mlp.experts.85.gate_proj", "model.layers.50.mlp.experts.86.gate_proj", "model.layers.50.mlp.experts.87.gate_proj", "model.layers.50.mlp.experts.88.gate_proj", "model.layers.50.mlp.experts.89.gate_proj", "model.layers.50.mlp.experts.90.gate_proj", "model.layers.50.mlp.experts.91.gate_proj", "model.layers.50.mlp.experts.92.gate_proj", "model.layers.50.mlp.experts.93.gate_proj", "model.layers.50.mlp.experts.94.gate_proj", "model.layers.50.mlp.experts.95.gate_proj", "model.layers.50.mlp.experts.96.gate_proj", "model.layers.50.mlp.experts.97.gate_proj", "model.layers.50.mlp.experts.98.gate_proj", "model.layers.50.mlp.experts.99.gate_proj", "model.layers.50.mlp.experts.100.gate_proj", "model.layers.50.mlp.experts.101.gate_proj", "model.layers.50.mlp.experts.102.gate_proj", "model.layers.50.mlp.experts.103.gate_proj", "model.layers.50.mlp.experts.104.gate_proj", "model.layers.50.mlp.experts.105.gate_proj", "model.layers.50.mlp.experts.106.gate_proj", "model.layers.50.mlp.experts.107.gate_proj", "model.layers.50.mlp.experts.108.gate_proj", "model.layers.50.mlp.experts.109.gate_proj", "model.layers.50.mlp.experts.110.gate_proj", "model.layers.50.mlp.experts.111.gate_proj", "model.layers.50.mlp.experts.112.gate_proj", "model.layers.50.mlp.experts.113.gate_proj", "model.layers.50.mlp.experts.114.gate_proj", "model.layers.50.mlp.experts.115.gate_proj", "model.layers.50.mlp.experts.116.gate_proj", "model.layers.50.mlp.experts.117.gate_proj", "model.layers.50.mlp.experts.118.gate_proj", "model.layers.50.mlp.experts.119.gate_proj", "model.layers.50.mlp.experts.120.gate_proj", "model.layers.50.mlp.experts.121.gate_proj", "model.layers.50.mlp.experts.122.gate_proj", "model.layers.50.mlp.experts.123.gate_proj", "model.layers.50.mlp.experts.124.gate_proj", "model.layers.50.mlp.experts.125.gate_proj", "model.layers.50.mlp.experts.126.gate_proj", "model.layers.50.mlp.experts.127.gate_proj", "model.layers.50.mlp.experts.128.gate_proj", "model.layers.50.mlp.experts.129.gate_proj", "model.layers.50.mlp.experts.130.gate_proj", "model.layers.50.mlp.experts.131.gate_proj", "model.layers.50.mlp.experts.132.gate_proj", "model.layers.50.mlp.experts.133.gate_proj", "model.layers.50.mlp.experts.134.gate_proj", "model.layers.50.mlp.experts.135.gate_proj", "model.layers.50.mlp.experts.136.gate_proj", "model.layers.50.mlp.experts.137.gate_proj", "model.layers.50.mlp.experts.138.gate_proj", "model.layers.50.mlp.experts.139.gate_proj", "model.layers.50.mlp.experts.140.gate_proj", "model.layers.50.mlp.experts.141.gate_proj", "model.layers.50.mlp.experts.142.gate_proj", "model.layers.50.mlp.experts.143.gate_proj", "model.layers.50.mlp.experts.144.gate_proj", "model.layers.50.mlp.experts.145.gate_proj", "model.layers.50.mlp.experts.146.gate_proj", "model.layers.50.mlp.experts.147.gate_proj", "model.layers.50.mlp.experts.148.gate_proj", "model.layers.50.mlp.experts.149.gate_proj", "model.layers.50.mlp.experts.150.gate_proj", "model.layers.50.mlp.experts.151.gate_proj", "model.layers.50.mlp.experts.152.gate_proj", "model.layers.50.mlp.experts.153.gate_proj", "model.layers.50.mlp.experts.154.gate_proj", "model.layers.50.mlp.experts.155.gate_proj", "model.layers.50.mlp.experts.156.gate_proj", "model.layers.50.mlp.experts.157.gate_proj", "model.layers.50.mlp.experts.158.gate_proj", "model.layers.50.mlp.experts.159.gate_proj", "model.layers.50.mlp.experts.0.up_proj", "model.layers.50.mlp.experts.1.up_proj", "model.layers.50.mlp.experts.2.up_proj", "model.layers.50.mlp.experts.3.up_proj", "model.layers.50.mlp.experts.4.up_proj", "model.layers.50.mlp.experts.5.up_proj", "model.layers.50.mlp.experts.6.up_proj", "model.layers.50.mlp.experts.7.up_proj", "model.layers.50.mlp.experts.8.up_proj", "model.layers.50.mlp.experts.9.up_proj", "model.layers.50.mlp.experts.10.up_proj", "model.layers.50.mlp.experts.11.up_proj", "model.layers.50.mlp.experts.12.up_proj", "model.layers.50.mlp.experts.13.up_proj", "model.layers.50.mlp.experts.14.up_proj", "model.layers.50.mlp.experts.15.up_proj", "model.layers.50.mlp.experts.16.up_proj", "model.layers.50.mlp.experts.17.up_proj", "model.layers.50.mlp.experts.18.up_proj", "model.layers.50.mlp.experts.19.up_proj", "model.layers.50.mlp.experts.20.up_proj", "model.layers.50.mlp.experts.21.up_proj", "model.layers.50.mlp.experts.22.up_proj", "model.layers.50.mlp.experts.23.up_proj", "model.layers.50.mlp.experts.24.up_proj", "model.layers.50.mlp.experts.25.up_proj", "model.layers.50.mlp.experts.26.up_proj", "model.layers.50.mlp.experts.27.up_proj", "model.layers.50.mlp.experts.28.up_proj", "model.layers.50.mlp.experts.29.up_proj", "model.layers.50.mlp.experts.30.up_proj", "model.layers.50.mlp.experts.31.up_proj", "model.layers.50.mlp.experts.32.up_proj", "model.layers.50.mlp.experts.33.up_proj", "model.layers.50.mlp.experts.34.up_proj", "model.layers.50.mlp.experts.35.up_proj", "model.layers.50.mlp.experts.36.up_proj", "model.layers.50.mlp.experts.37.up_proj", "model.layers.50.mlp.experts.38.up_proj", "model.layers.50.mlp.experts.39.up_proj", "model.layers.50.mlp.experts.40.up_proj", "model.layers.50.mlp.experts.41.up_proj", "model.layers.50.mlp.experts.42.up_proj", "model.layers.50.mlp.experts.43.up_proj", "model.layers.50.mlp.experts.44.up_proj", "model.layers.50.mlp.experts.45.up_proj", "model.layers.50.mlp.experts.46.up_proj", "model.layers.50.mlp.experts.47.up_proj", "model.layers.50.mlp.experts.48.up_proj", "model.layers.50.mlp.experts.49.up_proj", "model.layers.50.mlp.experts.50.up_proj", "model.layers.50.mlp.experts.51.up_proj", "model.layers.50.mlp.experts.52.up_proj", "model.layers.50.mlp.experts.53.up_proj", "model.layers.50.mlp.experts.54.up_proj", "model.layers.50.mlp.experts.55.up_proj", "model.layers.50.mlp.experts.56.up_proj", "model.layers.50.mlp.experts.57.up_proj", "model.layers.50.mlp.experts.58.up_proj", "model.layers.50.mlp.experts.59.up_proj", "model.layers.50.mlp.experts.60.up_proj", "model.layers.50.mlp.experts.61.up_proj", "model.layers.50.mlp.experts.62.up_proj", "model.layers.50.mlp.experts.63.up_proj", "model.layers.50.mlp.experts.64.up_proj", "model.layers.50.mlp.experts.65.up_proj", "model.layers.50.mlp.experts.66.up_proj", "model.layers.50.mlp.experts.67.up_proj", "model.layers.50.mlp.experts.68.up_proj", "model.layers.50.mlp.experts.69.up_proj", "model.layers.50.mlp.experts.70.up_proj", "model.layers.50.mlp.experts.71.up_proj", "model.layers.50.mlp.experts.72.up_proj", "model.layers.50.mlp.experts.73.up_proj", "model.layers.50.mlp.experts.74.up_proj", "model.layers.50.mlp.experts.75.up_proj", "model.layers.50.mlp.experts.76.up_proj", "model.layers.50.mlp.experts.77.up_proj", "model.layers.50.mlp.experts.78.up_proj", "model.layers.50.mlp.experts.79.up_proj", "model.layers.50.mlp.experts.80.up_proj", "model.layers.50.mlp.experts.81.up_proj", "model.layers.50.mlp.experts.82.up_proj", "model.layers.50.mlp.experts.83.up_proj", "model.layers.50.mlp.experts.84.up_proj", "model.layers.50.mlp.experts.85.up_proj", "model.layers.50.mlp.experts.86.up_proj", "model.layers.50.mlp.experts.87.up_proj", "model.layers.50.mlp.experts.88.up_proj", "model.layers.50.mlp.experts.89.up_proj", "model.layers.50.mlp.experts.90.up_proj", "model.layers.50.mlp.experts.91.up_proj", "model.layers.50.mlp.experts.92.up_proj", "model.layers.50.mlp.experts.93.up_proj", "model.layers.50.mlp.experts.94.up_proj", "model.layers.50.mlp.experts.95.up_proj", "model.layers.50.mlp.experts.96.up_proj", "model.layers.50.mlp.experts.97.up_proj", "model.layers.50.mlp.experts.98.up_proj", "model.layers.50.mlp.experts.99.up_proj", "model.layers.50.mlp.experts.100.up_proj", "model.layers.50.mlp.experts.101.up_proj", "model.layers.50.mlp.experts.102.up_proj", "model.layers.50.mlp.experts.103.up_proj", "model.layers.50.mlp.experts.104.up_proj", "model.layers.50.mlp.experts.105.up_proj", "model.layers.50.mlp.experts.106.up_proj", "model.layers.50.mlp.experts.107.up_proj", "model.layers.50.mlp.experts.108.up_proj", "model.layers.50.mlp.experts.109.up_proj", "model.layers.50.mlp.experts.110.up_proj", "model.layers.50.mlp.experts.111.up_proj", "model.layers.50.mlp.experts.112.up_proj", "model.layers.50.mlp.experts.113.up_proj", "model.layers.50.mlp.experts.114.up_proj", "model.layers.50.mlp.experts.115.up_proj", "model.layers.50.mlp.experts.116.up_proj", "model.layers.50.mlp.experts.117.up_proj", "model.layers.50.mlp.experts.118.up_proj", "model.layers.50.mlp.experts.119.up_proj", "model.layers.50.mlp.experts.120.up_proj", "model.layers.50.mlp.experts.121.up_proj", "model.layers.50.mlp.experts.122.up_proj", "model.layers.50.mlp.experts.123.up_proj", "model.layers.50.mlp.experts.124.up_proj", "model.layers.50.mlp.experts.125.up_proj", "model.layers.50.mlp.experts.126.up_proj", "model.layers.50.mlp.experts.127.up_proj", "model.layers.50.mlp.experts.128.up_proj", "model.layers.50.mlp.experts.129.up_proj", "model.layers.50.mlp.experts.130.up_proj", "model.layers.50.mlp.experts.131.up_proj", "model.layers.50.mlp.experts.132.up_proj", "model.layers.50.mlp.experts.133.up_proj", "model.layers.50.mlp.experts.134.up_proj", "model.layers.50.mlp.experts.135.up_proj", "model.layers.50.mlp.experts.136.up_proj", "model.layers.50.mlp.experts.137.up_proj", "model.layers.50.mlp.experts.138.up_proj", "model.layers.50.mlp.experts.139.up_proj", "model.layers.50.mlp.experts.140.up_proj", "model.layers.50.mlp.experts.141.up_proj", "model.layers.50.mlp.experts.142.up_proj", "model.layers.50.mlp.experts.143.up_proj", "model.layers.50.mlp.experts.144.up_proj", "model.layers.50.mlp.experts.145.up_proj", "model.layers.50.mlp.experts.146.up_proj", "model.layers.50.mlp.experts.147.up_proj", "model.layers.50.mlp.experts.148.up_proj", "model.layers.50.mlp.experts.149.up_proj", "model.layers.50.mlp.experts.150.up_proj", "model.layers.50.mlp.experts.151.up_proj", "model.layers.50.mlp.experts.152.up_proj", "model.layers.50.mlp.experts.153.up_proj", "model.layers.50.mlp.experts.154.up_proj", "model.layers.50.mlp.experts.155.up_proj", "model.layers.50.mlp.experts.156.up_proj", "model.layers.50.mlp.experts.157.up_proj", "model.layers.50.mlp.experts.158.up_proj", "model.layers.50.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0012435689568519592, "dbits": 2516582400 } ] }, { "idx": 302, "layers": [ "model.layers.50.mlp.experts.0.down_proj", "model.layers.50.mlp.experts.1.down_proj", "model.layers.50.mlp.experts.2.down_proj", "model.layers.50.mlp.experts.3.down_proj", "model.layers.50.mlp.experts.4.down_proj", "model.layers.50.mlp.experts.5.down_proj", "model.layers.50.mlp.experts.6.down_proj", "model.layers.50.mlp.experts.7.down_proj", "model.layers.50.mlp.experts.8.down_proj", "model.layers.50.mlp.experts.9.down_proj", "model.layers.50.mlp.experts.10.down_proj", "model.layers.50.mlp.experts.11.down_proj", "model.layers.50.mlp.experts.12.down_proj", "model.layers.50.mlp.experts.13.down_proj", "model.layers.50.mlp.experts.14.down_proj", "model.layers.50.mlp.experts.15.down_proj", "model.layers.50.mlp.experts.16.down_proj", "model.layers.50.mlp.experts.17.down_proj", "model.layers.50.mlp.experts.18.down_proj", "model.layers.50.mlp.experts.19.down_proj", "model.layers.50.mlp.experts.20.down_proj", "model.layers.50.mlp.experts.21.down_proj", "model.layers.50.mlp.experts.22.down_proj", "model.layers.50.mlp.experts.23.down_proj", "model.layers.50.mlp.experts.24.down_proj", "model.layers.50.mlp.experts.25.down_proj", "model.layers.50.mlp.experts.26.down_proj", "model.layers.50.mlp.experts.27.down_proj", "model.layers.50.mlp.experts.28.down_proj", "model.layers.50.mlp.experts.29.down_proj", "model.layers.50.mlp.experts.30.down_proj", "model.layers.50.mlp.experts.31.down_proj", "model.layers.50.mlp.experts.32.down_proj", "model.layers.50.mlp.experts.33.down_proj", "model.layers.50.mlp.experts.34.down_proj", "model.layers.50.mlp.experts.35.down_proj", "model.layers.50.mlp.experts.36.down_proj", "model.layers.50.mlp.experts.37.down_proj", "model.layers.50.mlp.experts.38.down_proj", "model.layers.50.mlp.experts.39.down_proj", "model.layers.50.mlp.experts.40.down_proj", "model.layers.50.mlp.experts.41.down_proj", "model.layers.50.mlp.experts.42.down_proj", "model.layers.50.mlp.experts.43.down_proj", "model.layers.50.mlp.experts.44.down_proj", "model.layers.50.mlp.experts.45.down_proj", "model.layers.50.mlp.experts.46.down_proj", "model.layers.50.mlp.experts.47.down_proj", "model.layers.50.mlp.experts.48.down_proj", "model.layers.50.mlp.experts.49.down_proj", "model.layers.50.mlp.experts.50.down_proj", "model.layers.50.mlp.experts.51.down_proj", "model.layers.50.mlp.experts.52.down_proj", "model.layers.50.mlp.experts.53.down_proj", "model.layers.50.mlp.experts.54.down_proj", "model.layers.50.mlp.experts.55.down_proj", "model.layers.50.mlp.experts.56.down_proj", "model.layers.50.mlp.experts.57.down_proj", "model.layers.50.mlp.experts.58.down_proj", "model.layers.50.mlp.experts.59.down_proj", "model.layers.50.mlp.experts.60.down_proj", "model.layers.50.mlp.experts.61.down_proj", "model.layers.50.mlp.experts.62.down_proj", "model.layers.50.mlp.experts.63.down_proj", "model.layers.50.mlp.experts.64.down_proj", "model.layers.50.mlp.experts.65.down_proj", "model.layers.50.mlp.experts.66.down_proj", "model.layers.50.mlp.experts.67.down_proj", "model.layers.50.mlp.experts.68.down_proj", "model.layers.50.mlp.experts.69.down_proj", "model.layers.50.mlp.experts.70.down_proj", "model.layers.50.mlp.experts.71.down_proj", "model.layers.50.mlp.experts.72.down_proj", "model.layers.50.mlp.experts.73.down_proj", "model.layers.50.mlp.experts.74.down_proj", "model.layers.50.mlp.experts.75.down_proj", "model.layers.50.mlp.experts.76.down_proj", "model.layers.50.mlp.experts.77.down_proj", "model.layers.50.mlp.experts.78.down_proj", "model.layers.50.mlp.experts.79.down_proj", "model.layers.50.mlp.experts.80.down_proj", "model.layers.50.mlp.experts.81.down_proj", "model.layers.50.mlp.experts.82.down_proj", "model.layers.50.mlp.experts.83.down_proj", "model.layers.50.mlp.experts.84.down_proj", "model.layers.50.mlp.experts.85.down_proj", "model.layers.50.mlp.experts.86.down_proj", "model.layers.50.mlp.experts.87.down_proj", "model.layers.50.mlp.experts.88.down_proj", "model.layers.50.mlp.experts.89.down_proj", "model.layers.50.mlp.experts.90.down_proj", "model.layers.50.mlp.experts.91.down_proj", "model.layers.50.mlp.experts.92.down_proj", "model.layers.50.mlp.experts.93.down_proj", "model.layers.50.mlp.experts.94.down_proj", "model.layers.50.mlp.experts.95.down_proj", "model.layers.50.mlp.experts.96.down_proj", "model.layers.50.mlp.experts.97.down_proj", "model.layers.50.mlp.experts.98.down_proj", "model.layers.50.mlp.experts.99.down_proj", "model.layers.50.mlp.experts.100.down_proj", "model.layers.50.mlp.experts.101.down_proj", "model.layers.50.mlp.experts.102.down_proj", "model.layers.50.mlp.experts.103.down_proj", "model.layers.50.mlp.experts.104.down_proj", "model.layers.50.mlp.experts.105.down_proj", "model.layers.50.mlp.experts.106.down_proj", "model.layers.50.mlp.experts.107.down_proj", "model.layers.50.mlp.experts.108.down_proj", "model.layers.50.mlp.experts.109.down_proj", "model.layers.50.mlp.experts.110.down_proj", "model.layers.50.mlp.experts.111.down_proj", "model.layers.50.mlp.experts.112.down_proj", "model.layers.50.mlp.experts.113.down_proj", "model.layers.50.mlp.experts.114.down_proj", "model.layers.50.mlp.experts.115.down_proj", "model.layers.50.mlp.experts.116.down_proj", "model.layers.50.mlp.experts.117.down_proj", "model.layers.50.mlp.experts.118.down_proj", "model.layers.50.mlp.experts.119.down_proj", "model.layers.50.mlp.experts.120.down_proj", "model.layers.50.mlp.experts.121.down_proj", "model.layers.50.mlp.experts.122.down_proj", "model.layers.50.mlp.experts.123.down_proj", "model.layers.50.mlp.experts.124.down_proj", "model.layers.50.mlp.experts.125.down_proj", "model.layers.50.mlp.experts.126.down_proj", "model.layers.50.mlp.experts.127.down_proj", "model.layers.50.mlp.experts.128.down_proj", "model.layers.50.mlp.experts.129.down_proj", "model.layers.50.mlp.experts.130.down_proj", "model.layers.50.mlp.experts.131.down_proj", "model.layers.50.mlp.experts.132.down_proj", "model.layers.50.mlp.experts.133.down_proj", "model.layers.50.mlp.experts.134.down_proj", "model.layers.50.mlp.experts.135.down_proj", "model.layers.50.mlp.experts.136.down_proj", "model.layers.50.mlp.experts.137.down_proj", "model.layers.50.mlp.experts.138.down_proj", "model.layers.50.mlp.experts.139.down_proj", "model.layers.50.mlp.experts.140.down_proj", "model.layers.50.mlp.experts.141.down_proj", "model.layers.50.mlp.experts.142.down_proj", "model.layers.50.mlp.experts.143.down_proj", "model.layers.50.mlp.experts.144.down_proj", "model.layers.50.mlp.experts.145.down_proj", "model.layers.50.mlp.experts.146.down_proj", "model.layers.50.mlp.experts.147.down_proj", "model.layers.50.mlp.experts.148.down_proj", "model.layers.50.mlp.experts.149.down_proj", "model.layers.50.mlp.experts.150.down_proj", "model.layers.50.mlp.experts.151.down_proj", "model.layers.50.mlp.experts.152.down_proj", "model.layers.50.mlp.experts.153.down_proj", "model.layers.50.mlp.experts.154.down_proj", "model.layers.50.mlp.experts.155.down_proj", "model.layers.50.mlp.experts.156.down_proj", "model.layers.50.mlp.experts.157.down_proj", "model.layers.50.mlp.experts.158.down_proj", "model.layers.50.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006457269191741943, "dbits": 1258291200 } ] }, { "idx": 303, "layers": [ "model.layers.51.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0013870999217033275, "dbits": 62914560 } ] }, { "idx": 304, "layers": [ "model.layers.51.self_attn.k_proj", "model.layers.51.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0010780155658721702, "dbits": 10485760 } ] }, { "idx": 305, "layers": [ "model.layers.51.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0015597656369208956, "dbits": 62914560 } ] }, { "idx": 306, "layers": [ "model.layers.51.mlp.shared_experts.gate_proj", "model.layers.51.mlp.shared_experts.up_proj", "model.layers.51.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00011052116751669727, "dbits": 23592960 } ] }, { "idx": 307, "layers": [ "model.layers.51.mlp.experts.0.gate_proj", "model.layers.51.mlp.experts.1.gate_proj", "model.layers.51.mlp.experts.2.gate_proj", "model.layers.51.mlp.experts.3.gate_proj", "model.layers.51.mlp.experts.4.gate_proj", "model.layers.51.mlp.experts.5.gate_proj", "model.layers.51.mlp.experts.6.gate_proj", "model.layers.51.mlp.experts.7.gate_proj", "model.layers.51.mlp.experts.8.gate_proj", "model.layers.51.mlp.experts.9.gate_proj", "model.layers.51.mlp.experts.10.gate_proj", "model.layers.51.mlp.experts.11.gate_proj", "model.layers.51.mlp.experts.12.gate_proj", "model.layers.51.mlp.experts.13.gate_proj", "model.layers.51.mlp.experts.14.gate_proj", "model.layers.51.mlp.experts.15.gate_proj", "model.layers.51.mlp.experts.16.gate_proj", "model.layers.51.mlp.experts.17.gate_proj", "model.layers.51.mlp.experts.18.gate_proj", "model.layers.51.mlp.experts.19.gate_proj", "model.layers.51.mlp.experts.20.gate_proj", "model.layers.51.mlp.experts.21.gate_proj", "model.layers.51.mlp.experts.22.gate_proj", "model.layers.51.mlp.experts.23.gate_proj", "model.layers.51.mlp.experts.24.gate_proj", "model.layers.51.mlp.experts.25.gate_proj", "model.layers.51.mlp.experts.26.gate_proj", "model.layers.51.mlp.experts.27.gate_proj", "model.layers.51.mlp.experts.28.gate_proj", "model.layers.51.mlp.experts.29.gate_proj", "model.layers.51.mlp.experts.30.gate_proj", "model.layers.51.mlp.experts.31.gate_proj", "model.layers.51.mlp.experts.32.gate_proj", "model.layers.51.mlp.experts.33.gate_proj", "model.layers.51.mlp.experts.34.gate_proj", "model.layers.51.mlp.experts.35.gate_proj", "model.layers.51.mlp.experts.36.gate_proj", "model.layers.51.mlp.experts.37.gate_proj", "model.layers.51.mlp.experts.38.gate_proj", "model.layers.51.mlp.experts.39.gate_proj", "model.layers.51.mlp.experts.40.gate_proj", "model.layers.51.mlp.experts.41.gate_proj", "model.layers.51.mlp.experts.42.gate_proj", "model.layers.51.mlp.experts.43.gate_proj", "model.layers.51.mlp.experts.44.gate_proj", "model.layers.51.mlp.experts.45.gate_proj", "model.layers.51.mlp.experts.46.gate_proj", "model.layers.51.mlp.experts.47.gate_proj", "model.layers.51.mlp.experts.48.gate_proj", "model.layers.51.mlp.experts.49.gate_proj", "model.layers.51.mlp.experts.50.gate_proj", "model.layers.51.mlp.experts.51.gate_proj", "model.layers.51.mlp.experts.52.gate_proj", "model.layers.51.mlp.experts.53.gate_proj", "model.layers.51.mlp.experts.54.gate_proj", "model.layers.51.mlp.experts.55.gate_proj", "model.layers.51.mlp.experts.56.gate_proj", "model.layers.51.mlp.experts.57.gate_proj", "model.layers.51.mlp.experts.58.gate_proj", "model.layers.51.mlp.experts.59.gate_proj", "model.layers.51.mlp.experts.60.gate_proj", "model.layers.51.mlp.experts.61.gate_proj", "model.layers.51.mlp.experts.62.gate_proj", "model.layers.51.mlp.experts.63.gate_proj", "model.layers.51.mlp.experts.64.gate_proj", "model.layers.51.mlp.experts.65.gate_proj", "model.layers.51.mlp.experts.66.gate_proj", "model.layers.51.mlp.experts.67.gate_proj", "model.layers.51.mlp.experts.68.gate_proj", "model.layers.51.mlp.experts.69.gate_proj", "model.layers.51.mlp.experts.70.gate_proj", "model.layers.51.mlp.experts.71.gate_proj", "model.layers.51.mlp.experts.72.gate_proj", "model.layers.51.mlp.experts.73.gate_proj", "model.layers.51.mlp.experts.74.gate_proj", "model.layers.51.mlp.experts.75.gate_proj", "model.layers.51.mlp.experts.76.gate_proj", "model.layers.51.mlp.experts.77.gate_proj", "model.layers.51.mlp.experts.78.gate_proj", "model.layers.51.mlp.experts.79.gate_proj", "model.layers.51.mlp.experts.80.gate_proj", "model.layers.51.mlp.experts.81.gate_proj", "model.layers.51.mlp.experts.82.gate_proj", "model.layers.51.mlp.experts.83.gate_proj", "model.layers.51.mlp.experts.84.gate_proj", "model.layers.51.mlp.experts.85.gate_proj", "model.layers.51.mlp.experts.86.gate_proj", "model.layers.51.mlp.experts.87.gate_proj", "model.layers.51.mlp.experts.88.gate_proj", "model.layers.51.mlp.experts.89.gate_proj", "model.layers.51.mlp.experts.90.gate_proj", "model.layers.51.mlp.experts.91.gate_proj", "model.layers.51.mlp.experts.92.gate_proj", "model.layers.51.mlp.experts.93.gate_proj", "model.layers.51.mlp.experts.94.gate_proj", "model.layers.51.mlp.experts.95.gate_proj", "model.layers.51.mlp.experts.96.gate_proj", "model.layers.51.mlp.experts.97.gate_proj", "model.layers.51.mlp.experts.98.gate_proj", "model.layers.51.mlp.experts.99.gate_proj", "model.layers.51.mlp.experts.100.gate_proj", "model.layers.51.mlp.experts.101.gate_proj", "model.layers.51.mlp.experts.102.gate_proj", "model.layers.51.mlp.experts.103.gate_proj", "model.layers.51.mlp.experts.104.gate_proj", "model.layers.51.mlp.experts.105.gate_proj", "model.layers.51.mlp.experts.106.gate_proj", "model.layers.51.mlp.experts.107.gate_proj", "model.layers.51.mlp.experts.108.gate_proj", "model.layers.51.mlp.experts.109.gate_proj", "model.layers.51.mlp.experts.110.gate_proj", "model.layers.51.mlp.experts.111.gate_proj", "model.layers.51.mlp.experts.112.gate_proj", "model.layers.51.mlp.experts.113.gate_proj", "model.layers.51.mlp.experts.114.gate_proj", "model.layers.51.mlp.experts.115.gate_proj", "model.layers.51.mlp.experts.116.gate_proj", "model.layers.51.mlp.experts.117.gate_proj", "model.layers.51.mlp.experts.118.gate_proj", "model.layers.51.mlp.experts.119.gate_proj", "model.layers.51.mlp.experts.120.gate_proj", "model.layers.51.mlp.experts.121.gate_proj", "model.layers.51.mlp.experts.122.gate_proj", "model.layers.51.mlp.experts.123.gate_proj", "model.layers.51.mlp.experts.124.gate_proj", "model.layers.51.mlp.experts.125.gate_proj", "model.layers.51.mlp.experts.126.gate_proj", "model.layers.51.mlp.experts.127.gate_proj", "model.layers.51.mlp.experts.128.gate_proj", "model.layers.51.mlp.experts.129.gate_proj", "model.layers.51.mlp.experts.130.gate_proj", "model.layers.51.mlp.experts.131.gate_proj", "model.layers.51.mlp.experts.132.gate_proj", "model.layers.51.mlp.experts.133.gate_proj", "model.layers.51.mlp.experts.134.gate_proj", "model.layers.51.mlp.experts.135.gate_proj", "model.layers.51.mlp.experts.136.gate_proj", "model.layers.51.mlp.experts.137.gate_proj", "model.layers.51.mlp.experts.138.gate_proj", "model.layers.51.mlp.experts.139.gate_proj", "model.layers.51.mlp.experts.140.gate_proj", "model.layers.51.mlp.experts.141.gate_proj", "model.layers.51.mlp.experts.142.gate_proj", "model.layers.51.mlp.experts.143.gate_proj", "model.layers.51.mlp.experts.144.gate_proj", "model.layers.51.mlp.experts.145.gate_proj", "model.layers.51.mlp.experts.146.gate_proj", "model.layers.51.mlp.experts.147.gate_proj", "model.layers.51.mlp.experts.148.gate_proj", "model.layers.51.mlp.experts.149.gate_proj", "model.layers.51.mlp.experts.150.gate_proj", "model.layers.51.mlp.experts.151.gate_proj", "model.layers.51.mlp.experts.152.gate_proj", "model.layers.51.mlp.experts.153.gate_proj", "model.layers.51.mlp.experts.154.gate_proj", "model.layers.51.mlp.experts.155.gate_proj", "model.layers.51.mlp.experts.156.gate_proj", "model.layers.51.mlp.experts.157.gate_proj", "model.layers.51.mlp.experts.158.gate_proj", "model.layers.51.mlp.experts.159.gate_proj", "model.layers.51.mlp.experts.0.up_proj", "model.layers.51.mlp.experts.1.up_proj", "model.layers.51.mlp.experts.2.up_proj", "model.layers.51.mlp.experts.3.up_proj", "model.layers.51.mlp.experts.4.up_proj", "model.layers.51.mlp.experts.5.up_proj", "model.layers.51.mlp.experts.6.up_proj", "model.layers.51.mlp.experts.7.up_proj", "model.layers.51.mlp.experts.8.up_proj", "model.layers.51.mlp.experts.9.up_proj", "model.layers.51.mlp.experts.10.up_proj", "model.layers.51.mlp.experts.11.up_proj", "model.layers.51.mlp.experts.12.up_proj", "model.layers.51.mlp.experts.13.up_proj", "model.layers.51.mlp.experts.14.up_proj", "model.layers.51.mlp.experts.15.up_proj", "model.layers.51.mlp.experts.16.up_proj", "model.layers.51.mlp.experts.17.up_proj", "model.layers.51.mlp.experts.18.up_proj", "model.layers.51.mlp.experts.19.up_proj", "model.layers.51.mlp.experts.20.up_proj", "model.layers.51.mlp.experts.21.up_proj", "model.layers.51.mlp.experts.22.up_proj", "model.layers.51.mlp.experts.23.up_proj", "model.layers.51.mlp.experts.24.up_proj", "model.layers.51.mlp.experts.25.up_proj", "model.layers.51.mlp.experts.26.up_proj", "model.layers.51.mlp.experts.27.up_proj", "model.layers.51.mlp.experts.28.up_proj", "model.layers.51.mlp.experts.29.up_proj", "model.layers.51.mlp.experts.30.up_proj", "model.layers.51.mlp.experts.31.up_proj", "model.layers.51.mlp.experts.32.up_proj", "model.layers.51.mlp.experts.33.up_proj", "model.layers.51.mlp.experts.34.up_proj", "model.layers.51.mlp.experts.35.up_proj", "model.layers.51.mlp.experts.36.up_proj", "model.layers.51.mlp.experts.37.up_proj", "model.layers.51.mlp.experts.38.up_proj", "model.layers.51.mlp.experts.39.up_proj", "model.layers.51.mlp.experts.40.up_proj", "model.layers.51.mlp.experts.41.up_proj", "model.layers.51.mlp.experts.42.up_proj", "model.layers.51.mlp.experts.43.up_proj", "model.layers.51.mlp.experts.44.up_proj", "model.layers.51.mlp.experts.45.up_proj", "model.layers.51.mlp.experts.46.up_proj", "model.layers.51.mlp.experts.47.up_proj", "model.layers.51.mlp.experts.48.up_proj", "model.layers.51.mlp.experts.49.up_proj", "model.layers.51.mlp.experts.50.up_proj", "model.layers.51.mlp.experts.51.up_proj", "model.layers.51.mlp.experts.52.up_proj", "model.layers.51.mlp.experts.53.up_proj", "model.layers.51.mlp.experts.54.up_proj", "model.layers.51.mlp.experts.55.up_proj", "model.layers.51.mlp.experts.56.up_proj", "model.layers.51.mlp.experts.57.up_proj", "model.layers.51.mlp.experts.58.up_proj", "model.layers.51.mlp.experts.59.up_proj", "model.layers.51.mlp.experts.60.up_proj", "model.layers.51.mlp.experts.61.up_proj", "model.layers.51.mlp.experts.62.up_proj", "model.layers.51.mlp.experts.63.up_proj", "model.layers.51.mlp.experts.64.up_proj", "model.layers.51.mlp.experts.65.up_proj", "model.layers.51.mlp.experts.66.up_proj", "model.layers.51.mlp.experts.67.up_proj", "model.layers.51.mlp.experts.68.up_proj", "model.layers.51.mlp.experts.69.up_proj", "model.layers.51.mlp.experts.70.up_proj", "model.layers.51.mlp.experts.71.up_proj", "model.layers.51.mlp.experts.72.up_proj", "model.layers.51.mlp.experts.73.up_proj", "model.layers.51.mlp.experts.74.up_proj", "model.layers.51.mlp.experts.75.up_proj", "model.layers.51.mlp.experts.76.up_proj", "model.layers.51.mlp.experts.77.up_proj", "model.layers.51.mlp.experts.78.up_proj", "model.layers.51.mlp.experts.79.up_proj", "model.layers.51.mlp.experts.80.up_proj", "model.layers.51.mlp.experts.81.up_proj", "model.layers.51.mlp.experts.82.up_proj", "model.layers.51.mlp.experts.83.up_proj", "model.layers.51.mlp.experts.84.up_proj", "model.layers.51.mlp.experts.85.up_proj", "model.layers.51.mlp.experts.86.up_proj", "model.layers.51.mlp.experts.87.up_proj", "model.layers.51.mlp.experts.88.up_proj", "model.layers.51.mlp.experts.89.up_proj", "model.layers.51.mlp.experts.90.up_proj", "model.layers.51.mlp.experts.91.up_proj", "model.layers.51.mlp.experts.92.up_proj", "model.layers.51.mlp.experts.93.up_proj", "model.layers.51.mlp.experts.94.up_proj", "model.layers.51.mlp.experts.95.up_proj", "model.layers.51.mlp.experts.96.up_proj", "model.layers.51.mlp.experts.97.up_proj", "model.layers.51.mlp.experts.98.up_proj", "model.layers.51.mlp.experts.99.up_proj", "model.layers.51.mlp.experts.100.up_proj", "model.layers.51.mlp.experts.101.up_proj", "model.layers.51.mlp.experts.102.up_proj", "model.layers.51.mlp.experts.103.up_proj", "model.layers.51.mlp.experts.104.up_proj", "model.layers.51.mlp.experts.105.up_proj", "model.layers.51.mlp.experts.106.up_proj", "model.layers.51.mlp.experts.107.up_proj", "model.layers.51.mlp.experts.108.up_proj", "model.layers.51.mlp.experts.109.up_proj", "model.layers.51.mlp.experts.110.up_proj", "model.layers.51.mlp.experts.111.up_proj", "model.layers.51.mlp.experts.112.up_proj", "model.layers.51.mlp.experts.113.up_proj", "model.layers.51.mlp.experts.114.up_proj", "model.layers.51.mlp.experts.115.up_proj", "model.layers.51.mlp.experts.116.up_proj", "model.layers.51.mlp.experts.117.up_proj", "model.layers.51.mlp.experts.118.up_proj", "model.layers.51.mlp.experts.119.up_proj", "model.layers.51.mlp.experts.120.up_proj", "model.layers.51.mlp.experts.121.up_proj", "model.layers.51.mlp.experts.122.up_proj", "model.layers.51.mlp.experts.123.up_proj", "model.layers.51.mlp.experts.124.up_proj", "model.layers.51.mlp.experts.125.up_proj", "model.layers.51.mlp.experts.126.up_proj", "model.layers.51.mlp.experts.127.up_proj", "model.layers.51.mlp.experts.128.up_proj", "model.layers.51.mlp.experts.129.up_proj", "model.layers.51.mlp.experts.130.up_proj", "model.layers.51.mlp.experts.131.up_proj", "model.layers.51.mlp.experts.132.up_proj", "model.layers.51.mlp.experts.133.up_proj", "model.layers.51.mlp.experts.134.up_proj", "model.layers.51.mlp.experts.135.up_proj", "model.layers.51.mlp.experts.136.up_proj", "model.layers.51.mlp.experts.137.up_proj", "model.layers.51.mlp.experts.138.up_proj", "model.layers.51.mlp.experts.139.up_proj", "model.layers.51.mlp.experts.140.up_proj", "model.layers.51.mlp.experts.141.up_proj", "model.layers.51.mlp.experts.142.up_proj", "model.layers.51.mlp.experts.143.up_proj", "model.layers.51.mlp.experts.144.up_proj", "model.layers.51.mlp.experts.145.up_proj", "model.layers.51.mlp.experts.146.up_proj", "model.layers.51.mlp.experts.147.up_proj", "model.layers.51.mlp.experts.148.up_proj", "model.layers.51.mlp.experts.149.up_proj", "model.layers.51.mlp.experts.150.up_proj", "model.layers.51.mlp.experts.151.up_proj", "model.layers.51.mlp.experts.152.up_proj", "model.layers.51.mlp.experts.153.up_proj", "model.layers.51.mlp.experts.154.up_proj", "model.layers.51.mlp.experts.155.up_proj", "model.layers.51.mlp.experts.156.up_proj", "model.layers.51.mlp.experts.157.up_proj", "model.layers.51.mlp.experts.158.up_proj", "model.layers.51.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0005790814757347107, "dbits": 2516582400 } ] }, { "idx": 308, "layers": [ "model.layers.51.mlp.experts.0.down_proj", "model.layers.51.mlp.experts.1.down_proj", "model.layers.51.mlp.experts.2.down_proj", "model.layers.51.mlp.experts.3.down_proj", "model.layers.51.mlp.experts.4.down_proj", "model.layers.51.mlp.experts.5.down_proj", "model.layers.51.mlp.experts.6.down_proj", "model.layers.51.mlp.experts.7.down_proj", "model.layers.51.mlp.experts.8.down_proj", "model.layers.51.mlp.experts.9.down_proj", "model.layers.51.mlp.experts.10.down_proj", "model.layers.51.mlp.experts.11.down_proj", "model.layers.51.mlp.experts.12.down_proj", "model.layers.51.mlp.experts.13.down_proj", "model.layers.51.mlp.experts.14.down_proj", "model.layers.51.mlp.experts.15.down_proj", "model.layers.51.mlp.experts.16.down_proj", "model.layers.51.mlp.experts.17.down_proj", "model.layers.51.mlp.experts.18.down_proj", "model.layers.51.mlp.experts.19.down_proj", "model.layers.51.mlp.experts.20.down_proj", "model.layers.51.mlp.experts.21.down_proj", "model.layers.51.mlp.experts.22.down_proj", "model.layers.51.mlp.experts.23.down_proj", "model.layers.51.mlp.experts.24.down_proj", "model.layers.51.mlp.experts.25.down_proj", "model.layers.51.mlp.experts.26.down_proj", "model.layers.51.mlp.experts.27.down_proj", "model.layers.51.mlp.experts.28.down_proj", "model.layers.51.mlp.experts.29.down_proj", "model.layers.51.mlp.experts.30.down_proj", "model.layers.51.mlp.experts.31.down_proj", "model.layers.51.mlp.experts.32.down_proj", "model.layers.51.mlp.experts.33.down_proj", "model.layers.51.mlp.experts.34.down_proj", "model.layers.51.mlp.experts.35.down_proj", "model.layers.51.mlp.experts.36.down_proj", "model.layers.51.mlp.experts.37.down_proj", "model.layers.51.mlp.experts.38.down_proj", "model.layers.51.mlp.experts.39.down_proj", "model.layers.51.mlp.experts.40.down_proj", "model.layers.51.mlp.experts.41.down_proj", "model.layers.51.mlp.experts.42.down_proj", "model.layers.51.mlp.experts.43.down_proj", "model.layers.51.mlp.experts.44.down_proj", "model.layers.51.mlp.experts.45.down_proj", "model.layers.51.mlp.experts.46.down_proj", "model.layers.51.mlp.experts.47.down_proj", "model.layers.51.mlp.experts.48.down_proj", "model.layers.51.mlp.experts.49.down_proj", "model.layers.51.mlp.experts.50.down_proj", "model.layers.51.mlp.experts.51.down_proj", "model.layers.51.mlp.experts.52.down_proj", "model.layers.51.mlp.experts.53.down_proj", "model.layers.51.mlp.experts.54.down_proj", "model.layers.51.mlp.experts.55.down_proj", "model.layers.51.mlp.experts.56.down_proj", "model.layers.51.mlp.experts.57.down_proj", "model.layers.51.mlp.experts.58.down_proj", "model.layers.51.mlp.experts.59.down_proj", "model.layers.51.mlp.experts.60.down_proj", "model.layers.51.mlp.experts.61.down_proj", "model.layers.51.mlp.experts.62.down_proj", "model.layers.51.mlp.experts.63.down_proj", "model.layers.51.mlp.experts.64.down_proj", "model.layers.51.mlp.experts.65.down_proj", "model.layers.51.mlp.experts.66.down_proj", "model.layers.51.mlp.experts.67.down_proj", "model.layers.51.mlp.experts.68.down_proj", "model.layers.51.mlp.experts.69.down_proj", "model.layers.51.mlp.experts.70.down_proj", "model.layers.51.mlp.experts.71.down_proj", "model.layers.51.mlp.experts.72.down_proj", "model.layers.51.mlp.experts.73.down_proj", "model.layers.51.mlp.experts.74.down_proj", "model.layers.51.mlp.experts.75.down_proj", "model.layers.51.mlp.experts.76.down_proj", "model.layers.51.mlp.experts.77.down_proj", "model.layers.51.mlp.experts.78.down_proj", "model.layers.51.mlp.experts.79.down_proj", "model.layers.51.mlp.experts.80.down_proj", "model.layers.51.mlp.experts.81.down_proj", "model.layers.51.mlp.experts.82.down_proj", "model.layers.51.mlp.experts.83.down_proj", "model.layers.51.mlp.experts.84.down_proj", "model.layers.51.mlp.experts.85.down_proj", "model.layers.51.mlp.experts.86.down_proj", "model.layers.51.mlp.experts.87.down_proj", "model.layers.51.mlp.experts.88.down_proj", "model.layers.51.mlp.experts.89.down_proj", "model.layers.51.mlp.experts.90.down_proj", "model.layers.51.mlp.experts.91.down_proj", "model.layers.51.mlp.experts.92.down_proj", "model.layers.51.mlp.experts.93.down_proj", "model.layers.51.mlp.experts.94.down_proj", "model.layers.51.mlp.experts.95.down_proj", "model.layers.51.mlp.experts.96.down_proj", "model.layers.51.mlp.experts.97.down_proj", "model.layers.51.mlp.experts.98.down_proj", "model.layers.51.mlp.experts.99.down_proj", "model.layers.51.mlp.experts.100.down_proj", "model.layers.51.mlp.experts.101.down_proj", "model.layers.51.mlp.experts.102.down_proj", "model.layers.51.mlp.experts.103.down_proj", "model.layers.51.mlp.experts.104.down_proj", "model.layers.51.mlp.experts.105.down_proj", "model.layers.51.mlp.experts.106.down_proj", "model.layers.51.mlp.experts.107.down_proj", "model.layers.51.mlp.experts.108.down_proj", "model.layers.51.mlp.experts.109.down_proj", "model.layers.51.mlp.experts.110.down_proj", "model.layers.51.mlp.experts.111.down_proj", "model.layers.51.mlp.experts.112.down_proj", "model.layers.51.mlp.experts.113.down_proj", "model.layers.51.mlp.experts.114.down_proj", "model.layers.51.mlp.experts.115.down_proj", "model.layers.51.mlp.experts.116.down_proj", "model.layers.51.mlp.experts.117.down_proj", "model.layers.51.mlp.experts.118.down_proj", "model.layers.51.mlp.experts.119.down_proj", "model.layers.51.mlp.experts.120.down_proj", "model.layers.51.mlp.experts.121.down_proj", "model.layers.51.mlp.experts.122.down_proj", "model.layers.51.mlp.experts.123.down_proj", "model.layers.51.mlp.experts.124.down_proj", "model.layers.51.mlp.experts.125.down_proj", "model.layers.51.mlp.experts.126.down_proj", "model.layers.51.mlp.experts.127.down_proj", "model.layers.51.mlp.experts.128.down_proj", "model.layers.51.mlp.experts.129.down_proj", "model.layers.51.mlp.experts.130.down_proj", "model.layers.51.mlp.experts.131.down_proj", "model.layers.51.mlp.experts.132.down_proj", "model.layers.51.mlp.experts.133.down_proj", "model.layers.51.mlp.experts.134.down_proj", "model.layers.51.mlp.experts.135.down_proj", "model.layers.51.mlp.experts.136.down_proj", "model.layers.51.mlp.experts.137.down_proj", "model.layers.51.mlp.experts.138.down_proj", "model.layers.51.mlp.experts.139.down_proj", "model.layers.51.mlp.experts.140.down_proj", "model.layers.51.mlp.experts.141.down_proj", "model.layers.51.mlp.experts.142.down_proj", "model.layers.51.mlp.experts.143.down_proj", "model.layers.51.mlp.experts.144.down_proj", "model.layers.51.mlp.experts.145.down_proj", "model.layers.51.mlp.experts.146.down_proj", "model.layers.51.mlp.experts.147.down_proj", "model.layers.51.mlp.experts.148.down_proj", "model.layers.51.mlp.experts.149.down_proj", "model.layers.51.mlp.experts.150.down_proj", "model.layers.51.mlp.experts.151.down_proj", "model.layers.51.mlp.experts.152.down_proj", "model.layers.51.mlp.experts.153.down_proj", "model.layers.51.mlp.experts.154.down_proj", "model.layers.51.mlp.experts.155.down_proj", "model.layers.51.mlp.experts.156.down_proj", "model.layers.51.mlp.experts.157.down_proj", "model.layers.51.mlp.experts.158.down_proj", "model.layers.51.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006013467907905468, "dbits": 1258291200 } ] }, { "idx": 309, "layers": [ "model.layers.52.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0001091487705707217, "dbits": 62914560 } ] }, { "idx": 310, "layers": [ "model.layers.52.self_attn.k_proj", "model.layers.52.self_attn.v_proj" ], "candidates": [ { "dkld": -0.003292652964591969, "dbits": 10485760 } ] }, { "idx": 311, "layers": [ "model.layers.52.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0005625464022159243, "dbits": 62914560 } ] }, { "idx": 312, "layers": [ "model.layers.52.mlp.shared_experts.gate_proj", "model.layers.52.mlp.shared_experts.up_proj", "model.layers.52.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0011545047163963318, "dbits": 23592960 } ] }, { "idx": 313, "layers": [ "model.layers.52.mlp.experts.0.gate_proj", "model.layers.52.mlp.experts.1.gate_proj", "model.layers.52.mlp.experts.2.gate_proj", "model.layers.52.mlp.experts.3.gate_proj", "model.layers.52.mlp.experts.4.gate_proj", "model.layers.52.mlp.experts.5.gate_proj", "model.layers.52.mlp.experts.6.gate_proj", "model.layers.52.mlp.experts.7.gate_proj", "model.layers.52.mlp.experts.8.gate_proj", "model.layers.52.mlp.experts.9.gate_proj", "model.layers.52.mlp.experts.10.gate_proj", "model.layers.52.mlp.experts.11.gate_proj", "model.layers.52.mlp.experts.12.gate_proj", "model.layers.52.mlp.experts.13.gate_proj", "model.layers.52.mlp.experts.14.gate_proj", "model.layers.52.mlp.experts.15.gate_proj", "model.layers.52.mlp.experts.16.gate_proj", "model.layers.52.mlp.experts.17.gate_proj", "model.layers.52.mlp.experts.18.gate_proj", "model.layers.52.mlp.experts.19.gate_proj", "model.layers.52.mlp.experts.20.gate_proj", "model.layers.52.mlp.experts.21.gate_proj", "model.layers.52.mlp.experts.22.gate_proj", "model.layers.52.mlp.experts.23.gate_proj", "model.layers.52.mlp.experts.24.gate_proj", "model.layers.52.mlp.experts.25.gate_proj", "model.layers.52.mlp.experts.26.gate_proj", "model.layers.52.mlp.experts.27.gate_proj", "model.layers.52.mlp.experts.28.gate_proj", "model.layers.52.mlp.experts.29.gate_proj", "model.layers.52.mlp.experts.30.gate_proj", "model.layers.52.mlp.experts.31.gate_proj", "model.layers.52.mlp.experts.32.gate_proj", "model.layers.52.mlp.experts.33.gate_proj", "model.layers.52.mlp.experts.34.gate_proj", "model.layers.52.mlp.experts.35.gate_proj", "model.layers.52.mlp.experts.36.gate_proj", "model.layers.52.mlp.experts.37.gate_proj", "model.layers.52.mlp.experts.38.gate_proj", "model.layers.52.mlp.experts.39.gate_proj", "model.layers.52.mlp.experts.40.gate_proj", "model.layers.52.mlp.experts.41.gate_proj", "model.layers.52.mlp.experts.42.gate_proj", "model.layers.52.mlp.experts.43.gate_proj", "model.layers.52.mlp.experts.44.gate_proj", "model.layers.52.mlp.experts.45.gate_proj", "model.layers.52.mlp.experts.46.gate_proj", "model.layers.52.mlp.experts.47.gate_proj", "model.layers.52.mlp.experts.48.gate_proj", "model.layers.52.mlp.experts.49.gate_proj", "model.layers.52.mlp.experts.50.gate_proj", "model.layers.52.mlp.experts.51.gate_proj", "model.layers.52.mlp.experts.52.gate_proj", "model.layers.52.mlp.experts.53.gate_proj", "model.layers.52.mlp.experts.54.gate_proj", "model.layers.52.mlp.experts.55.gate_proj", "model.layers.52.mlp.experts.56.gate_proj", "model.layers.52.mlp.experts.57.gate_proj", "model.layers.52.mlp.experts.58.gate_proj", "model.layers.52.mlp.experts.59.gate_proj", "model.layers.52.mlp.experts.60.gate_proj", "model.layers.52.mlp.experts.61.gate_proj", "model.layers.52.mlp.experts.62.gate_proj", "model.layers.52.mlp.experts.63.gate_proj", "model.layers.52.mlp.experts.64.gate_proj", "model.layers.52.mlp.experts.65.gate_proj", "model.layers.52.mlp.experts.66.gate_proj", "model.layers.52.mlp.experts.67.gate_proj", "model.layers.52.mlp.experts.68.gate_proj", "model.layers.52.mlp.experts.69.gate_proj", "model.layers.52.mlp.experts.70.gate_proj", "model.layers.52.mlp.experts.71.gate_proj", "model.layers.52.mlp.experts.72.gate_proj", "model.layers.52.mlp.experts.73.gate_proj", "model.layers.52.mlp.experts.74.gate_proj", "model.layers.52.mlp.experts.75.gate_proj", "model.layers.52.mlp.experts.76.gate_proj", "model.layers.52.mlp.experts.77.gate_proj", "model.layers.52.mlp.experts.78.gate_proj", "model.layers.52.mlp.experts.79.gate_proj", "model.layers.52.mlp.experts.80.gate_proj", "model.layers.52.mlp.experts.81.gate_proj", "model.layers.52.mlp.experts.82.gate_proj", "model.layers.52.mlp.experts.83.gate_proj", "model.layers.52.mlp.experts.84.gate_proj", "model.layers.52.mlp.experts.85.gate_proj", "model.layers.52.mlp.experts.86.gate_proj", "model.layers.52.mlp.experts.87.gate_proj", "model.layers.52.mlp.experts.88.gate_proj", "model.layers.52.mlp.experts.89.gate_proj", "model.layers.52.mlp.experts.90.gate_proj", "model.layers.52.mlp.experts.91.gate_proj", "model.layers.52.mlp.experts.92.gate_proj", "model.layers.52.mlp.experts.93.gate_proj", "model.layers.52.mlp.experts.94.gate_proj", "model.layers.52.mlp.experts.95.gate_proj", "model.layers.52.mlp.experts.96.gate_proj", "model.layers.52.mlp.experts.97.gate_proj", "model.layers.52.mlp.experts.98.gate_proj", "model.layers.52.mlp.experts.99.gate_proj", "model.layers.52.mlp.experts.100.gate_proj", "model.layers.52.mlp.experts.101.gate_proj", "model.layers.52.mlp.experts.102.gate_proj", "model.layers.52.mlp.experts.103.gate_proj", "model.layers.52.mlp.experts.104.gate_proj", "model.layers.52.mlp.experts.105.gate_proj", "model.layers.52.mlp.experts.106.gate_proj", "model.layers.52.mlp.experts.107.gate_proj", "model.layers.52.mlp.experts.108.gate_proj", "model.layers.52.mlp.experts.109.gate_proj", "model.layers.52.mlp.experts.110.gate_proj", "model.layers.52.mlp.experts.111.gate_proj", "model.layers.52.mlp.experts.112.gate_proj", "model.layers.52.mlp.experts.113.gate_proj", "model.layers.52.mlp.experts.114.gate_proj", "model.layers.52.mlp.experts.115.gate_proj", "model.layers.52.mlp.experts.116.gate_proj", "model.layers.52.mlp.experts.117.gate_proj", "model.layers.52.mlp.experts.118.gate_proj", "model.layers.52.mlp.experts.119.gate_proj", "model.layers.52.mlp.experts.120.gate_proj", "model.layers.52.mlp.experts.121.gate_proj", "model.layers.52.mlp.experts.122.gate_proj", "model.layers.52.mlp.experts.123.gate_proj", "model.layers.52.mlp.experts.124.gate_proj", "model.layers.52.mlp.experts.125.gate_proj", "model.layers.52.mlp.experts.126.gate_proj", "model.layers.52.mlp.experts.127.gate_proj", "model.layers.52.mlp.experts.128.gate_proj", "model.layers.52.mlp.experts.129.gate_proj", "model.layers.52.mlp.experts.130.gate_proj", "model.layers.52.mlp.experts.131.gate_proj", "model.layers.52.mlp.experts.132.gate_proj", "model.layers.52.mlp.experts.133.gate_proj", "model.layers.52.mlp.experts.134.gate_proj", "model.layers.52.mlp.experts.135.gate_proj", "model.layers.52.mlp.experts.136.gate_proj", "model.layers.52.mlp.experts.137.gate_proj", "model.layers.52.mlp.experts.138.gate_proj", "model.layers.52.mlp.experts.139.gate_proj", "model.layers.52.mlp.experts.140.gate_proj", "model.layers.52.mlp.experts.141.gate_proj", "model.layers.52.mlp.experts.142.gate_proj", "model.layers.52.mlp.experts.143.gate_proj", "model.layers.52.mlp.experts.144.gate_proj", "model.layers.52.mlp.experts.145.gate_proj", "model.layers.52.mlp.experts.146.gate_proj", "model.layers.52.mlp.experts.147.gate_proj", "model.layers.52.mlp.experts.148.gate_proj", "model.layers.52.mlp.experts.149.gate_proj", "model.layers.52.mlp.experts.150.gate_proj", "model.layers.52.mlp.experts.151.gate_proj", "model.layers.52.mlp.experts.152.gate_proj", "model.layers.52.mlp.experts.153.gate_proj", "model.layers.52.mlp.experts.154.gate_proj", "model.layers.52.mlp.experts.155.gate_proj", "model.layers.52.mlp.experts.156.gate_proj", "model.layers.52.mlp.experts.157.gate_proj", "model.layers.52.mlp.experts.158.gate_proj", "model.layers.52.mlp.experts.159.gate_proj", "model.layers.52.mlp.experts.0.up_proj", "model.layers.52.mlp.experts.1.up_proj", "model.layers.52.mlp.experts.2.up_proj", "model.layers.52.mlp.experts.3.up_proj", "model.layers.52.mlp.experts.4.up_proj", "model.layers.52.mlp.experts.5.up_proj", "model.layers.52.mlp.experts.6.up_proj", "model.layers.52.mlp.experts.7.up_proj", "model.layers.52.mlp.experts.8.up_proj", "model.layers.52.mlp.experts.9.up_proj", "model.layers.52.mlp.experts.10.up_proj", "model.layers.52.mlp.experts.11.up_proj", "model.layers.52.mlp.experts.12.up_proj", "model.layers.52.mlp.experts.13.up_proj", "model.layers.52.mlp.experts.14.up_proj", "model.layers.52.mlp.experts.15.up_proj", "model.layers.52.mlp.experts.16.up_proj", "model.layers.52.mlp.experts.17.up_proj", "model.layers.52.mlp.experts.18.up_proj", "model.layers.52.mlp.experts.19.up_proj", "model.layers.52.mlp.experts.20.up_proj", "model.layers.52.mlp.experts.21.up_proj", "model.layers.52.mlp.experts.22.up_proj", "model.layers.52.mlp.experts.23.up_proj", "model.layers.52.mlp.experts.24.up_proj", "model.layers.52.mlp.experts.25.up_proj", "model.layers.52.mlp.experts.26.up_proj", "model.layers.52.mlp.experts.27.up_proj", "model.layers.52.mlp.experts.28.up_proj", "model.layers.52.mlp.experts.29.up_proj", "model.layers.52.mlp.experts.30.up_proj", "model.layers.52.mlp.experts.31.up_proj", "model.layers.52.mlp.experts.32.up_proj", "model.layers.52.mlp.experts.33.up_proj", "model.layers.52.mlp.experts.34.up_proj", "model.layers.52.mlp.experts.35.up_proj", "model.layers.52.mlp.experts.36.up_proj", "model.layers.52.mlp.experts.37.up_proj", "model.layers.52.mlp.experts.38.up_proj", "model.layers.52.mlp.experts.39.up_proj", "model.layers.52.mlp.experts.40.up_proj", "model.layers.52.mlp.experts.41.up_proj", "model.layers.52.mlp.experts.42.up_proj", "model.layers.52.mlp.experts.43.up_proj", "model.layers.52.mlp.experts.44.up_proj", "model.layers.52.mlp.experts.45.up_proj", "model.layers.52.mlp.experts.46.up_proj", "model.layers.52.mlp.experts.47.up_proj", "model.layers.52.mlp.experts.48.up_proj", "model.layers.52.mlp.experts.49.up_proj", "model.layers.52.mlp.experts.50.up_proj", "model.layers.52.mlp.experts.51.up_proj", "model.layers.52.mlp.experts.52.up_proj", "model.layers.52.mlp.experts.53.up_proj", "model.layers.52.mlp.experts.54.up_proj", "model.layers.52.mlp.experts.55.up_proj", "model.layers.52.mlp.experts.56.up_proj", "model.layers.52.mlp.experts.57.up_proj", "model.layers.52.mlp.experts.58.up_proj", "model.layers.52.mlp.experts.59.up_proj", "model.layers.52.mlp.experts.60.up_proj", "model.layers.52.mlp.experts.61.up_proj", "model.layers.52.mlp.experts.62.up_proj", "model.layers.52.mlp.experts.63.up_proj", "model.layers.52.mlp.experts.64.up_proj", "model.layers.52.mlp.experts.65.up_proj", "model.layers.52.mlp.experts.66.up_proj", "model.layers.52.mlp.experts.67.up_proj", "model.layers.52.mlp.experts.68.up_proj", "model.layers.52.mlp.experts.69.up_proj", "model.layers.52.mlp.experts.70.up_proj", "model.layers.52.mlp.experts.71.up_proj", "model.layers.52.mlp.experts.72.up_proj", "model.layers.52.mlp.experts.73.up_proj", "model.layers.52.mlp.experts.74.up_proj", "model.layers.52.mlp.experts.75.up_proj", "model.layers.52.mlp.experts.76.up_proj", "model.layers.52.mlp.experts.77.up_proj", "model.layers.52.mlp.experts.78.up_proj", "model.layers.52.mlp.experts.79.up_proj", "model.layers.52.mlp.experts.80.up_proj", "model.layers.52.mlp.experts.81.up_proj", "model.layers.52.mlp.experts.82.up_proj", "model.layers.52.mlp.experts.83.up_proj", "model.layers.52.mlp.experts.84.up_proj", "model.layers.52.mlp.experts.85.up_proj", "model.layers.52.mlp.experts.86.up_proj", "model.layers.52.mlp.experts.87.up_proj", "model.layers.52.mlp.experts.88.up_proj", "model.layers.52.mlp.experts.89.up_proj", "model.layers.52.mlp.experts.90.up_proj", "model.layers.52.mlp.experts.91.up_proj", "model.layers.52.mlp.experts.92.up_proj", "model.layers.52.mlp.experts.93.up_proj", "model.layers.52.mlp.experts.94.up_proj", "model.layers.52.mlp.experts.95.up_proj", "model.layers.52.mlp.experts.96.up_proj", "model.layers.52.mlp.experts.97.up_proj", "model.layers.52.mlp.experts.98.up_proj", "model.layers.52.mlp.experts.99.up_proj", "model.layers.52.mlp.experts.100.up_proj", "model.layers.52.mlp.experts.101.up_proj", "model.layers.52.mlp.experts.102.up_proj", "model.layers.52.mlp.experts.103.up_proj", "model.layers.52.mlp.experts.104.up_proj", "model.layers.52.mlp.experts.105.up_proj", "model.layers.52.mlp.experts.106.up_proj", "model.layers.52.mlp.experts.107.up_proj", "model.layers.52.mlp.experts.108.up_proj", "model.layers.52.mlp.experts.109.up_proj", "model.layers.52.mlp.experts.110.up_proj", "model.layers.52.mlp.experts.111.up_proj", "model.layers.52.mlp.experts.112.up_proj", "model.layers.52.mlp.experts.113.up_proj", "model.layers.52.mlp.experts.114.up_proj", "model.layers.52.mlp.experts.115.up_proj", "model.layers.52.mlp.experts.116.up_proj", "model.layers.52.mlp.experts.117.up_proj", "model.layers.52.mlp.experts.118.up_proj", "model.layers.52.mlp.experts.119.up_proj", "model.layers.52.mlp.experts.120.up_proj", "model.layers.52.mlp.experts.121.up_proj", "model.layers.52.mlp.experts.122.up_proj", "model.layers.52.mlp.experts.123.up_proj", "model.layers.52.mlp.experts.124.up_proj", "model.layers.52.mlp.experts.125.up_proj", "model.layers.52.mlp.experts.126.up_proj", "model.layers.52.mlp.experts.127.up_proj", "model.layers.52.mlp.experts.128.up_proj", "model.layers.52.mlp.experts.129.up_proj", "model.layers.52.mlp.experts.130.up_proj", "model.layers.52.mlp.experts.131.up_proj", "model.layers.52.mlp.experts.132.up_proj", "model.layers.52.mlp.experts.133.up_proj", "model.layers.52.mlp.experts.134.up_proj", "model.layers.52.mlp.experts.135.up_proj", "model.layers.52.mlp.experts.136.up_proj", "model.layers.52.mlp.experts.137.up_proj", "model.layers.52.mlp.experts.138.up_proj", "model.layers.52.mlp.experts.139.up_proj", "model.layers.52.mlp.experts.140.up_proj", "model.layers.52.mlp.experts.141.up_proj", "model.layers.52.mlp.experts.142.up_proj", "model.layers.52.mlp.experts.143.up_proj", "model.layers.52.mlp.experts.144.up_proj", "model.layers.52.mlp.experts.145.up_proj", "model.layers.52.mlp.experts.146.up_proj", "model.layers.52.mlp.experts.147.up_proj", "model.layers.52.mlp.experts.148.up_proj", "model.layers.52.mlp.experts.149.up_proj", "model.layers.52.mlp.experts.150.up_proj", "model.layers.52.mlp.experts.151.up_proj", "model.layers.52.mlp.experts.152.up_proj", "model.layers.52.mlp.experts.153.up_proj", "model.layers.52.mlp.experts.154.up_proj", "model.layers.52.mlp.experts.155.up_proj", "model.layers.52.mlp.experts.156.up_proj", "model.layers.52.mlp.experts.157.up_proj", "model.layers.52.mlp.experts.158.up_proj", "model.layers.52.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0018425136804580688, "dbits": 2516582400 } ] }, { "idx": 314, "layers": [ "model.layers.52.mlp.experts.0.down_proj", "model.layers.52.mlp.experts.1.down_proj", "model.layers.52.mlp.experts.2.down_proj", "model.layers.52.mlp.experts.3.down_proj", "model.layers.52.mlp.experts.4.down_proj", "model.layers.52.mlp.experts.5.down_proj", "model.layers.52.mlp.experts.6.down_proj", "model.layers.52.mlp.experts.7.down_proj", "model.layers.52.mlp.experts.8.down_proj", "model.layers.52.mlp.experts.9.down_proj", "model.layers.52.mlp.experts.10.down_proj", "model.layers.52.mlp.experts.11.down_proj", "model.layers.52.mlp.experts.12.down_proj", "model.layers.52.mlp.experts.13.down_proj", "model.layers.52.mlp.experts.14.down_proj", "model.layers.52.mlp.experts.15.down_proj", "model.layers.52.mlp.experts.16.down_proj", "model.layers.52.mlp.experts.17.down_proj", "model.layers.52.mlp.experts.18.down_proj", "model.layers.52.mlp.experts.19.down_proj", "model.layers.52.mlp.experts.20.down_proj", "model.layers.52.mlp.experts.21.down_proj", "model.layers.52.mlp.experts.22.down_proj", "model.layers.52.mlp.experts.23.down_proj", "model.layers.52.mlp.experts.24.down_proj", "model.layers.52.mlp.experts.25.down_proj", "model.layers.52.mlp.experts.26.down_proj", "model.layers.52.mlp.experts.27.down_proj", "model.layers.52.mlp.experts.28.down_proj", "model.layers.52.mlp.experts.29.down_proj", "model.layers.52.mlp.experts.30.down_proj", "model.layers.52.mlp.experts.31.down_proj", "model.layers.52.mlp.experts.32.down_proj", "model.layers.52.mlp.experts.33.down_proj", "model.layers.52.mlp.experts.34.down_proj", "model.layers.52.mlp.experts.35.down_proj", "model.layers.52.mlp.experts.36.down_proj", "model.layers.52.mlp.experts.37.down_proj", "model.layers.52.mlp.experts.38.down_proj", "model.layers.52.mlp.experts.39.down_proj", "model.layers.52.mlp.experts.40.down_proj", "model.layers.52.mlp.experts.41.down_proj", "model.layers.52.mlp.experts.42.down_proj", "model.layers.52.mlp.experts.43.down_proj", "model.layers.52.mlp.experts.44.down_proj", "model.layers.52.mlp.experts.45.down_proj", "model.layers.52.mlp.experts.46.down_proj", "model.layers.52.mlp.experts.47.down_proj", "model.layers.52.mlp.experts.48.down_proj", "model.layers.52.mlp.experts.49.down_proj", "model.layers.52.mlp.experts.50.down_proj", "model.layers.52.mlp.experts.51.down_proj", "model.layers.52.mlp.experts.52.down_proj", "model.layers.52.mlp.experts.53.down_proj", "model.layers.52.mlp.experts.54.down_proj", "model.layers.52.mlp.experts.55.down_proj", "model.layers.52.mlp.experts.56.down_proj", "model.layers.52.mlp.experts.57.down_proj", "model.layers.52.mlp.experts.58.down_proj", "model.layers.52.mlp.experts.59.down_proj", "model.layers.52.mlp.experts.60.down_proj", "model.layers.52.mlp.experts.61.down_proj", "model.layers.52.mlp.experts.62.down_proj", "model.layers.52.mlp.experts.63.down_proj", "model.layers.52.mlp.experts.64.down_proj", "model.layers.52.mlp.experts.65.down_proj", "model.layers.52.mlp.experts.66.down_proj", "model.layers.52.mlp.experts.67.down_proj", "model.layers.52.mlp.experts.68.down_proj", "model.layers.52.mlp.experts.69.down_proj", "model.layers.52.mlp.experts.70.down_proj", "model.layers.52.mlp.experts.71.down_proj", "model.layers.52.mlp.experts.72.down_proj", "model.layers.52.mlp.experts.73.down_proj", "model.layers.52.mlp.experts.74.down_proj", "model.layers.52.mlp.experts.75.down_proj", "model.layers.52.mlp.experts.76.down_proj", "model.layers.52.mlp.experts.77.down_proj", "model.layers.52.mlp.experts.78.down_proj", "model.layers.52.mlp.experts.79.down_proj", "model.layers.52.mlp.experts.80.down_proj", "model.layers.52.mlp.experts.81.down_proj", "model.layers.52.mlp.experts.82.down_proj", "model.layers.52.mlp.experts.83.down_proj", "model.layers.52.mlp.experts.84.down_proj", "model.layers.52.mlp.experts.85.down_proj", "model.layers.52.mlp.experts.86.down_proj", "model.layers.52.mlp.experts.87.down_proj", "model.layers.52.mlp.experts.88.down_proj", "model.layers.52.mlp.experts.89.down_proj", "model.layers.52.mlp.experts.90.down_proj", "model.layers.52.mlp.experts.91.down_proj", "model.layers.52.mlp.experts.92.down_proj", "model.layers.52.mlp.experts.93.down_proj", "model.layers.52.mlp.experts.94.down_proj", "model.layers.52.mlp.experts.95.down_proj", "model.layers.52.mlp.experts.96.down_proj", "model.layers.52.mlp.experts.97.down_proj", "model.layers.52.mlp.experts.98.down_proj", "model.layers.52.mlp.experts.99.down_proj", "model.layers.52.mlp.experts.100.down_proj", "model.layers.52.mlp.experts.101.down_proj", "model.layers.52.mlp.experts.102.down_proj", "model.layers.52.mlp.experts.103.down_proj", "model.layers.52.mlp.experts.104.down_proj", "model.layers.52.mlp.experts.105.down_proj", "model.layers.52.mlp.experts.106.down_proj", "model.layers.52.mlp.experts.107.down_proj", "model.layers.52.mlp.experts.108.down_proj", "model.layers.52.mlp.experts.109.down_proj", "model.layers.52.mlp.experts.110.down_proj", "model.layers.52.mlp.experts.111.down_proj", "model.layers.52.mlp.experts.112.down_proj", "model.layers.52.mlp.experts.113.down_proj", "model.layers.52.mlp.experts.114.down_proj", "model.layers.52.mlp.experts.115.down_proj", "model.layers.52.mlp.experts.116.down_proj", "model.layers.52.mlp.experts.117.down_proj", "model.layers.52.mlp.experts.118.down_proj", "model.layers.52.mlp.experts.119.down_proj", "model.layers.52.mlp.experts.120.down_proj", "model.layers.52.mlp.experts.121.down_proj", "model.layers.52.mlp.experts.122.down_proj", "model.layers.52.mlp.experts.123.down_proj", "model.layers.52.mlp.experts.124.down_proj", "model.layers.52.mlp.experts.125.down_proj", "model.layers.52.mlp.experts.126.down_proj", "model.layers.52.mlp.experts.127.down_proj", "model.layers.52.mlp.experts.128.down_proj", "model.layers.52.mlp.experts.129.down_proj", "model.layers.52.mlp.experts.130.down_proj", "model.layers.52.mlp.experts.131.down_proj", "model.layers.52.mlp.experts.132.down_proj", "model.layers.52.mlp.experts.133.down_proj", "model.layers.52.mlp.experts.134.down_proj", "model.layers.52.mlp.experts.135.down_proj", "model.layers.52.mlp.experts.136.down_proj", "model.layers.52.mlp.experts.137.down_proj", "model.layers.52.mlp.experts.138.down_proj", "model.layers.52.mlp.experts.139.down_proj", "model.layers.52.mlp.experts.140.down_proj", "model.layers.52.mlp.experts.141.down_proj", "model.layers.52.mlp.experts.142.down_proj", "model.layers.52.mlp.experts.143.down_proj", "model.layers.52.mlp.experts.144.down_proj", "model.layers.52.mlp.experts.145.down_proj", "model.layers.52.mlp.experts.146.down_proj", "model.layers.52.mlp.experts.147.down_proj", "model.layers.52.mlp.experts.148.down_proj", "model.layers.52.mlp.experts.149.down_proj", "model.layers.52.mlp.experts.150.down_proj", "model.layers.52.mlp.experts.151.down_proj", "model.layers.52.mlp.experts.152.down_proj", "model.layers.52.mlp.experts.153.down_proj", "model.layers.52.mlp.experts.154.down_proj", "model.layers.52.mlp.experts.155.down_proj", "model.layers.52.mlp.experts.156.down_proj", "model.layers.52.mlp.experts.157.down_proj", "model.layers.52.mlp.experts.158.down_proj", "model.layers.52.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006991431117057689, "dbits": 1258291200 } ] }, { "idx": 315, "layers": [ "model.layers.53.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0008858934044838063, "dbits": 62914560 } ] }, { "idx": 316, "layers": [ "model.layers.53.self_attn.k_proj", "model.layers.53.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0017427749931812064, "dbits": 10485760 } ] }, { "idx": 317, "layers": [ "model.layers.53.self_attn.o_proj" ], "candidates": [ { "dkld": -5.222633481027916e-05, "dbits": 62914560 } ] }, { "idx": 318, "layers": [ "model.layers.53.mlp.shared_experts.gate_proj", "model.layers.53.mlp.shared_experts.up_proj", "model.layers.53.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0003148697316646687, "dbits": 23592960 } ] }, { "idx": 319, "layers": [ "model.layers.53.mlp.experts.0.gate_proj", "model.layers.53.mlp.experts.1.gate_proj", "model.layers.53.mlp.experts.2.gate_proj", "model.layers.53.mlp.experts.3.gate_proj", "model.layers.53.mlp.experts.4.gate_proj", "model.layers.53.mlp.experts.5.gate_proj", "model.layers.53.mlp.experts.6.gate_proj", "model.layers.53.mlp.experts.7.gate_proj", "model.layers.53.mlp.experts.8.gate_proj", "model.layers.53.mlp.experts.9.gate_proj", "model.layers.53.mlp.experts.10.gate_proj", "model.layers.53.mlp.experts.11.gate_proj", "model.layers.53.mlp.experts.12.gate_proj", "model.layers.53.mlp.experts.13.gate_proj", "model.layers.53.mlp.experts.14.gate_proj", "model.layers.53.mlp.experts.15.gate_proj", "model.layers.53.mlp.experts.16.gate_proj", "model.layers.53.mlp.experts.17.gate_proj", "model.layers.53.mlp.experts.18.gate_proj", "model.layers.53.mlp.experts.19.gate_proj", "model.layers.53.mlp.experts.20.gate_proj", "model.layers.53.mlp.experts.21.gate_proj", "model.layers.53.mlp.experts.22.gate_proj", "model.layers.53.mlp.experts.23.gate_proj", "model.layers.53.mlp.experts.24.gate_proj", "model.layers.53.mlp.experts.25.gate_proj", "model.layers.53.mlp.experts.26.gate_proj", "model.layers.53.mlp.experts.27.gate_proj", "model.layers.53.mlp.experts.28.gate_proj", "model.layers.53.mlp.experts.29.gate_proj", "model.layers.53.mlp.experts.30.gate_proj", "model.layers.53.mlp.experts.31.gate_proj", "model.layers.53.mlp.experts.32.gate_proj", "model.layers.53.mlp.experts.33.gate_proj", "model.layers.53.mlp.experts.34.gate_proj", "model.layers.53.mlp.experts.35.gate_proj", "model.layers.53.mlp.experts.36.gate_proj", "model.layers.53.mlp.experts.37.gate_proj", "model.layers.53.mlp.experts.38.gate_proj", "model.layers.53.mlp.experts.39.gate_proj", "model.layers.53.mlp.experts.40.gate_proj", "model.layers.53.mlp.experts.41.gate_proj", "model.layers.53.mlp.experts.42.gate_proj", "model.layers.53.mlp.experts.43.gate_proj", "model.layers.53.mlp.experts.44.gate_proj", "model.layers.53.mlp.experts.45.gate_proj", "model.layers.53.mlp.experts.46.gate_proj", "model.layers.53.mlp.experts.47.gate_proj", "model.layers.53.mlp.experts.48.gate_proj", "model.layers.53.mlp.experts.49.gate_proj", "model.layers.53.mlp.experts.50.gate_proj", "model.layers.53.mlp.experts.51.gate_proj", "model.layers.53.mlp.experts.52.gate_proj", "model.layers.53.mlp.experts.53.gate_proj", "model.layers.53.mlp.experts.54.gate_proj", "model.layers.53.mlp.experts.55.gate_proj", "model.layers.53.mlp.experts.56.gate_proj", "model.layers.53.mlp.experts.57.gate_proj", "model.layers.53.mlp.experts.58.gate_proj", "model.layers.53.mlp.experts.59.gate_proj", "model.layers.53.mlp.experts.60.gate_proj", "model.layers.53.mlp.experts.61.gate_proj", "model.layers.53.mlp.experts.62.gate_proj", "model.layers.53.mlp.experts.63.gate_proj", "model.layers.53.mlp.experts.64.gate_proj", "model.layers.53.mlp.experts.65.gate_proj", "model.layers.53.mlp.experts.66.gate_proj", "model.layers.53.mlp.experts.67.gate_proj", "model.layers.53.mlp.experts.68.gate_proj", "model.layers.53.mlp.experts.69.gate_proj", "model.layers.53.mlp.experts.70.gate_proj", "model.layers.53.mlp.experts.71.gate_proj", "model.layers.53.mlp.experts.72.gate_proj", "model.layers.53.mlp.experts.73.gate_proj", "model.layers.53.mlp.experts.74.gate_proj", "model.layers.53.mlp.experts.75.gate_proj", "model.layers.53.mlp.experts.76.gate_proj", "model.layers.53.mlp.experts.77.gate_proj", "model.layers.53.mlp.experts.78.gate_proj", "model.layers.53.mlp.experts.79.gate_proj", "model.layers.53.mlp.experts.80.gate_proj", "model.layers.53.mlp.experts.81.gate_proj", "model.layers.53.mlp.experts.82.gate_proj", "model.layers.53.mlp.experts.83.gate_proj", "model.layers.53.mlp.experts.84.gate_proj", "model.layers.53.mlp.experts.85.gate_proj", "model.layers.53.mlp.experts.86.gate_proj", "model.layers.53.mlp.experts.87.gate_proj", "model.layers.53.mlp.experts.88.gate_proj", "model.layers.53.mlp.experts.89.gate_proj", "model.layers.53.mlp.experts.90.gate_proj", "model.layers.53.mlp.experts.91.gate_proj", "model.layers.53.mlp.experts.92.gate_proj", "model.layers.53.mlp.experts.93.gate_proj", "model.layers.53.mlp.experts.94.gate_proj", "model.layers.53.mlp.experts.95.gate_proj", "model.layers.53.mlp.experts.96.gate_proj", "model.layers.53.mlp.experts.97.gate_proj", "model.layers.53.mlp.experts.98.gate_proj", "model.layers.53.mlp.experts.99.gate_proj", "model.layers.53.mlp.experts.100.gate_proj", "model.layers.53.mlp.experts.101.gate_proj", "model.layers.53.mlp.experts.102.gate_proj", "model.layers.53.mlp.experts.103.gate_proj", "model.layers.53.mlp.experts.104.gate_proj", "model.layers.53.mlp.experts.105.gate_proj", "model.layers.53.mlp.experts.106.gate_proj", "model.layers.53.mlp.experts.107.gate_proj", "model.layers.53.mlp.experts.108.gate_proj", "model.layers.53.mlp.experts.109.gate_proj", "model.layers.53.mlp.experts.110.gate_proj", "model.layers.53.mlp.experts.111.gate_proj", "model.layers.53.mlp.experts.112.gate_proj", "model.layers.53.mlp.experts.113.gate_proj", "model.layers.53.mlp.experts.114.gate_proj", "model.layers.53.mlp.experts.115.gate_proj", "model.layers.53.mlp.experts.116.gate_proj", "model.layers.53.mlp.experts.117.gate_proj", "model.layers.53.mlp.experts.118.gate_proj", "model.layers.53.mlp.experts.119.gate_proj", "model.layers.53.mlp.experts.120.gate_proj", "model.layers.53.mlp.experts.121.gate_proj", "model.layers.53.mlp.experts.122.gate_proj", "model.layers.53.mlp.experts.123.gate_proj", "model.layers.53.mlp.experts.124.gate_proj", "model.layers.53.mlp.experts.125.gate_proj", "model.layers.53.mlp.experts.126.gate_proj", "model.layers.53.mlp.experts.127.gate_proj", "model.layers.53.mlp.experts.128.gate_proj", "model.layers.53.mlp.experts.129.gate_proj", "model.layers.53.mlp.experts.130.gate_proj", "model.layers.53.mlp.experts.131.gate_proj", "model.layers.53.mlp.experts.132.gate_proj", "model.layers.53.mlp.experts.133.gate_proj", "model.layers.53.mlp.experts.134.gate_proj", "model.layers.53.mlp.experts.135.gate_proj", "model.layers.53.mlp.experts.136.gate_proj", "model.layers.53.mlp.experts.137.gate_proj", "model.layers.53.mlp.experts.138.gate_proj", "model.layers.53.mlp.experts.139.gate_proj", "model.layers.53.mlp.experts.140.gate_proj", "model.layers.53.mlp.experts.141.gate_proj", "model.layers.53.mlp.experts.142.gate_proj", "model.layers.53.mlp.experts.143.gate_proj", "model.layers.53.mlp.experts.144.gate_proj", "model.layers.53.mlp.experts.145.gate_proj", "model.layers.53.mlp.experts.146.gate_proj", "model.layers.53.mlp.experts.147.gate_proj", "model.layers.53.mlp.experts.148.gate_proj", "model.layers.53.mlp.experts.149.gate_proj", "model.layers.53.mlp.experts.150.gate_proj", "model.layers.53.mlp.experts.151.gate_proj", "model.layers.53.mlp.experts.152.gate_proj", "model.layers.53.mlp.experts.153.gate_proj", "model.layers.53.mlp.experts.154.gate_proj", "model.layers.53.mlp.experts.155.gate_proj", "model.layers.53.mlp.experts.156.gate_proj", "model.layers.53.mlp.experts.157.gate_proj", "model.layers.53.mlp.experts.158.gate_proj", "model.layers.53.mlp.experts.159.gate_proj", "model.layers.53.mlp.experts.0.up_proj", "model.layers.53.mlp.experts.1.up_proj", "model.layers.53.mlp.experts.2.up_proj", "model.layers.53.mlp.experts.3.up_proj", "model.layers.53.mlp.experts.4.up_proj", "model.layers.53.mlp.experts.5.up_proj", "model.layers.53.mlp.experts.6.up_proj", "model.layers.53.mlp.experts.7.up_proj", "model.layers.53.mlp.experts.8.up_proj", "model.layers.53.mlp.experts.9.up_proj", "model.layers.53.mlp.experts.10.up_proj", "model.layers.53.mlp.experts.11.up_proj", "model.layers.53.mlp.experts.12.up_proj", "model.layers.53.mlp.experts.13.up_proj", "model.layers.53.mlp.experts.14.up_proj", "model.layers.53.mlp.experts.15.up_proj", "model.layers.53.mlp.experts.16.up_proj", "model.layers.53.mlp.experts.17.up_proj", "model.layers.53.mlp.experts.18.up_proj", "model.layers.53.mlp.experts.19.up_proj", "model.layers.53.mlp.experts.20.up_proj", "model.layers.53.mlp.experts.21.up_proj", "model.layers.53.mlp.experts.22.up_proj", "model.layers.53.mlp.experts.23.up_proj", "model.layers.53.mlp.experts.24.up_proj", "model.layers.53.mlp.experts.25.up_proj", "model.layers.53.mlp.experts.26.up_proj", "model.layers.53.mlp.experts.27.up_proj", "model.layers.53.mlp.experts.28.up_proj", "model.layers.53.mlp.experts.29.up_proj", "model.layers.53.mlp.experts.30.up_proj", "model.layers.53.mlp.experts.31.up_proj", "model.layers.53.mlp.experts.32.up_proj", "model.layers.53.mlp.experts.33.up_proj", "model.layers.53.mlp.experts.34.up_proj", "model.layers.53.mlp.experts.35.up_proj", "model.layers.53.mlp.experts.36.up_proj", "model.layers.53.mlp.experts.37.up_proj", "model.layers.53.mlp.experts.38.up_proj", "model.layers.53.mlp.experts.39.up_proj", "model.layers.53.mlp.experts.40.up_proj", "model.layers.53.mlp.experts.41.up_proj", "model.layers.53.mlp.experts.42.up_proj", "model.layers.53.mlp.experts.43.up_proj", "model.layers.53.mlp.experts.44.up_proj", "model.layers.53.mlp.experts.45.up_proj", "model.layers.53.mlp.experts.46.up_proj", "model.layers.53.mlp.experts.47.up_proj", "model.layers.53.mlp.experts.48.up_proj", "model.layers.53.mlp.experts.49.up_proj", "model.layers.53.mlp.experts.50.up_proj", "model.layers.53.mlp.experts.51.up_proj", "model.layers.53.mlp.experts.52.up_proj", "model.layers.53.mlp.experts.53.up_proj", "model.layers.53.mlp.experts.54.up_proj", "model.layers.53.mlp.experts.55.up_proj", "model.layers.53.mlp.experts.56.up_proj", "model.layers.53.mlp.experts.57.up_proj", "model.layers.53.mlp.experts.58.up_proj", "model.layers.53.mlp.experts.59.up_proj", "model.layers.53.mlp.experts.60.up_proj", "model.layers.53.mlp.experts.61.up_proj", "model.layers.53.mlp.experts.62.up_proj", "model.layers.53.mlp.experts.63.up_proj", "model.layers.53.mlp.experts.64.up_proj", "model.layers.53.mlp.experts.65.up_proj", "model.layers.53.mlp.experts.66.up_proj", "model.layers.53.mlp.experts.67.up_proj", "model.layers.53.mlp.experts.68.up_proj", "model.layers.53.mlp.experts.69.up_proj", "model.layers.53.mlp.experts.70.up_proj", "model.layers.53.mlp.experts.71.up_proj", "model.layers.53.mlp.experts.72.up_proj", "model.layers.53.mlp.experts.73.up_proj", "model.layers.53.mlp.experts.74.up_proj", "model.layers.53.mlp.experts.75.up_proj", "model.layers.53.mlp.experts.76.up_proj", "model.layers.53.mlp.experts.77.up_proj", "model.layers.53.mlp.experts.78.up_proj", "model.layers.53.mlp.experts.79.up_proj", "model.layers.53.mlp.experts.80.up_proj", "model.layers.53.mlp.experts.81.up_proj", "model.layers.53.mlp.experts.82.up_proj", "model.layers.53.mlp.experts.83.up_proj", "model.layers.53.mlp.experts.84.up_proj", "model.layers.53.mlp.experts.85.up_proj", "model.layers.53.mlp.experts.86.up_proj", "model.layers.53.mlp.experts.87.up_proj", "model.layers.53.mlp.experts.88.up_proj", "model.layers.53.mlp.experts.89.up_proj", "model.layers.53.mlp.experts.90.up_proj", "model.layers.53.mlp.experts.91.up_proj", "model.layers.53.mlp.experts.92.up_proj", "model.layers.53.mlp.experts.93.up_proj", "model.layers.53.mlp.experts.94.up_proj", "model.layers.53.mlp.experts.95.up_proj", "model.layers.53.mlp.experts.96.up_proj", "model.layers.53.mlp.experts.97.up_proj", "model.layers.53.mlp.experts.98.up_proj", "model.layers.53.mlp.experts.99.up_proj", "model.layers.53.mlp.experts.100.up_proj", "model.layers.53.mlp.experts.101.up_proj", "model.layers.53.mlp.experts.102.up_proj", "model.layers.53.mlp.experts.103.up_proj", "model.layers.53.mlp.experts.104.up_proj", "model.layers.53.mlp.experts.105.up_proj", "model.layers.53.mlp.experts.106.up_proj", "model.layers.53.mlp.experts.107.up_proj", "model.layers.53.mlp.experts.108.up_proj", "model.layers.53.mlp.experts.109.up_proj", "model.layers.53.mlp.experts.110.up_proj", "model.layers.53.mlp.experts.111.up_proj", "model.layers.53.mlp.experts.112.up_proj", "model.layers.53.mlp.experts.113.up_proj", "model.layers.53.mlp.experts.114.up_proj", "model.layers.53.mlp.experts.115.up_proj", "model.layers.53.mlp.experts.116.up_proj", "model.layers.53.mlp.experts.117.up_proj", "model.layers.53.mlp.experts.118.up_proj", "model.layers.53.mlp.experts.119.up_proj", "model.layers.53.mlp.experts.120.up_proj", "model.layers.53.mlp.experts.121.up_proj", "model.layers.53.mlp.experts.122.up_proj", "model.layers.53.mlp.experts.123.up_proj", "model.layers.53.mlp.experts.124.up_proj", "model.layers.53.mlp.experts.125.up_proj", "model.layers.53.mlp.experts.126.up_proj", "model.layers.53.mlp.experts.127.up_proj", "model.layers.53.mlp.experts.128.up_proj", "model.layers.53.mlp.experts.129.up_proj", "model.layers.53.mlp.experts.130.up_proj", "model.layers.53.mlp.experts.131.up_proj", "model.layers.53.mlp.experts.132.up_proj", "model.layers.53.mlp.experts.133.up_proj", "model.layers.53.mlp.experts.134.up_proj", "model.layers.53.mlp.experts.135.up_proj", "model.layers.53.mlp.experts.136.up_proj", "model.layers.53.mlp.experts.137.up_proj", "model.layers.53.mlp.experts.138.up_proj", "model.layers.53.mlp.experts.139.up_proj", "model.layers.53.mlp.experts.140.up_proj", "model.layers.53.mlp.experts.141.up_proj", "model.layers.53.mlp.experts.142.up_proj", "model.layers.53.mlp.experts.143.up_proj", "model.layers.53.mlp.experts.144.up_proj", "model.layers.53.mlp.experts.145.up_proj", "model.layers.53.mlp.experts.146.up_proj", "model.layers.53.mlp.experts.147.up_proj", "model.layers.53.mlp.experts.148.up_proj", "model.layers.53.mlp.experts.149.up_proj", "model.layers.53.mlp.experts.150.up_proj", "model.layers.53.mlp.experts.151.up_proj", "model.layers.53.mlp.experts.152.up_proj", "model.layers.53.mlp.experts.153.up_proj", "model.layers.53.mlp.experts.154.up_proj", "model.layers.53.mlp.experts.155.up_proj", "model.layers.53.mlp.experts.156.up_proj", "model.layers.53.mlp.experts.157.up_proj", "model.layers.53.mlp.experts.158.up_proj", "model.layers.53.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0007080405950546376, "dbits": 2516582400 } ] }, { "idx": 320, "layers": [ "model.layers.53.mlp.experts.0.down_proj", "model.layers.53.mlp.experts.1.down_proj", "model.layers.53.mlp.experts.2.down_proj", "model.layers.53.mlp.experts.3.down_proj", "model.layers.53.mlp.experts.4.down_proj", "model.layers.53.mlp.experts.5.down_proj", "model.layers.53.mlp.experts.6.down_proj", "model.layers.53.mlp.experts.7.down_proj", "model.layers.53.mlp.experts.8.down_proj", "model.layers.53.mlp.experts.9.down_proj", "model.layers.53.mlp.experts.10.down_proj", "model.layers.53.mlp.experts.11.down_proj", "model.layers.53.mlp.experts.12.down_proj", "model.layers.53.mlp.experts.13.down_proj", "model.layers.53.mlp.experts.14.down_proj", "model.layers.53.mlp.experts.15.down_proj", "model.layers.53.mlp.experts.16.down_proj", "model.layers.53.mlp.experts.17.down_proj", "model.layers.53.mlp.experts.18.down_proj", "model.layers.53.mlp.experts.19.down_proj", "model.layers.53.mlp.experts.20.down_proj", "model.layers.53.mlp.experts.21.down_proj", "model.layers.53.mlp.experts.22.down_proj", "model.layers.53.mlp.experts.23.down_proj", "model.layers.53.mlp.experts.24.down_proj", "model.layers.53.mlp.experts.25.down_proj", "model.layers.53.mlp.experts.26.down_proj", "model.layers.53.mlp.experts.27.down_proj", "model.layers.53.mlp.experts.28.down_proj", "model.layers.53.mlp.experts.29.down_proj", "model.layers.53.mlp.experts.30.down_proj", "model.layers.53.mlp.experts.31.down_proj", "model.layers.53.mlp.experts.32.down_proj", "model.layers.53.mlp.experts.33.down_proj", "model.layers.53.mlp.experts.34.down_proj", "model.layers.53.mlp.experts.35.down_proj", "model.layers.53.mlp.experts.36.down_proj", "model.layers.53.mlp.experts.37.down_proj", "model.layers.53.mlp.experts.38.down_proj", "model.layers.53.mlp.experts.39.down_proj", "model.layers.53.mlp.experts.40.down_proj", "model.layers.53.mlp.experts.41.down_proj", "model.layers.53.mlp.experts.42.down_proj", "model.layers.53.mlp.experts.43.down_proj", "model.layers.53.mlp.experts.44.down_proj", "model.layers.53.mlp.experts.45.down_proj", "model.layers.53.mlp.experts.46.down_proj", "model.layers.53.mlp.experts.47.down_proj", "model.layers.53.mlp.experts.48.down_proj", "model.layers.53.mlp.experts.49.down_proj", "model.layers.53.mlp.experts.50.down_proj", "model.layers.53.mlp.experts.51.down_proj", "model.layers.53.mlp.experts.52.down_proj", "model.layers.53.mlp.experts.53.down_proj", "model.layers.53.mlp.experts.54.down_proj", "model.layers.53.mlp.experts.55.down_proj", "model.layers.53.mlp.experts.56.down_proj", "model.layers.53.mlp.experts.57.down_proj", "model.layers.53.mlp.experts.58.down_proj", "model.layers.53.mlp.experts.59.down_proj", "model.layers.53.mlp.experts.60.down_proj", "model.layers.53.mlp.experts.61.down_proj", "model.layers.53.mlp.experts.62.down_proj", "model.layers.53.mlp.experts.63.down_proj", "model.layers.53.mlp.experts.64.down_proj", "model.layers.53.mlp.experts.65.down_proj", "model.layers.53.mlp.experts.66.down_proj", "model.layers.53.mlp.experts.67.down_proj", "model.layers.53.mlp.experts.68.down_proj", "model.layers.53.mlp.experts.69.down_proj", "model.layers.53.mlp.experts.70.down_proj", "model.layers.53.mlp.experts.71.down_proj", "model.layers.53.mlp.experts.72.down_proj", "model.layers.53.mlp.experts.73.down_proj", "model.layers.53.mlp.experts.74.down_proj", "model.layers.53.mlp.experts.75.down_proj", "model.layers.53.mlp.experts.76.down_proj", "model.layers.53.mlp.experts.77.down_proj", "model.layers.53.mlp.experts.78.down_proj", "model.layers.53.mlp.experts.79.down_proj", "model.layers.53.mlp.experts.80.down_proj", "model.layers.53.mlp.experts.81.down_proj", "model.layers.53.mlp.experts.82.down_proj", "model.layers.53.mlp.experts.83.down_proj", "model.layers.53.mlp.experts.84.down_proj", "model.layers.53.mlp.experts.85.down_proj", "model.layers.53.mlp.experts.86.down_proj", "model.layers.53.mlp.experts.87.down_proj", "model.layers.53.mlp.experts.88.down_proj", "model.layers.53.mlp.experts.89.down_proj", "model.layers.53.mlp.experts.90.down_proj", "model.layers.53.mlp.experts.91.down_proj", "model.layers.53.mlp.experts.92.down_proj", "model.layers.53.mlp.experts.93.down_proj", "model.layers.53.mlp.experts.94.down_proj", "model.layers.53.mlp.experts.95.down_proj", "model.layers.53.mlp.experts.96.down_proj", "model.layers.53.mlp.experts.97.down_proj", "model.layers.53.mlp.experts.98.down_proj", "model.layers.53.mlp.experts.99.down_proj", "model.layers.53.mlp.experts.100.down_proj", "model.layers.53.mlp.experts.101.down_proj", "model.layers.53.mlp.experts.102.down_proj", "model.layers.53.mlp.experts.103.down_proj", "model.layers.53.mlp.experts.104.down_proj", "model.layers.53.mlp.experts.105.down_proj", "model.layers.53.mlp.experts.106.down_proj", "model.layers.53.mlp.experts.107.down_proj", "model.layers.53.mlp.experts.108.down_proj", "model.layers.53.mlp.experts.109.down_proj", "model.layers.53.mlp.experts.110.down_proj", "model.layers.53.mlp.experts.111.down_proj", "model.layers.53.mlp.experts.112.down_proj", "model.layers.53.mlp.experts.113.down_proj", "model.layers.53.mlp.experts.114.down_proj", "model.layers.53.mlp.experts.115.down_proj", "model.layers.53.mlp.experts.116.down_proj", "model.layers.53.mlp.experts.117.down_proj", "model.layers.53.mlp.experts.118.down_proj", "model.layers.53.mlp.experts.119.down_proj", "model.layers.53.mlp.experts.120.down_proj", "model.layers.53.mlp.experts.121.down_proj", "model.layers.53.mlp.experts.122.down_proj", "model.layers.53.mlp.experts.123.down_proj", "model.layers.53.mlp.experts.124.down_proj", "model.layers.53.mlp.experts.125.down_proj", "model.layers.53.mlp.experts.126.down_proj", "model.layers.53.mlp.experts.127.down_proj", "model.layers.53.mlp.experts.128.down_proj", "model.layers.53.mlp.experts.129.down_proj", "model.layers.53.mlp.experts.130.down_proj", "model.layers.53.mlp.experts.131.down_proj", "model.layers.53.mlp.experts.132.down_proj", "model.layers.53.mlp.experts.133.down_proj", "model.layers.53.mlp.experts.134.down_proj", "model.layers.53.mlp.experts.135.down_proj", "model.layers.53.mlp.experts.136.down_proj", "model.layers.53.mlp.experts.137.down_proj", "model.layers.53.mlp.experts.138.down_proj", "model.layers.53.mlp.experts.139.down_proj", "model.layers.53.mlp.experts.140.down_proj", "model.layers.53.mlp.experts.141.down_proj", "model.layers.53.mlp.experts.142.down_proj", "model.layers.53.mlp.experts.143.down_proj", "model.layers.53.mlp.experts.144.down_proj", "model.layers.53.mlp.experts.145.down_proj", "model.layers.53.mlp.experts.146.down_proj", "model.layers.53.mlp.experts.147.down_proj", "model.layers.53.mlp.experts.148.down_proj", "model.layers.53.mlp.experts.149.down_proj", "model.layers.53.mlp.experts.150.down_proj", "model.layers.53.mlp.experts.151.down_proj", "model.layers.53.mlp.experts.152.down_proj", "model.layers.53.mlp.experts.153.down_proj", "model.layers.53.mlp.experts.154.down_proj", "model.layers.53.mlp.experts.155.down_proj", "model.layers.53.mlp.experts.156.down_proj", "model.layers.53.mlp.experts.157.down_proj", "model.layers.53.mlp.experts.158.down_proj", "model.layers.53.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006081290543079598, "dbits": 1258291200 } ] }, { "idx": 321, "layers": [ "model.layers.54.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0014413036406040192, "dbits": 62914560 } ] }, { "idx": 322, "layers": [ "model.layers.54.self_attn.k_proj", "model.layers.54.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0013847142457962258, "dbits": 10485760 } ] }, { "idx": 323, "layers": [ "model.layers.54.self_attn.o_proj" ], "candidates": [ { "dkld": -0.002720507234334979, "dbits": 62914560 } ] }, { "idx": 324, "layers": [ "model.layers.54.mlp.shared_experts.gate_proj", "model.layers.54.mlp.shared_experts.up_proj", "model.layers.54.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00011642128229144566, "dbits": 23592960 } ] }, { "idx": 325, "layers": [ "model.layers.54.mlp.experts.0.gate_proj", "model.layers.54.mlp.experts.1.gate_proj", "model.layers.54.mlp.experts.2.gate_proj", "model.layers.54.mlp.experts.3.gate_proj", "model.layers.54.mlp.experts.4.gate_proj", "model.layers.54.mlp.experts.5.gate_proj", "model.layers.54.mlp.experts.6.gate_proj", "model.layers.54.mlp.experts.7.gate_proj", "model.layers.54.mlp.experts.8.gate_proj", "model.layers.54.mlp.experts.9.gate_proj", "model.layers.54.mlp.experts.10.gate_proj", "model.layers.54.mlp.experts.11.gate_proj", "model.layers.54.mlp.experts.12.gate_proj", "model.layers.54.mlp.experts.13.gate_proj", "model.layers.54.mlp.experts.14.gate_proj", "model.layers.54.mlp.experts.15.gate_proj", "model.layers.54.mlp.experts.16.gate_proj", "model.layers.54.mlp.experts.17.gate_proj", "model.layers.54.mlp.experts.18.gate_proj", "model.layers.54.mlp.experts.19.gate_proj", "model.layers.54.mlp.experts.20.gate_proj", "model.layers.54.mlp.experts.21.gate_proj", "model.layers.54.mlp.experts.22.gate_proj", "model.layers.54.mlp.experts.23.gate_proj", "model.layers.54.mlp.experts.24.gate_proj", "model.layers.54.mlp.experts.25.gate_proj", "model.layers.54.mlp.experts.26.gate_proj", "model.layers.54.mlp.experts.27.gate_proj", "model.layers.54.mlp.experts.28.gate_proj", "model.layers.54.mlp.experts.29.gate_proj", "model.layers.54.mlp.experts.30.gate_proj", "model.layers.54.mlp.experts.31.gate_proj", "model.layers.54.mlp.experts.32.gate_proj", "model.layers.54.mlp.experts.33.gate_proj", "model.layers.54.mlp.experts.34.gate_proj", "model.layers.54.mlp.experts.35.gate_proj", "model.layers.54.mlp.experts.36.gate_proj", "model.layers.54.mlp.experts.37.gate_proj", "model.layers.54.mlp.experts.38.gate_proj", "model.layers.54.mlp.experts.39.gate_proj", "model.layers.54.mlp.experts.40.gate_proj", "model.layers.54.mlp.experts.41.gate_proj", "model.layers.54.mlp.experts.42.gate_proj", "model.layers.54.mlp.experts.43.gate_proj", "model.layers.54.mlp.experts.44.gate_proj", "model.layers.54.mlp.experts.45.gate_proj", "model.layers.54.mlp.experts.46.gate_proj", "model.layers.54.mlp.experts.47.gate_proj", "model.layers.54.mlp.experts.48.gate_proj", "model.layers.54.mlp.experts.49.gate_proj", "model.layers.54.mlp.experts.50.gate_proj", "model.layers.54.mlp.experts.51.gate_proj", "model.layers.54.mlp.experts.52.gate_proj", "model.layers.54.mlp.experts.53.gate_proj", "model.layers.54.mlp.experts.54.gate_proj", "model.layers.54.mlp.experts.55.gate_proj", "model.layers.54.mlp.experts.56.gate_proj", "model.layers.54.mlp.experts.57.gate_proj", "model.layers.54.mlp.experts.58.gate_proj", "model.layers.54.mlp.experts.59.gate_proj", "model.layers.54.mlp.experts.60.gate_proj", "model.layers.54.mlp.experts.61.gate_proj", "model.layers.54.mlp.experts.62.gate_proj", "model.layers.54.mlp.experts.63.gate_proj", "model.layers.54.mlp.experts.64.gate_proj", "model.layers.54.mlp.experts.65.gate_proj", "model.layers.54.mlp.experts.66.gate_proj", "model.layers.54.mlp.experts.67.gate_proj", "model.layers.54.mlp.experts.68.gate_proj", "model.layers.54.mlp.experts.69.gate_proj", "model.layers.54.mlp.experts.70.gate_proj", "model.layers.54.mlp.experts.71.gate_proj", "model.layers.54.mlp.experts.72.gate_proj", "model.layers.54.mlp.experts.73.gate_proj", "model.layers.54.mlp.experts.74.gate_proj", "model.layers.54.mlp.experts.75.gate_proj", "model.layers.54.mlp.experts.76.gate_proj", "model.layers.54.mlp.experts.77.gate_proj", "model.layers.54.mlp.experts.78.gate_proj", "model.layers.54.mlp.experts.79.gate_proj", "model.layers.54.mlp.experts.80.gate_proj", "model.layers.54.mlp.experts.81.gate_proj", "model.layers.54.mlp.experts.82.gate_proj", "model.layers.54.mlp.experts.83.gate_proj", "model.layers.54.mlp.experts.84.gate_proj", "model.layers.54.mlp.experts.85.gate_proj", "model.layers.54.mlp.experts.86.gate_proj", "model.layers.54.mlp.experts.87.gate_proj", "model.layers.54.mlp.experts.88.gate_proj", "model.layers.54.mlp.experts.89.gate_proj", "model.layers.54.mlp.experts.90.gate_proj", "model.layers.54.mlp.experts.91.gate_proj", "model.layers.54.mlp.experts.92.gate_proj", "model.layers.54.mlp.experts.93.gate_proj", "model.layers.54.mlp.experts.94.gate_proj", "model.layers.54.mlp.experts.95.gate_proj", "model.layers.54.mlp.experts.96.gate_proj", "model.layers.54.mlp.experts.97.gate_proj", "model.layers.54.mlp.experts.98.gate_proj", "model.layers.54.mlp.experts.99.gate_proj", "model.layers.54.mlp.experts.100.gate_proj", "model.layers.54.mlp.experts.101.gate_proj", "model.layers.54.mlp.experts.102.gate_proj", "model.layers.54.mlp.experts.103.gate_proj", "model.layers.54.mlp.experts.104.gate_proj", "model.layers.54.mlp.experts.105.gate_proj", "model.layers.54.mlp.experts.106.gate_proj", "model.layers.54.mlp.experts.107.gate_proj", "model.layers.54.mlp.experts.108.gate_proj", "model.layers.54.mlp.experts.109.gate_proj", "model.layers.54.mlp.experts.110.gate_proj", "model.layers.54.mlp.experts.111.gate_proj", "model.layers.54.mlp.experts.112.gate_proj", "model.layers.54.mlp.experts.113.gate_proj", "model.layers.54.mlp.experts.114.gate_proj", "model.layers.54.mlp.experts.115.gate_proj", "model.layers.54.mlp.experts.116.gate_proj", "model.layers.54.mlp.experts.117.gate_proj", "model.layers.54.mlp.experts.118.gate_proj", "model.layers.54.mlp.experts.119.gate_proj", "model.layers.54.mlp.experts.120.gate_proj", "model.layers.54.mlp.experts.121.gate_proj", "model.layers.54.mlp.experts.122.gate_proj", "model.layers.54.mlp.experts.123.gate_proj", "model.layers.54.mlp.experts.124.gate_proj", "model.layers.54.mlp.experts.125.gate_proj", "model.layers.54.mlp.experts.126.gate_proj", "model.layers.54.mlp.experts.127.gate_proj", "model.layers.54.mlp.experts.128.gate_proj", "model.layers.54.mlp.experts.129.gate_proj", "model.layers.54.mlp.experts.130.gate_proj", "model.layers.54.mlp.experts.131.gate_proj", "model.layers.54.mlp.experts.132.gate_proj", "model.layers.54.mlp.experts.133.gate_proj", "model.layers.54.mlp.experts.134.gate_proj", "model.layers.54.mlp.experts.135.gate_proj", "model.layers.54.mlp.experts.136.gate_proj", "model.layers.54.mlp.experts.137.gate_proj", "model.layers.54.mlp.experts.138.gate_proj", "model.layers.54.mlp.experts.139.gate_proj", "model.layers.54.mlp.experts.140.gate_proj", "model.layers.54.mlp.experts.141.gate_proj", "model.layers.54.mlp.experts.142.gate_proj", "model.layers.54.mlp.experts.143.gate_proj", "model.layers.54.mlp.experts.144.gate_proj", "model.layers.54.mlp.experts.145.gate_proj", "model.layers.54.mlp.experts.146.gate_proj", "model.layers.54.mlp.experts.147.gate_proj", "model.layers.54.mlp.experts.148.gate_proj", "model.layers.54.mlp.experts.149.gate_proj", "model.layers.54.mlp.experts.150.gate_proj", "model.layers.54.mlp.experts.151.gate_proj", "model.layers.54.mlp.experts.152.gate_proj", "model.layers.54.mlp.experts.153.gate_proj", "model.layers.54.mlp.experts.154.gate_proj", "model.layers.54.mlp.experts.155.gate_proj", "model.layers.54.mlp.experts.156.gate_proj", "model.layers.54.mlp.experts.157.gate_proj", "model.layers.54.mlp.experts.158.gate_proj", "model.layers.54.mlp.experts.159.gate_proj", "model.layers.54.mlp.experts.0.up_proj", "model.layers.54.mlp.experts.1.up_proj", "model.layers.54.mlp.experts.2.up_proj", "model.layers.54.mlp.experts.3.up_proj", "model.layers.54.mlp.experts.4.up_proj", "model.layers.54.mlp.experts.5.up_proj", "model.layers.54.mlp.experts.6.up_proj", "model.layers.54.mlp.experts.7.up_proj", "model.layers.54.mlp.experts.8.up_proj", "model.layers.54.mlp.experts.9.up_proj", "model.layers.54.mlp.experts.10.up_proj", "model.layers.54.mlp.experts.11.up_proj", "model.layers.54.mlp.experts.12.up_proj", "model.layers.54.mlp.experts.13.up_proj", "model.layers.54.mlp.experts.14.up_proj", "model.layers.54.mlp.experts.15.up_proj", "model.layers.54.mlp.experts.16.up_proj", "model.layers.54.mlp.experts.17.up_proj", "model.layers.54.mlp.experts.18.up_proj", "model.layers.54.mlp.experts.19.up_proj", "model.layers.54.mlp.experts.20.up_proj", "model.layers.54.mlp.experts.21.up_proj", "model.layers.54.mlp.experts.22.up_proj", "model.layers.54.mlp.experts.23.up_proj", "model.layers.54.mlp.experts.24.up_proj", "model.layers.54.mlp.experts.25.up_proj", "model.layers.54.mlp.experts.26.up_proj", "model.layers.54.mlp.experts.27.up_proj", "model.layers.54.mlp.experts.28.up_proj", "model.layers.54.mlp.experts.29.up_proj", "model.layers.54.mlp.experts.30.up_proj", "model.layers.54.mlp.experts.31.up_proj", "model.layers.54.mlp.experts.32.up_proj", "model.layers.54.mlp.experts.33.up_proj", "model.layers.54.mlp.experts.34.up_proj", "model.layers.54.mlp.experts.35.up_proj", "model.layers.54.mlp.experts.36.up_proj", "model.layers.54.mlp.experts.37.up_proj", "model.layers.54.mlp.experts.38.up_proj", "model.layers.54.mlp.experts.39.up_proj", "model.layers.54.mlp.experts.40.up_proj", "model.layers.54.mlp.experts.41.up_proj", "model.layers.54.mlp.experts.42.up_proj", "model.layers.54.mlp.experts.43.up_proj", "model.layers.54.mlp.experts.44.up_proj", "model.layers.54.mlp.experts.45.up_proj", "model.layers.54.mlp.experts.46.up_proj", "model.layers.54.mlp.experts.47.up_proj", "model.layers.54.mlp.experts.48.up_proj", "model.layers.54.mlp.experts.49.up_proj", "model.layers.54.mlp.experts.50.up_proj", "model.layers.54.mlp.experts.51.up_proj", "model.layers.54.mlp.experts.52.up_proj", "model.layers.54.mlp.experts.53.up_proj", "model.layers.54.mlp.experts.54.up_proj", "model.layers.54.mlp.experts.55.up_proj", "model.layers.54.mlp.experts.56.up_proj", "model.layers.54.mlp.experts.57.up_proj", "model.layers.54.mlp.experts.58.up_proj", "model.layers.54.mlp.experts.59.up_proj", "model.layers.54.mlp.experts.60.up_proj", "model.layers.54.mlp.experts.61.up_proj", "model.layers.54.mlp.experts.62.up_proj", "model.layers.54.mlp.experts.63.up_proj", "model.layers.54.mlp.experts.64.up_proj", "model.layers.54.mlp.experts.65.up_proj", "model.layers.54.mlp.experts.66.up_proj", "model.layers.54.mlp.experts.67.up_proj", "model.layers.54.mlp.experts.68.up_proj", "model.layers.54.mlp.experts.69.up_proj", "model.layers.54.mlp.experts.70.up_proj", "model.layers.54.mlp.experts.71.up_proj", "model.layers.54.mlp.experts.72.up_proj", "model.layers.54.mlp.experts.73.up_proj", "model.layers.54.mlp.experts.74.up_proj", "model.layers.54.mlp.experts.75.up_proj", "model.layers.54.mlp.experts.76.up_proj", "model.layers.54.mlp.experts.77.up_proj", "model.layers.54.mlp.experts.78.up_proj", "model.layers.54.mlp.experts.79.up_proj", "model.layers.54.mlp.experts.80.up_proj", "model.layers.54.mlp.experts.81.up_proj", "model.layers.54.mlp.experts.82.up_proj", "model.layers.54.mlp.experts.83.up_proj", "model.layers.54.mlp.experts.84.up_proj", "model.layers.54.mlp.experts.85.up_proj", "model.layers.54.mlp.experts.86.up_proj", "model.layers.54.mlp.experts.87.up_proj", "model.layers.54.mlp.experts.88.up_proj", "model.layers.54.mlp.experts.89.up_proj", "model.layers.54.mlp.experts.90.up_proj", "model.layers.54.mlp.experts.91.up_proj", "model.layers.54.mlp.experts.92.up_proj", "model.layers.54.mlp.experts.93.up_proj", "model.layers.54.mlp.experts.94.up_proj", "model.layers.54.mlp.experts.95.up_proj", "model.layers.54.mlp.experts.96.up_proj", "model.layers.54.mlp.experts.97.up_proj", "model.layers.54.mlp.experts.98.up_proj", "model.layers.54.mlp.experts.99.up_proj", "model.layers.54.mlp.experts.100.up_proj", "model.layers.54.mlp.experts.101.up_proj", "model.layers.54.mlp.experts.102.up_proj", "model.layers.54.mlp.experts.103.up_proj", "model.layers.54.mlp.experts.104.up_proj", "model.layers.54.mlp.experts.105.up_proj", "model.layers.54.mlp.experts.106.up_proj", "model.layers.54.mlp.experts.107.up_proj", "model.layers.54.mlp.experts.108.up_proj", "model.layers.54.mlp.experts.109.up_proj", "model.layers.54.mlp.experts.110.up_proj", "model.layers.54.mlp.experts.111.up_proj", "model.layers.54.mlp.experts.112.up_proj", "model.layers.54.mlp.experts.113.up_proj", "model.layers.54.mlp.experts.114.up_proj", "model.layers.54.mlp.experts.115.up_proj", "model.layers.54.mlp.experts.116.up_proj", "model.layers.54.mlp.experts.117.up_proj", "model.layers.54.mlp.experts.118.up_proj", "model.layers.54.mlp.experts.119.up_proj", "model.layers.54.mlp.experts.120.up_proj", "model.layers.54.mlp.experts.121.up_proj", "model.layers.54.mlp.experts.122.up_proj", "model.layers.54.mlp.experts.123.up_proj", "model.layers.54.mlp.experts.124.up_proj", "model.layers.54.mlp.experts.125.up_proj", "model.layers.54.mlp.experts.126.up_proj", "model.layers.54.mlp.experts.127.up_proj", "model.layers.54.mlp.experts.128.up_proj", "model.layers.54.mlp.experts.129.up_proj", "model.layers.54.mlp.experts.130.up_proj", "model.layers.54.mlp.experts.131.up_proj", "model.layers.54.mlp.experts.132.up_proj", "model.layers.54.mlp.experts.133.up_proj", "model.layers.54.mlp.experts.134.up_proj", "model.layers.54.mlp.experts.135.up_proj", "model.layers.54.mlp.experts.136.up_proj", "model.layers.54.mlp.experts.137.up_proj", "model.layers.54.mlp.experts.138.up_proj", "model.layers.54.mlp.experts.139.up_proj", "model.layers.54.mlp.experts.140.up_proj", "model.layers.54.mlp.experts.141.up_proj", "model.layers.54.mlp.experts.142.up_proj", "model.layers.54.mlp.experts.143.up_proj", "model.layers.54.mlp.experts.144.up_proj", "model.layers.54.mlp.experts.145.up_proj", "model.layers.54.mlp.experts.146.up_proj", "model.layers.54.mlp.experts.147.up_proj", "model.layers.54.mlp.experts.148.up_proj", "model.layers.54.mlp.experts.149.up_proj", "model.layers.54.mlp.experts.150.up_proj", "model.layers.54.mlp.experts.151.up_proj", "model.layers.54.mlp.experts.152.up_proj", "model.layers.54.mlp.experts.153.up_proj", "model.layers.54.mlp.experts.154.up_proj", "model.layers.54.mlp.experts.155.up_proj", "model.layers.54.mlp.experts.156.up_proj", "model.layers.54.mlp.experts.157.up_proj", "model.layers.54.mlp.experts.158.up_proj", "model.layers.54.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0007874459028244241, "dbits": 2516582400 } ] }, { "idx": 326, "layers": [ "model.layers.54.mlp.experts.0.down_proj", "model.layers.54.mlp.experts.1.down_proj", "model.layers.54.mlp.experts.2.down_proj", "model.layers.54.mlp.experts.3.down_proj", "model.layers.54.mlp.experts.4.down_proj", "model.layers.54.mlp.experts.5.down_proj", "model.layers.54.mlp.experts.6.down_proj", "model.layers.54.mlp.experts.7.down_proj", "model.layers.54.mlp.experts.8.down_proj", "model.layers.54.mlp.experts.9.down_proj", "model.layers.54.mlp.experts.10.down_proj", "model.layers.54.mlp.experts.11.down_proj", "model.layers.54.mlp.experts.12.down_proj", "model.layers.54.mlp.experts.13.down_proj", "model.layers.54.mlp.experts.14.down_proj", "model.layers.54.mlp.experts.15.down_proj", "model.layers.54.mlp.experts.16.down_proj", "model.layers.54.mlp.experts.17.down_proj", "model.layers.54.mlp.experts.18.down_proj", "model.layers.54.mlp.experts.19.down_proj", "model.layers.54.mlp.experts.20.down_proj", "model.layers.54.mlp.experts.21.down_proj", "model.layers.54.mlp.experts.22.down_proj", "model.layers.54.mlp.experts.23.down_proj", "model.layers.54.mlp.experts.24.down_proj", "model.layers.54.mlp.experts.25.down_proj", "model.layers.54.mlp.experts.26.down_proj", "model.layers.54.mlp.experts.27.down_proj", "model.layers.54.mlp.experts.28.down_proj", "model.layers.54.mlp.experts.29.down_proj", "model.layers.54.mlp.experts.30.down_proj", "model.layers.54.mlp.experts.31.down_proj", "model.layers.54.mlp.experts.32.down_proj", "model.layers.54.mlp.experts.33.down_proj", "model.layers.54.mlp.experts.34.down_proj", "model.layers.54.mlp.experts.35.down_proj", "model.layers.54.mlp.experts.36.down_proj", "model.layers.54.mlp.experts.37.down_proj", "model.layers.54.mlp.experts.38.down_proj", "model.layers.54.mlp.experts.39.down_proj", "model.layers.54.mlp.experts.40.down_proj", "model.layers.54.mlp.experts.41.down_proj", "model.layers.54.mlp.experts.42.down_proj", "model.layers.54.mlp.experts.43.down_proj", "model.layers.54.mlp.experts.44.down_proj", "model.layers.54.mlp.experts.45.down_proj", "model.layers.54.mlp.experts.46.down_proj", "model.layers.54.mlp.experts.47.down_proj", "model.layers.54.mlp.experts.48.down_proj", "model.layers.54.mlp.experts.49.down_proj", "model.layers.54.mlp.experts.50.down_proj", "model.layers.54.mlp.experts.51.down_proj", "model.layers.54.mlp.experts.52.down_proj", "model.layers.54.mlp.experts.53.down_proj", "model.layers.54.mlp.experts.54.down_proj", "model.layers.54.mlp.experts.55.down_proj", "model.layers.54.mlp.experts.56.down_proj", "model.layers.54.mlp.experts.57.down_proj", "model.layers.54.mlp.experts.58.down_proj", "model.layers.54.mlp.experts.59.down_proj", "model.layers.54.mlp.experts.60.down_proj", "model.layers.54.mlp.experts.61.down_proj", "model.layers.54.mlp.experts.62.down_proj", "model.layers.54.mlp.experts.63.down_proj", "model.layers.54.mlp.experts.64.down_proj", "model.layers.54.mlp.experts.65.down_proj", "model.layers.54.mlp.experts.66.down_proj", "model.layers.54.mlp.experts.67.down_proj", "model.layers.54.mlp.experts.68.down_proj", "model.layers.54.mlp.experts.69.down_proj", "model.layers.54.mlp.experts.70.down_proj", "model.layers.54.mlp.experts.71.down_proj", "model.layers.54.mlp.experts.72.down_proj", "model.layers.54.mlp.experts.73.down_proj", "model.layers.54.mlp.experts.74.down_proj", "model.layers.54.mlp.experts.75.down_proj", "model.layers.54.mlp.experts.76.down_proj", "model.layers.54.mlp.experts.77.down_proj", "model.layers.54.mlp.experts.78.down_proj", "model.layers.54.mlp.experts.79.down_proj", "model.layers.54.mlp.experts.80.down_proj", "model.layers.54.mlp.experts.81.down_proj", "model.layers.54.mlp.experts.82.down_proj", "model.layers.54.mlp.experts.83.down_proj", "model.layers.54.mlp.experts.84.down_proj", "model.layers.54.mlp.experts.85.down_proj", "model.layers.54.mlp.experts.86.down_proj", "model.layers.54.mlp.experts.87.down_proj", "model.layers.54.mlp.experts.88.down_proj", "model.layers.54.mlp.experts.89.down_proj", "model.layers.54.mlp.experts.90.down_proj", "model.layers.54.mlp.experts.91.down_proj", "model.layers.54.mlp.experts.92.down_proj", "model.layers.54.mlp.experts.93.down_proj", "model.layers.54.mlp.experts.94.down_proj", "model.layers.54.mlp.experts.95.down_proj", "model.layers.54.mlp.experts.96.down_proj", "model.layers.54.mlp.experts.97.down_proj", "model.layers.54.mlp.experts.98.down_proj", "model.layers.54.mlp.experts.99.down_proj", "model.layers.54.mlp.experts.100.down_proj", "model.layers.54.mlp.experts.101.down_proj", "model.layers.54.mlp.experts.102.down_proj", "model.layers.54.mlp.experts.103.down_proj", "model.layers.54.mlp.experts.104.down_proj", "model.layers.54.mlp.experts.105.down_proj", "model.layers.54.mlp.experts.106.down_proj", "model.layers.54.mlp.experts.107.down_proj", "model.layers.54.mlp.experts.108.down_proj", "model.layers.54.mlp.experts.109.down_proj", "model.layers.54.mlp.experts.110.down_proj", "model.layers.54.mlp.experts.111.down_proj", "model.layers.54.mlp.experts.112.down_proj", "model.layers.54.mlp.experts.113.down_proj", "model.layers.54.mlp.experts.114.down_proj", "model.layers.54.mlp.experts.115.down_proj", "model.layers.54.mlp.experts.116.down_proj", "model.layers.54.mlp.experts.117.down_proj", "model.layers.54.mlp.experts.118.down_proj", "model.layers.54.mlp.experts.119.down_proj", "model.layers.54.mlp.experts.120.down_proj", "model.layers.54.mlp.experts.121.down_proj", "model.layers.54.mlp.experts.122.down_proj", "model.layers.54.mlp.experts.123.down_proj", "model.layers.54.mlp.experts.124.down_proj", "model.layers.54.mlp.experts.125.down_proj", "model.layers.54.mlp.experts.126.down_proj", "model.layers.54.mlp.experts.127.down_proj", "model.layers.54.mlp.experts.128.down_proj", "model.layers.54.mlp.experts.129.down_proj", "model.layers.54.mlp.experts.130.down_proj", "model.layers.54.mlp.experts.131.down_proj", "model.layers.54.mlp.experts.132.down_proj", "model.layers.54.mlp.experts.133.down_proj", "model.layers.54.mlp.experts.134.down_proj", "model.layers.54.mlp.experts.135.down_proj", "model.layers.54.mlp.experts.136.down_proj", "model.layers.54.mlp.experts.137.down_proj", "model.layers.54.mlp.experts.138.down_proj", "model.layers.54.mlp.experts.139.down_proj", "model.layers.54.mlp.experts.140.down_proj", "model.layers.54.mlp.experts.141.down_proj", "model.layers.54.mlp.experts.142.down_proj", "model.layers.54.mlp.experts.143.down_proj", "model.layers.54.mlp.experts.144.down_proj", "model.layers.54.mlp.experts.145.down_proj", "model.layers.54.mlp.experts.146.down_proj", "model.layers.54.mlp.experts.147.down_proj", "model.layers.54.mlp.experts.148.down_proj", "model.layers.54.mlp.experts.149.down_proj", "model.layers.54.mlp.experts.150.down_proj", "model.layers.54.mlp.experts.151.down_proj", "model.layers.54.mlp.experts.152.down_proj", "model.layers.54.mlp.experts.153.down_proj", "model.layers.54.mlp.experts.154.down_proj", "model.layers.54.mlp.experts.155.down_proj", "model.layers.54.mlp.experts.156.down_proj", "model.layers.54.mlp.experts.157.down_proj", "model.layers.54.mlp.experts.158.down_proj", "model.layers.54.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0003643833100795968, "dbits": 1258291200 } ] }, { "idx": 327, "layers": [ "model.layers.55.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0016520783305167819, "dbits": 62914560 } ] }, { "idx": 328, "layers": [ "model.layers.55.self_attn.k_proj", "model.layers.55.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0011264942586421967, "dbits": 10485760 } ] }, { "idx": 329, "layers": [ "model.layers.55.self_attn.o_proj" ], "candidates": [ { "dkld": -0.000569809228181839, "dbits": 62914560 } ] }, { "idx": 330, "layers": [ "model.layers.55.mlp.shared_experts.gate_proj", "model.layers.55.mlp.shared_experts.up_proj", "model.layers.55.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0012850657105445973, "dbits": 23592960 } ] }, { "idx": 331, "layers": [ "model.layers.55.mlp.experts.0.gate_proj", "model.layers.55.mlp.experts.1.gate_proj", "model.layers.55.mlp.experts.2.gate_proj", "model.layers.55.mlp.experts.3.gate_proj", "model.layers.55.mlp.experts.4.gate_proj", "model.layers.55.mlp.experts.5.gate_proj", "model.layers.55.mlp.experts.6.gate_proj", "model.layers.55.mlp.experts.7.gate_proj", "model.layers.55.mlp.experts.8.gate_proj", "model.layers.55.mlp.experts.9.gate_proj", "model.layers.55.mlp.experts.10.gate_proj", "model.layers.55.mlp.experts.11.gate_proj", "model.layers.55.mlp.experts.12.gate_proj", "model.layers.55.mlp.experts.13.gate_proj", "model.layers.55.mlp.experts.14.gate_proj", "model.layers.55.mlp.experts.15.gate_proj", "model.layers.55.mlp.experts.16.gate_proj", "model.layers.55.mlp.experts.17.gate_proj", "model.layers.55.mlp.experts.18.gate_proj", "model.layers.55.mlp.experts.19.gate_proj", "model.layers.55.mlp.experts.20.gate_proj", "model.layers.55.mlp.experts.21.gate_proj", "model.layers.55.mlp.experts.22.gate_proj", "model.layers.55.mlp.experts.23.gate_proj", "model.layers.55.mlp.experts.24.gate_proj", "model.layers.55.mlp.experts.25.gate_proj", "model.layers.55.mlp.experts.26.gate_proj", "model.layers.55.mlp.experts.27.gate_proj", "model.layers.55.mlp.experts.28.gate_proj", "model.layers.55.mlp.experts.29.gate_proj", "model.layers.55.mlp.experts.30.gate_proj", "model.layers.55.mlp.experts.31.gate_proj", "model.layers.55.mlp.experts.32.gate_proj", "model.layers.55.mlp.experts.33.gate_proj", "model.layers.55.mlp.experts.34.gate_proj", "model.layers.55.mlp.experts.35.gate_proj", "model.layers.55.mlp.experts.36.gate_proj", "model.layers.55.mlp.experts.37.gate_proj", "model.layers.55.mlp.experts.38.gate_proj", "model.layers.55.mlp.experts.39.gate_proj", "model.layers.55.mlp.experts.40.gate_proj", "model.layers.55.mlp.experts.41.gate_proj", "model.layers.55.mlp.experts.42.gate_proj", "model.layers.55.mlp.experts.43.gate_proj", "model.layers.55.mlp.experts.44.gate_proj", "model.layers.55.mlp.experts.45.gate_proj", "model.layers.55.mlp.experts.46.gate_proj", "model.layers.55.mlp.experts.47.gate_proj", "model.layers.55.mlp.experts.48.gate_proj", "model.layers.55.mlp.experts.49.gate_proj", "model.layers.55.mlp.experts.50.gate_proj", "model.layers.55.mlp.experts.51.gate_proj", "model.layers.55.mlp.experts.52.gate_proj", "model.layers.55.mlp.experts.53.gate_proj", "model.layers.55.mlp.experts.54.gate_proj", "model.layers.55.mlp.experts.55.gate_proj", "model.layers.55.mlp.experts.56.gate_proj", "model.layers.55.mlp.experts.57.gate_proj", "model.layers.55.mlp.experts.58.gate_proj", "model.layers.55.mlp.experts.59.gate_proj", "model.layers.55.mlp.experts.60.gate_proj", "model.layers.55.mlp.experts.61.gate_proj", "model.layers.55.mlp.experts.62.gate_proj", "model.layers.55.mlp.experts.63.gate_proj", "model.layers.55.mlp.experts.64.gate_proj", "model.layers.55.mlp.experts.65.gate_proj", "model.layers.55.mlp.experts.66.gate_proj", "model.layers.55.mlp.experts.67.gate_proj", "model.layers.55.mlp.experts.68.gate_proj", "model.layers.55.mlp.experts.69.gate_proj", "model.layers.55.mlp.experts.70.gate_proj", "model.layers.55.mlp.experts.71.gate_proj", "model.layers.55.mlp.experts.72.gate_proj", "model.layers.55.mlp.experts.73.gate_proj", "model.layers.55.mlp.experts.74.gate_proj", "model.layers.55.mlp.experts.75.gate_proj", "model.layers.55.mlp.experts.76.gate_proj", "model.layers.55.mlp.experts.77.gate_proj", "model.layers.55.mlp.experts.78.gate_proj", "model.layers.55.mlp.experts.79.gate_proj", "model.layers.55.mlp.experts.80.gate_proj", "model.layers.55.mlp.experts.81.gate_proj", "model.layers.55.mlp.experts.82.gate_proj", "model.layers.55.mlp.experts.83.gate_proj", "model.layers.55.mlp.experts.84.gate_proj", "model.layers.55.mlp.experts.85.gate_proj", "model.layers.55.mlp.experts.86.gate_proj", "model.layers.55.mlp.experts.87.gate_proj", "model.layers.55.mlp.experts.88.gate_proj", "model.layers.55.mlp.experts.89.gate_proj", "model.layers.55.mlp.experts.90.gate_proj", "model.layers.55.mlp.experts.91.gate_proj", "model.layers.55.mlp.experts.92.gate_proj", "model.layers.55.mlp.experts.93.gate_proj", "model.layers.55.mlp.experts.94.gate_proj", "model.layers.55.mlp.experts.95.gate_proj", "model.layers.55.mlp.experts.96.gate_proj", "model.layers.55.mlp.experts.97.gate_proj", "model.layers.55.mlp.experts.98.gate_proj", "model.layers.55.mlp.experts.99.gate_proj", "model.layers.55.mlp.experts.100.gate_proj", "model.layers.55.mlp.experts.101.gate_proj", "model.layers.55.mlp.experts.102.gate_proj", "model.layers.55.mlp.experts.103.gate_proj", "model.layers.55.mlp.experts.104.gate_proj", "model.layers.55.mlp.experts.105.gate_proj", "model.layers.55.mlp.experts.106.gate_proj", "model.layers.55.mlp.experts.107.gate_proj", "model.layers.55.mlp.experts.108.gate_proj", "model.layers.55.mlp.experts.109.gate_proj", "model.layers.55.mlp.experts.110.gate_proj", "model.layers.55.mlp.experts.111.gate_proj", "model.layers.55.mlp.experts.112.gate_proj", "model.layers.55.mlp.experts.113.gate_proj", "model.layers.55.mlp.experts.114.gate_proj", "model.layers.55.mlp.experts.115.gate_proj", "model.layers.55.mlp.experts.116.gate_proj", "model.layers.55.mlp.experts.117.gate_proj", "model.layers.55.mlp.experts.118.gate_proj", "model.layers.55.mlp.experts.119.gate_proj", "model.layers.55.mlp.experts.120.gate_proj", "model.layers.55.mlp.experts.121.gate_proj", "model.layers.55.mlp.experts.122.gate_proj", "model.layers.55.mlp.experts.123.gate_proj", "model.layers.55.mlp.experts.124.gate_proj", "model.layers.55.mlp.experts.125.gate_proj", "model.layers.55.mlp.experts.126.gate_proj", "model.layers.55.mlp.experts.127.gate_proj", "model.layers.55.mlp.experts.128.gate_proj", "model.layers.55.mlp.experts.129.gate_proj", "model.layers.55.mlp.experts.130.gate_proj", "model.layers.55.mlp.experts.131.gate_proj", "model.layers.55.mlp.experts.132.gate_proj", "model.layers.55.mlp.experts.133.gate_proj", "model.layers.55.mlp.experts.134.gate_proj", "model.layers.55.mlp.experts.135.gate_proj", "model.layers.55.mlp.experts.136.gate_proj", "model.layers.55.mlp.experts.137.gate_proj", "model.layers.55.mlp.experts.138.gate_proj", "model.layers.55.mlp.experts.139.gate_proj", "model.layers.55.mlp.experts.140.gate_proj", "model.layers.55.mlp.experts.141.gate_proj", "model.layers.55.mlp.experts.142.gate_proj", "model.layers.55.mlp.experts.143.gate_proj", "model.layers.55.mlp.experts.144.gate_proj", "model.layers.55.mlp.experts.145.gate_proj", "model.layers.55.mlp.experts.146.gate_proj", "model.layers.55.mlp.experts.147.gate_proj", "model.layers.55.mlp.experts.148.gate_proj", "model.layers.55.mlp.experts.149.gate_proj", "model.layers.55.mlp.experts.150.gate_proj", "model.layers.55.mlp.experts.151.gate_proj", "model.layers.55.mlp.experts.152.gate_proj", "model.layers.55.mlp.experts.153.gate_proj", "model.layers.55.mlp.experts.154.gate_proj", "model.layers.55.mlp.experts.155.gate_proj", "model.layers.55.mlp.experts.156.gate_proj", "model.layers.55.mlp.experts.157.gate_proj", "model.layers.55.mlp.experts.158.gate_proj", "model.layers.55.mlp.experts.159.gate_proj", "model.layers.55.mlp.experts.0.up_proj", "model.layers.55.mlp.experts.1.up_proj", "model.layers.55.mlp.experts.2.up_proj", "model.layers.55.mlp.experts.3.up_proj", "model.layers.55.mlp.experts.4.up_proj", "model.layers.55.mlp.experts.5.up_proj", "model.layers.55.mlp.experts.6.up_proj", "model.layers.55.mlp.experts.7.up_proj", "model.layers.55.mlp.experts.8.up_proj", "model.layers.55.mlp.experts.9.up_proj", "model.layers.55.mlp.experts.10.up_proj", "model.layers.55.mlp.experts.11.up_proj", "model.layers.55.mlp.experts.12.up_proj", "model.layers.55.mlp.experts.13.up_proj", "model.layers.55.mlp.experts.14.up_proj", "model.layers.55.mlp.experts.15.up_proj", "model.layers.55.mlp.experts.16.up_proj", "model.layers.55.mlp.experts.17.up_proj", "model.layers.55.mlp.experts.18.up_proj", "model.layers.55.mlp.experts.19.up_proj", "model.layers.55.mlp.experts.20.up_proj", "model.layers.55.mlp.experts.21.up_proj", "model.layers.55.mlp.experts.22.up_proj", "model.layers.55.mlp.experts.23.up_proj", "model.layers.55.mlp.experts.24.up_proj", "model.layers.55.mlp.experts.25.up_proj", "model.layers.55.mlp.experts.26.up_proj", "model.layers.55.mlp.experts.27.up_proj", "model.layers.55.mlp.experts.28.up_proj", "model.layers.55.mlp.experts.29.up_proj", "model.layers.55.mlp.experts.30.up_proj", "model.layers.55.mlp.experts.31.up_proj", "model.layers.55.mlp.experts.32.up_proj", "model.layers.55.mlp.experts.33.up_proj", "model.layers.55.mlp.experts.34.up_proj", "model.layers.55.mlp.experts.35.up_proj", "model.layers.55.mlp.experts.36.up_proj", "model.layers.55.mlp.experts.37.up_proj", "model.layers.55.mlp.experts.38.up_proj", "model.layers.55.mlp.experts.39.up_proj", "model.layers.55.mlp.experts.40.up_proj", "model.layers.55.mlp.experts.41.up_proj", "model.layers.55.mlp.experts.42.up_proj", "model.layers.55.mlp.experts.43.up_proj", "model.layers.55.mlp.experts.44.up_proj", "model.layers.55.mlp.experts.45.up_proj", "model.layers.55.mlp.experts.46.up_proj", "model.layers.55.mlp.experts.47.up_proj", "model.layers.55.mlp.experts.48.up_proj", "model.layers.55.mlp.experts.49.up_proj", "model.layers.55.mlp.experts.50.up_proj", "model.layers.55.mlp.experts.51.up_proj", "model.layers.55.mlp.experts.52.up_proj", "model.layers.55.mlp.experts.53.up_proj", "model.layers.55.mlp.experts.54.up_proj", "model.layers.55.mlp.experts.55.up_proj", "model.layers.55.mlp.experts.56.up_proj", "model.layers.55.mlp.experts.57.up_proj", "model.layers.55.mlp.experts.58.up_proj", "model.layers.55.mlp.experts.59.up_proj", "model.layers.55.mlp.experts.60.up_proj", "model.layers.55.mlp.experts.61.up_proj", "model.layers.55.mlp.experts.62.up_proj", "model.layers.55.mlp.experts.63.up_proj", "model.layers.55.mlp.experts.64.up_proj", "model.layers.55.mlp.experts.65.up_proj", "model.layers.55.mlp.experts.66.up_proj", "model.layers.55.mlp.experts.67.up_proj", "model.layers.55.mlp.experts.68.up_proj", "model.layers.55.mlp.experts.69.up_proj", "model.layers.55.mlp.experts.70.up_proj", "model.layers.55.mlp.experts.71.up_proj", "model.layers.55.mlp.experts.72.up_proj", "model.layers.55.mlp.experts.73.up_proj", "model.layers.55.mlp.experts.74.up_proj", "model.layers.55.mlp.experts.75.up_proj", "model.layers.55.mlp.experts.76.up_proj", "model.layers.55.mlp.experts.77.up_proj", "model.layers.55.mlp.experts.78.up_proj", "model.layers.55.mlp.experts.79.up_proj", "model.layers.55.mlp.experts.80.up_proj", "model.layers.55.mlp.experts.81.up_proj", "model.layers.55.mlp.experts.82.up_proj", "model.layers.55.mlp.experts.83.up_proj", "model.layers.55.mlp.experts.84.up_proj", "model.layers.55.mlp.experts.85.up_proj", "model.layers.55.mlp.experts.86.up_proj", "model.layers.55.mlp.experts.87.up_proj", "model.layers.55.mlp.experts.88.up_proj", "model.layers.55.mlp.experts.89.up_proj", "model.layers.55.mlp.experts.90.up_proj", "model.layers.55.mlp.experts.91.up_proj", "model.layers.55.mlp.experts.92.up_proj", "model.layers.55.mlp.experts.93.up_proj", "model.layers.55.mlp.experts.94.up_proj", "model.layers.55.mlp.experts.95.up_proj", "model.layers.55.mlp.experts.96.up_proj", "model.layers.55.mlp.experts.97.up_proj", "model.layers.55.mlp.experts.98.up_proj", "model.layers.55.mlp.experts.99.up_proj", "model.layers.55.mlp.experts.100.up_proj", "model.layers.55.mlp.experts.101.up_proj", "model.layers.55.mlp.experts.102.up_proj", "model.layers.55.mlp.experts.103.up_proj", "model.layers.55.mlp.experts.104.up_proj", "model.layers.55.mlp.experts.105.up_proj", "model.layers.55.mlp.experts.106.up_proj", "model.layers.55.mlp.experts.107.up_proj", "model.layers.55.mlp.experts.108.up_proj", "model.layers.55.mlp.experts.109.up_proj", "model.layers.55.mlp.experts.110.up_proj", "model.layers.55.mlp.experts.111.up_proj", "model.layers.55.mlp.experts.112.up_proj", "model.layers.55.mlp.experts.113.up_proj", "model.layers.55.mlp.experts.114.up_proj", "model.layers.55.mlp.experts.115.up_proj", "model.layers.55.mlp.experts.116.up_proj", "model.layers.55.mlp.experts.117.up_proj", "model.layers.55.mlp.experts.118.up_proj", "model.layers.55.mlp.experts.119.up_proj", "model.layers.55.mlp.experts.120.up_proj", "model.layers.55.mlp.experts.121.up_proj", "model.layers.55.mlp.experts.122.up_proj", "model.layers.55.mlp.experts.123.up_proj", "model.layers.55.mlp.experts.124.up_proj", "model.layers.55.mlp.experts.125.up_proj", "model.layers.55.mlp.experts.126.up_proj", "model.layers.55.mlp.experts.127.up_proj", "model.layers.55.mlp.experts.128.up_proj", "model.layers.55.mlp.experts.129.up_proj", "model.layers.55.mlp.experts.130.up_proj", "model.layers.55.mlp.experts.131.up_proj", "model.layers.55.mlp.experts.132.up_proj", "model.layers.55.mlp.experts.133.up_proj", "model.layers.55.mlp.experts.134.up_proj", "model.layers.55.mlp.experts.135.up_proj", "model.layers.55.mlp.experts.136.up_proj", "model.layers.55.mlp.experts.137.up_proj", "model.layers.55.mlp.experts.138.up_proj", "model.layers.55.mlp.experts.139.up_proj", "model.layers.55.mlp.experts.140.up_proj", "model.layers.55.mlp.experts.141.up_proj", "model.layers.55.mlp.experts.142.up_proj", "model.layers.55.mlp.experts.143.up_proj", "model.layers.55.mlp.experts.144.up_proj", "model.layers.55.mlp.experts.145.up_proj", "model.layers.55.mlp.experts.146.up_proj", "model.layers.55.mlp.experts.147.up_proj", "model.layers.55.mlp.experts.148.up_proj", "model.layers.55.mlp.experts.149.up_proj", "model.layers.55.mlp.experts.150.up_proj", "model.layers.55.mlp.experts.151.up_proj", "model.layers.55.mlp.experts.152.up_proj", "model.layers.55.mlp.experts.153.up_proj", "model.layers.55.mlp.experts.154.up_proj", "model.layers.55.mlp.experts.155.up_proj", "model.layers.55.mlp.experts.156.up_proj", "model.layers.55.mlp.experts.157.up_proj", "model.layers.55.mlp.experts.158.up_proj", "model.layers.55.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0004497356712818146, "dbits": 2516582400 } ] }, { "idx": 332, "layers": [ "model.layers.55.mlp.experts.0.down_proj", "model.layers.55.mlp.experts.1.down_proj", "model.layers.55.mlp.experts.2.down_proj", "model.layers.55.mlp.experts.3.down_proj", "model.layers.55.mlp.experts.4.down_proj", "model.layers.55.mlp.experts.5.down_proj", "model.layers.55.mlp.experts.6.down_proj", "model.layers.55.mlp.experts.7.down_proj", "model.layers.55.mlp.experts.8.down_proj", "model.layers.55.mlp.experts.9.down_proj", "model.layers.55.mlp.experts.10.down_proj", "model.layers.55.mlp.experts.11.down_proj", "model.layers.55.mlp.experts.12.down_proj", "model.layers.55.mlp.experts.13.down_proj", "model.layers.55.mlp.experts.14.down_proj", "model.layers.55.mlp.experts.15.down_proj", "model.layers.55.mlp.experts.16.down_proj", "model.layers.55.mlp.experts.17.down_proj", "model.layers.55.mlp.experts.18.down_proj", "model.layers.55.mlp.experts.19.down_proj", "model.layers.55.mlp.experts.20.down_proj", "model.layers.55.mlp.experts.21.down_proj", "model.layers.55.mlp.experts.22.down_proj", "model.layers.55.mlp.experts.23.down_proj", "model.layers.55.mlp.experts.24.down_proj", "model.layers.55.mlp.experts.25.down_proj", "model.layers.55.mlp.experts.26.down_proj", "model.layers.55.mlp.experts.27.down_proj", "model.layers.55.mlp.experts.28.down_proj", "model.layers.55.mlp.experts.29.down_proj", "model.layers.55.mlp.experts.30.down_proj", "model.layers.55.mlp.experts.31.down_proj", "model.layers.55.mlp.experts.32.down_proj", "model.layers.55.mlp.experts.33.down_proj", "model.layers.55.mlp.experts.34.down_proj", "model.layers.55.mlp.experts.35.down_proj", "model.layers.55.mlp.experts.36.down_proj", "model.layers.55.mlp.experts.37.down_proj", "model.layers.55.mlp.experts.38.down_proj", "model.layers.55.mlp.experts.39.down_proj", "model.layers.55.mlp.experts.40.down_proj", "model.layers.55.mlp.experts.41.down_proj", "model.layers.55.mlp.experts.42.down_proj", "model.layers.55.mlp.experts.43.down_proj", "model.layers.55.mlp.experts.44.down_proj", "model.layers.55.mlp.experts.45.down_proj", "model.layers.55.mlp.experts.46.down_proj", "model.layers.55.mlp.experts.47.down_proj", "model.layers.55.mlp.experts.48.down_proj", "model.layers.55.mlp.experts.49.down_proj", "model.layers.55.mlp.experts.50.down_proj", "model.layers.55.mlp.experts.51.down_proj", "model.layers.55.mlp.experts.52.down_proj", "model.layers.55.mlp.experts.53.down_proj", "model.layers.55.mlp.experts.54.down_proj", "model.layers.55.mlp.experts.55.down_proj", "model.layers.55.mlp.experts.56.down_proj", "model.layers.55.mlp.experts.57.down_proj", "model.layers.55.mlp.experts.58.down_proj", "model.layers.55.mlp.experts.59.down_proj", "model.layers.55.mlp.experts.60.down_proj", "model.layers.55.mlp.experts.61.down_proj", "model.layers.55.mlp.experts.62.down_proj", "model.layers.55.mlp.experts.63.down_proj", "model.layers.55.mlp.experts.64.down_proj", "model.layers.55.mlp.experts.65.down_proj", "model.layers.55.mlp.experts.66.down_proj", "model.layers.55.mlp.experts.67.down_proj", "model.layers.55.mlp.experts.68.down_proj", "model.layers.55.mlp.experts.69.down_proj", "model.layers.55.mlp.experts.70.down_proj", "model.layers.55.mlp.experts.71.down_proj", "model.layers.55.mlp.experts.72.down_proj", "model.layers.55.mlp.experts.73.down_proj", "model.layers.55.mlp.experts.74.down_proj", "model.layers.55.mlp.experts.75.down_proj", "model.layers.55.mlp.experts.76.down_proj", "model.layers.55.mlp.experts.77.down_proj", "model.layers.55.mlp.experts.78.down_proj", "model.layers.55.mlp.experts.79.down_proj", "model.layers.55.mlp.experts.80.down_proj", "model.layers.55.mlp.experts.81.down_proj", "model.layers.55.mlp.experts.82.down_proj", "model.layers.55.mlp.experts.83.down_proj", "model.layers.55.mlp.experts.84.down_proj", "model.layers.55.mlp.experts.85.down_proj", "model.layers.55.mlp.experts.86.down_proj", "model.layers.55.mlp.experts.87.down_proj", "model.layers.55.mlp.experts.88.down_proj", "model.layers.55.mlp.experts.89.down_proj", "model.layers.55.mlp.experts.90.down_proj", "model.layers.55.mlp.experts.91.down_proj", "model.layers.55.mlp.experts.92.down_proj", "model.layers.55.mlp.experts.93.down_proj", "model.layers.55.mlp.experts.94.down_proj", "model.layers.55.mlp.experts.95.down_proj", "model.layers.55.mlp.experts.96.down_proj", "model.layers.55.mlp.experts.97.down_proj", "model.layers.55.mlp.experts.98.down_proj", "model.layers.55.mlp.experts.99.down_proj", "model.layers.55.mlp.experts.100.down_proj", "model.layers.55.mlp.experts.101.down_proj", "model.layers.55.mlp.experts.102.down_proj", "model.layers.55.mlp.experts.103.down_proj", "model.layers.55.mlp.experts.104.down_proj", "model.layers.55.mlp.experts.105.down_proj", "model.layers.55.mlp.experts.106.down_proj", "model.layers.55.mlp.experts.107.down_proj", "model.layers.55.mlp.experts.108.down_proj", "model.layers.55.mlp.experts.109.down_proj", "model.layers.55.mlp.experts.110.down_proj", "model.layers.55.mlp.experts.111.down_proj", "model.layers.55.mlp.experts.112.down_proj", "model.layers.55.mlp.experts.113.down_proj", "model.layers.55.mlp.experts.114.down_proj", "model.layers.55.mlp.experts.115.down_proj", "model.layers.55.mlp.experts.116.down_proj", "model.layers.55.mlp.experts.117.down_proj", "model.layers.55.mlp.experts.118.down_proj", "model.layers.55.mlp.experts.119.down_proj", "model.layers.55.mlp.experts.120.down_proj", "model.layers.55.mlp.experts.121.down_proj", "model.layers.55.mlp.experts.122.down_proj", "model.layers.55.mlp.experts.123.down_proj", "model.layers.55.mlp.experts.124.down_proj", "model.layers.55.mlp.experts.125.down_proj", "model.layers.55.mlp.experts.126.down_proj", "model.layers.55.mlp.experts.127.down_proj", "model.layers.55.mlp.experts.128.down_proj", "model.layers.55.mlp.experts.129.down_proj", "model.layers.55.mlp.experts.130.down_proj", "model.layers.55.mlp.experts.131.down_proj", "model.layers.55.mlp.experts.132.down_proj", "model.layers.55.mlp.experts.133.down_proj", "model.layers.55.mlp.experts.134.down_proj", "model.layers.55.mlp.experts.135.down_proj", "model.layers.55.mlp.experts.136.down_proj", "model.layers.55.mlp.experts.137.down_proj", "model.layers.55.mlp.experts.138.down_proj", "model.layers.55.mlp.experts.139.down_proj", "model.layers.55.mlp.experts.140.down_proj", "model.layers.55.mlp.experts.141.down_proj", "model.layers.55.mlp.experts.142.down_proj", "model.layers.55.mlp.experts.143.down_proj", "model.layers.55.mlp.experts.144.down_proj", "model.layers.55.mlp.experts.145.down_proj", "model.layers.55.mlp.experts.146.down_proj", "model.layers.55.mlp.experts.147.down_proj", "model.layers.55.mlp.experts.148.down_proj", "model.layers.55.mlp.experts.149.down_proj", "model.layers.55.mlp.experts.150.down_proj", "model.layers.55.mlp.experts.151.down_proj", "model.layers.55.mlp.experts.152.down_proj", "model.layers.55.mlp.experts.153.down_proj", "model.layers.55.mlp.experts.154.down_proj", "model.layers.55.mlp.experts.155.down_proj", "model.layers.55.mlp.experts.156.down_proj", "model.layers.55.mlp.experts.157.down_proj", "model.layers.55.mlp.experts.158.down_proj", "model.layers.55.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.000547044724226009, "dbits": 1258291200 } ] }, { "idx": 333, "layers": [ "model.layers.56.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0003718368709087483, "dbits": 62914560 } ] }, { "idx": 334, "layers": [ "model.layers.56.self_attn.k_proj", "model.layers.56.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00019620656967161976, "dbits": 10485760 } ] }, { "idx": 335, "layers": [ "model.layers.56.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0004824429750442394, "dbits": 62914560 } ] }, { "idx": 336, "layers": [ "model.layers.56.mlp.shared_experts.gate_proj", "model.layers.56.mlp.shared_experts.up_proj", "model.layers.56.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0012966319918632507, "dbits": 23592960 } ] }, { "idx": 337, "layers": [ "model.layers.56.mlp.experts.0.gate_proj", "model.layers.56.mlp.experts.1.gate_proj", "model.layers.56.mlp.experts.2.gate_proj", "model.layers.56.mlp.experts.3.gate_proj", "model.layers.56.mlp.experts.4.gate_proj", "model.layers.56.mlp.experts.5.gate_proj", "model.layers.56.mlp.experts.6.gate_proj", "model.layers.56.mlp.experts.7.gate_proj", "model.layers.56.mlp.experts.8.gate_proj", "model.layers.56.mlp.experts.9.gate_proj", "model.layers.56.mlp.experts.10.gate_proj", "model.layers.56.mlp.experts.11.gate_proj", "model.layers.56.mlp.experts.12.gate_proj", "model.layers.56.mlp.experts.13.gate_proj", "model.layers.56.mlp.experts.14.gate_proj", "model.layers.56.mlp.experts.15.gate_proj", "model.layers.56.mlp.experts.16.gate_proj", "model.layers.56.mlp.experts.17.gate_proj", "model.layers.56.mlp.experts.18.gate_proj", "model.layers.56.mlp.experts.19.gate_proj", "model.layers.56.mlp.experts.20.gate_proj", "model.layers.56.mlp.experts.21.gate_proj", "model.layers.56.mlp.experts.22.gate_proj", "model.layers.56.mlp.experts.23.gate_proj", "model.layers.56.mlp.experts.24.gate_proj", "model.layers.56.mlp.experts.25.gate_proj", "model.layers.56.mlp.experts.26.gate_proj", "model.layers.56.mlp.experts.27.gate_proj", "model.layers.56.mlp.experts.28.gate_proj", "model.layers.56.mlp.experts.29.gate_proj", "model.layers.56.mlp.experts.30.gate_proj", "model.layers.56.mlp.experts.31.gate_proj", "model.layers.56.mlp.experts.32.gate_proj", "model.layers.56.mlp.experts.33.gate_proj", "model.layers.56.mlp.experts.34.gate_proj", "model.layers.56.mlp.experts.35.gate_proj", "model.layers.56.mlp.experts.36.gate_proj", "model.layers.56.mlp.experts.37.gate_proj", "model.layers.56.mlp.experts.38.gate_proj", "model.layers.56.mlp.experts.39.gate_proj", "model.layers.56.mlp.experts.40.gate_proj", "model.layers.56.mlp.experts.41.gate_proj", "model.layers.56.mlp.experts.42.gate_proj", "model.layers.56.mlp.experts.43.gate_proj", "model.layers.56.mlp.experts.44.gate_proj", "model.layers.56.mlp.experts.45.gate_proj", "model.layers.56.mlp.experts.46.gate_proj", "model.layers.56.mlp.experts.47.gate_proj", "model.layers.56.mlp.experts.48.gate_proj", "model.layers.56.mlp.experts.49.gate_proj", "model.layers.56.mlp.experts.50.gate_proj", "model.layers.56.mlp.experts.51.gate_proj", "model.layers.56.mlp.experts.52.gate_proj", "model.layers.56.mlp.experts.53.gate_proj", "model.layers.56.mlp.experts.54.gate_proj", "model.layers.56.mlp.experts.55.gate_proj", "model.layers.56.mlp.experts.56.gate_proj", "model.layers.56.mlp.experts.57.gate_proj", "model.layers.56.mlp.experts.58.gate_proj", "model.layers.56.mlp.experts.59.gate_proj", "model.layers.56.mlp.experts.60.gate_proj", "model.layers.56.mlp.experts.61.gate_proj", "model.layers.56.mlp.experts.62.gate_proj", "model.layers.56.mlp.experts.63.gate_proj", "model.layers.56.mlp.experts.64.gate_proj", "model.layers.56.mlp.experts.65.gate_proj", "model.layers.56.mlp.experts.66.gate_proj", "model.layers.56.mlp.experts.67.gate_proj", "model.layers.56.mlp.experts.68.gate_proj", "model.layers.56.mlp.experts.69.gate_proj", "model.layers.56.mlp.experts.70.gate_proj", "model.layers.56.mlp.experts.71.gate_proj", "model.layers.56.mlp.experts.72.gate_proj", "model.layers.56.mlp.experts.73.gate_proj", "model.layers.56.mlp.experts.74.gate_proj", "model.layers.56.mlp.experts.75.gate_proj", "model.layers.56.mlp.experts.76.gate_proj", "model.layers.56.mlp.experts.77.gate_proj", "model.layers.56.mlp.experts.78.gate_proj", "model.layers.56.mlp.experts.79.gate_proj", "model.layers.56.mlp.experts.80.gate_proj", "model.layers.56.mlp.experts.81.gate_proj", "model.layers.56.mlp.experts.82.gate_proj", "model.layers.56.mlp.experts.83.gate_proj", "model.layers.56.mlp.experts.84.gate_proj", "model.layers.56.mlp.experts.85.gate_proj", "model.layers.56.mlp.experts.86.gate_proj", "model.layers.56.mlp.experts.87.gate_proj", "model.layers.56.mlp.experts.88.gate_proj", "model.layers.56.mlp.experts.89.gate_proj", "model.layers.56.mlp.experts.90.gate_proj", "model.layers.56.mlp.experts.91.gate_proj", "model.layers.56.mlp.experts.92.gate_proj", "model.layers.56.mlp.experts.93.gate_proj", "model.layers.56.mlp.experts.94.gate_proj", "model.layers.56.mlp.experts.95.gate_proj", "model.layers.56.mlp.experts.96.gate_proj", "model.layers.56.mlp.experts.97.gate_proj", "model.layers.56.mlp.experts.98.gate_proj", "model.layers.56.mlp.experts.99.gate_proj", "model.layers.56.mlp.experts.100.gate_proj", "model.layers.56.mlp.experts.101.gate_proj", "model.layers.56.mlp.experts.102.gate_proj", "model.layers.56.mlp.experts.103.gate_proj", "model.layers.56.mlp.experts.104.gate_proj", "model.layers.56.mlp.experts.105.gate_proj", "model.layers.56.mlp.experts.106.gate_proj", "model.layers.56.mlp.experts.107.gate_proj", "model.layers.56.mlp.experts.108.gate_proj", "model.layers.56.mlp.experts.109.gate_proj", "model.layers.56.mlp.experts.110.gate_proj", "model.layers.56.mlp.experts.111.gate_proj", "model.layers.56.mlp.experts.112.gate_proj", "model.layers.56.mlp.experts.113.gate_proj", "model.layers.56.mlp.experts.114.gate_proj", "model.layers.56.mlp.experts.115.gate_proj", "model.layers.56.mlp.experts.116.gate_proj", "model.layers.56.mlp.experts.117.gate_proj", "model.layers.56.mlp.experts.118.gate_proj", "model.layers.56.mlp.experts.119.gate_proj", "model.layers.56.mlp.experts.120.gate_proj", "model.layers.56.mlp.experts.121.gate_proj", "model.layers.56.mlp.experts.122.gate_proj", "model.layers.56.mlp.experts.123.gate_proj", "model.layers.56.mlp.experts.124.gate_proj", "model.layers.56.mlp.experts.125.gate_proj", "model.layers.56.mlp.experts.126.gate_proj", "model.layers.56.mlp.experts.127.gate_proj", "model.layers.56.mlp.experts.128.gate_proj", "model.layers.56.mlp.experts.129.gate_proj", "model.layers.56.mlp.experts.130.gate_proj", "model.layers.56.mlp.experts.131.gate_proj", "model.layers.56.mlp.experts.132.gate_proj", "model.layers.56.mlp.experts.133.gate_proj", "model.layers.56.mlp.experts.134.gate_proj", "model.layers.56.mlp.experts.135.gate_proj", "model.layers.56.mlp.experts.136.gate_proj", "model.layers.56.mlp.experts.137.gate_proj", "model.layers.56.mlp.experts.138.gate_proj", "model.layers.56.mlp.experts.139.gate_proj", "model.layers.56.mlp.experts.140.gate_proj", "model.layers.56.mlp.experts.141.gate_proj", "model.layers.56.mlp.experts.142.gate_proj", "model.layers.56.mlp.experts.143.gate_proj", "model.layers.56.mlp.experts.144.gate_proj", "model.layers.56.mlp.experts.145.gate_proj", "model.layers.56.mlp.experts.146.gate_proj", "model.layers.56.mlp.experts.147.gate_proj", "model.layers.56.mlp.experts.148.gate_proj", "model.layers.56.mlp.experts.149.gate_proj", "model.layers.56.mlp.experts.150.gate_proj", "model.layers.56.mlp.experts.151.gate_proj", "model.layers.56.mlp.experts.152.gate_proj", "model.layers.56.mlp.experts.153.gate_proj", "model.layers.56.mlp.experts.154.gate_proj", "model.layers.56.mlp.experts.155.gate_proj", "model.layers.56.mlp.experts.156.gate_proj", "model.layers.56.mlp.experts.157.gate_proj", "model.layers.56.mlp.experts.158.gate_proj", "model.layers.56.mlp.experts.159.gate_proj", "model.layers.56.mlp.experts.0.up_proj", "model.layers.56.mlp.experts.1.up_proj", "model.layers.56.mlp.experts.2.up_proj", "model.layers.56.mlp.experts.3.up_proj", "model.layers.56.mlp.experts.4.up_proj", "model.layers.56.mlp.experts.5.up_proj", "model.layers.56.mlp.experts.6.up_proj", "model.layers.56.mlp.experts.7.up_proj", "model.layers.56.mlp.experts.8.up_proj", "model.layers.56.mlp.experts.9.up_proj", "model.layers.56.mlp.experts.10.up_proj", "model.layers.56.mlp.experts.11.up_proj", "model.layers.56.mlp.experts.12.up_proj", "model.layers.56.mlp.experts.13.up_proj", "model.layers.56.mlp.experts.14.up_proj", "model.layers.56.mlp.experts.15.up_proj", "model.layers.56.mlp.experts.16.up_proj", "model.layers.56.mlp.experts.17.up_proj", "model.layers.56.mlp.experts.18.up_proj", "model.layers.56.mlp.experts.19.up_proj", "model.layers.56.mlp.experts.20.up_proj", "model.layers.56.mlp.experts.21.up_proj", "model.layers.56.mlp.experts.22.up_proj", "model.layers.56.mlp.experts.23.up_proj", "model.layers.56.mlp.experts.24.up_proj", "model.layers.56.mlp.experts.25.up_proj", "model.layers.56.mlp.experts.26.up_proj", "model.layers.56.mlp.experts.27.up_proj", "model.layers.56.mlp.experts.28.up_proj", "model.layers.56.mlp.experts.29.up_proj", "model.layers.56.mlp.experts.30.up_proj", "model.layers.56.mlp.experts.31.up_proj", "model.layers.56.mlp.experts.32.up_proj", "model.layers.56.mlp.experts.33.up_proj", "model.layers.56.mlp.experts.34.up_proj", "model.layers.56.mlp.experts.35.up_proj", "model.layers.56.mlp.experts.36.up_proj", "model.layers.56.mlp.experts.37.up_proj", "model.layers.56.mlp.experts.38.up_proj", "model.layers.56.mlp.experts.39.up_proj", "model.layers.56.mlp.experts.40.up_proj", "model.layers.56.mlp.experts.41.up_proj", "model.layers.56.mlp.experts.42.up_proj", "model.layers.56.mlp.experts.43.up_proj", "model.layers.56.mlp.experts.44.up_proj", "model.layers.56.mlp.experts.45.up_proj", "model.layers.56.mlp.experts.46.up_proj", "model.layers.56.mlp.experts.47.up_proj", "model.layers.56.mlp.experts.48.up_proj", "model.layers.56.mlp.experts.49.up_proj", "model.layers.56.mlp.experts.50.up_proj", "model.layers.56.mlp.experts.51.up_proj", "model.layers.56.mlp.experts.52.up_proj", "model.layers.56.mlp.experts.53.up_proj", "model.layers.56.mlp.experts.54.up_proj", "model.layers.56.mlp.experts.55.up_proj", "model.layers.56.mlp.experts.56.up_proj", "model.layers.56.mlp.experts.57.up_proj", "model.layers.56.mlp.experts.58.up_proj", "model.layers.56.mlp.experts.59.up_proj", "model.layers.56.mlp.experts.60.up_proj", "model.layers.56.mlp.experts.61.up_proj", "model.layers.56.mlp.experts.62.up_proj", "model.layers.56.mlp.experts.63.up_proj", "model.layers.56.mlp.experts.64.up_proj", "model.layers.56.mlp.experts.65.up_proj", "model.layers.56.mlp.experts.66.up_proj", "model.layers.56.mlp.experts.67.up_proj", "model.layers.56.mlp.experts.68.up_proj", "model.layers.56.mlp.experts.69.up_proj", "model.layers.56.mlp.experts.70.up_proj", "model.layers.56.mlp.experts.71.up_proj", "model.layers.56.mlp.experts.72.up_proj", "model.layers.56.mlp.experts.73.up_proj", "model.layers.56.mlp.experts.74.up_proj", "model.layers.56.mlp.experts.75.up_proj", "model.layers.56.mlp.experts.76.up_proj", "model.layers.56.mlp.experts.77.up_proj", "model.layers.56.mlp.experts.78.up_proj", "model.layers.56.mlp.experts.79.up_proj", "model.layers.56.mlp.experts.80.up_proj", "model.layers.56.mlp.experts.81.up_proj", "model.layers.56.mlp.experts.82.up_proj", "model.layers.56.mlp.experts.83.up_proj", "model.layers.56.mlp.experts.84.up_proj", "model.layers.56.mlp.experts.85.up_proj", "model.layers.56.mlp.experts.86.up_proj", "model.layers.56.mlp.experts.87.up_proj", "model.layers.56.mlp.experts.88.up_proj", "model.layers.56.mlp.experts.89.up_proj", "model.layers.56.mlp.experts.90.up_proj", "model.layers.56.mlp.experts.91.up_proj", "model.layers.56.mlp.experts.92.up_proj", "model.layers.56.mlp.experts.93.up_proj", "model.layers.56.mlp.experts.94.up_proj", "model.layers.56.mlp.experts.95.up_proj", "model.layers.56.mlp.experts.96.up_proj", "model.layers.56.mlp.experts.97.up_proj", "model.layers.56.mlp.experts.98.up_proj", "model.layers.56.mlp.experts.99.up_proj", "model.layers.56.mlp.experts.100.up_proj", "model.layers.56.mlp.experts.101.up_proj", "model.layers.56.mlp.experts.102.up_proj", "model.layers.56.mlp.experts.103.up_proj", "model.layers.56.mlp.experts.104.up_proj", "model.layers.56.mlp.experts.105.up_proj", "model.layers.56.mlp.experts.106.up_proj", "model.layers.56.mlp.experts.107.up_proj", "model.layers.56.mlp.experts.108.up_proj", "model.layers.56.mlp.experts.109.up_proj", "model.layers.56.mlp.experts.110.up_proj", "model.layers.56.mlp.experts.111.up_proj", "model.layers.56.mlp.experts.112.up_proj", "model.layers.56.mlp.experts.113.up_proj", "model.layers.56.mlp.experts.114.up_proj", "model.layers.56.mlp.experts.115.up_proj", "model.layers.56.mlp.experts.116.up_proj", "model.layers.56.mlp.experts.117.up_proj", "model.layers.56.mlp.experts.118.up_proj", "model.layers.56.mlp.experts.119.up_proj", "model.layers.56.mlp.experts.120.up_proj", "model.layers.56.mlp.experts.121.up_proj", "model.layers.56.mlp.experts.122.up_proj", "model.layers.56.mlp.experts.123.up_proj", "model.layers.56.mlp.experts.124.up_proj", "model.layers.56.mlp.experts.125.up_proj", "model.layers.56.mlp.experts.126.up_proj", "model.layers.56.mlp.experts.127.up_proj", "model.layers.56.mlp.experts.128.up_proj", "model.layers.56.mlp.experts.129.up_proj", "model.layers.56.mlp.experts.130.up_proj", "model.layers.56.mlp.experts.131.up_proj", "model.layers.56.mlp.experts.132.up_proj", "model.layers.56.mlp.experts.133.up_proj", "model.layers.56.mlp.experts.134.up_proj", "model.layers.56.mlp.experts.135.up_proj", "model.layers.56.mlp.experts.136.up_proj", "model.layers.56.mlp.experts.137.up_proj", "model.layers.56.mlp.experts.138.up_proj", "model.layers.56.mlp.experts.139.up_proj", "model.layers.56.mlp.experts.140.up_proj", "model.layers.56.mlp.experts.141.up_proj", "model.layers.56.mlp.experts.142.up_proj", "model.layers.56.mlp.experts.143.up_proj", "model.layers.56.mlp.experts.144.up_proj", "model.layers.56.mlp.experts.145.up_proj", "model.layers.56.mlp.experts.146.up_proj", "model.layers.56.mlp.experts.147.up_proj", "model.layers.56.mlp.experts.148.up_proj", "model.layers.56.mlp.experts.149.up_proj", "model.layers.56.mlp.experts.150.up_proj", "model.layers.56.mlp.experts.151.up_proj", "model.layers.56.mlp.experts.152.up_proj", "model.layers.56.mlp.experts.153.up_proj", "model.layers.56.mlp.experts.154.up_proj", "model.layers.56.mlp.experts.155.up_proj", "model.layers.56.mlp.experts.156.up_proj", "model.layers.56.mlp.experts.157.up_proj", "model.layers.56.mlp.experts.158.up_proj", "model.layers.56.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0007504388689995034, "dbits": 2516582400 } ] }, { "idx": 338, "layers": [ "model.layers.56.mlp.experts.0.down_proj", "model.layers.56.mlp.experts.1.down_proj", "model.layers.56.mlp.experts.2.down_proj", "model.layers.56.mlp.experts.3.down_proj", "model.layers.56.mlp.experts.4.down_proj", "model.layers.56.mlp.experts.5.down_proj", "model.layers.56.mlp.experts.6.down_proj", "model.layers.56.mlp.experts.7.down_proj", "model.layers.56.mlp.experts.8.down_proj", "model.layers.56.mlp.experts.9.down_proj", "model.layers.56.mlp.experts.10.down_proj", "model.layers.56.mlp.experts.11.down_proj", "model.layers.56.mlp.experts.12.down_proj", "model.layers.56.mlp.experts.13.down_proj", "model.layers.56.mlp.experts.14.down_proj", "model.layers.56.mlp.experts.15.down_proj", "model.layers.56.mlp.experts.16.down_proj", "model.layers.56.mlp.experts.17.down_proj", "model.layers.56.mlp.experts.18.down_proj", "model.layers.56.mlp.experts.19.down_proj", "model.layers.56.mlp.experts.20.down_proj", "model.layers.56.mlp.experts.21.down_proj", "model.layers.56.mlp.experts.22.down_proj", "model.layers.56.mlp.experts.23.down_proj", "model.layers.56.mlp.experts.24.down_proj", "model.layers.56.mlp.experts.25.down_proj", "model.layers.56.mlp.experts.26.down_proj", "model.layers.56.mlp.experts.27.down_proj", "model.layers.56.mlp.experts.28.down_proj", "model.layers.56.mlp.experts.29.down_proj", "model.layers.56.mlp.experts.30.down_proj", "model.layers.56.mlp.experts.31.down_proj", "model.layers.56.mlp.experts.32.down_proj", "model.layers.56.mlp.experts.33.down_proj", "model.layers.56.mlp.experts.34.down_proj", "model.layers.56.mlp.experts.35.down_proj", "model.layers.56.mlp.experts.36.down_proj", "model.layers.56.mlp.experts.37.down_proj", "model.layers.56.mlp.experts.38.down_proj", "model.layers.56.mlp.experts.39.down_proj", "model.layers.56.mlp.experts.40.down_proj", "model.layers.56.mlp.experts.41.down_proj", "model.layers.56.mlp.experts.42.down_proj", "model.layers.56.mlp.experts.43.down_proj", "model.layers.56.mlp.experts.44.down_proj", "model.layers.56.mlp.experts.45.down_proj", "model.layers.56.mlp.experts.46.down_proj", "model.layers.56.mlp.experts.47.down_proj", "model.layers.56.mlp.experts.48.down_proj", "model.layers.56.mlp.experts.49.down_proj", "model.layers.56.mlp.experts.50.down_proj", "model.layers.56.mlp.experts.51.down_proj", "model.layers.56.mlp.experts.52.down_proj", "model.layers.56.mlp.experts.53.down_proj", "model.layers.56.mlp.experts.54.down_proj", "model.layers.56.mlp.experts.55.down_proj", "model.layers.56.mlp.experts.56.down_proj", "model.layers.56.mlp.experts.57.down_proj", "model.layers.56.mlp.experts.58.down_proj", "model.layers.56.mlp.experts.59.down_proj", "model.layers.56.mlp.experts.60.down_proj", "model.layers.56.mlp.experts.61.down_proj", "model.layers.56.mlp.experts.62.down_proj", "model.layers.56.mlp.experts.63.down_proj", "model.layers.56.mlp.experts.64.down_proj", "model.layers.56.mlp.experts.65.down_proj", "model.layers.56.mlp.experts.66.down_proj", "model.layers.56.mlp.experts.67.down_proj", "model.layers.56.mlp.experts.68.down_proj", "model.layers.56.mlp.experts.69.down_proj", "model.layers.56.mlp.experts.70.down_proj", "model.layers.56.mlp.experts.71.down_proj", "model.layers.56.mlp.experts.72.down_proj", "model.layers.56.mlp.experts.73.down_proj", "model.layers.56.mlp.experts.74.down_proj", "model.layers.56.mlp.experts.75.down_proj", "model.layers.56.mlp.experts.76.down_proj", "model.layers.56.mlp.experts.77.down_proj", "model.layers.56.mlp.experts.78.down_proj", "model.layers.56.mlp.experts.79.down_proj", "model.layers.56.mlp.experts.80.down_proj", "model.layers.56.mlp.experts.81.down_proj", "model.layers.56.mlp.experts.82.down_proj", "model.layers.56.mlp.experts.83.down_proj", "model.layers.56.mlp.experts.84.down_proj", "model.layers.56.mlp.experts.85.down_proj", "model.layers.56.mlp.experts.86.down_proj", "model.layers.56.mlp.experts.87.down_proj", "model.layers.56.mlp.experts.88.down_proj", "model.layers.56.mlp.experts.89.down_proj", "model.layers.56.mlp.experts.90.down_proj", "model.layers.56.mlp.experts.91.down_proj", "model.layers.56.mlp.experts.92.down_proj", "model.layers.56.mlp.experts.93.down_proj", "model.layers.56.mlp.experts.94.down_proj", "model.layers.56.mlp.experts.95.down_proj", "model.layers.56.mlp.experts.96.down_proj", "model.layers.56.mlp.experts.97.down_proj", "model.layers.56.mlp.experts.98.down_proj", "model.layers.56.mlp.experts.99.down_proj", "model.layers.56.mlp.experts.100.down_proj", "model.layers.56.mlp.experts.101.down_proj", "model.layers.56.mlp.experts.102.down_proj", "model.layers.56.mlp.experts.103.down_proj", "model.layers.56.mlp.experts.104.down_proj", "model.layers.56.mlp.experts.105.down_proj", "model.layers.56.mlp.experts.106.down_proj", "model.layers.56.mlp.experts.107.down_proj", "model.layers.56.mlp.experts.108.down_proj", "model.layers.56.mlp.experts.109.down_proj", "model.layers.56.mlp.experts.110.down_proj", "model.layers.56.mlp.experts.111.down_proj", "model.layers.56.mlp.experts.112.down_proj", "model.layers.56.mlp.experts.113.down_proj", "model.layers.56.mlp.experts.114.down_proj", "model.layers.56.mlp.experts.115.down_proj", "model.layers.56.mlp.experts.116.down_proj", "model.layers.56.mlp.experts.117.down_proj", "model.layers.56.mlp.experts.118.down_proj", "model.layers.56.mlp.experts.119.down_proj", "model.layers.56.mlp.experts.120.down_proj", "model.layers.56.mlp.experts.121.down_proj", "model.layers.56.mlp.experts.122.down_proj", "model.layers.56.mlp.experts.123.down_proj", "model.layers.56.mlp.experts.124.down_proj", "model.layers.56.mlp.experts.125.down_proj", "model.layers.56.mlp.experts.126.down_proj", "model.layers.56.mlp.experts.127.down_proj", "model.layers.56.mlp.experts.128.down_proj", "model.layers.56.mlp.experts.129.down_proj", "model.layers.56.mlp.experts.130.down_proj", "model.layers.56.mlp.experts.131.down_proj", "model.layers.56.mlp.experts.132.down_proj", "model.layers.56.mlp.experts.133.down_proj", "model.layers.56.mlp.experts.134.down_proj", "model.layers.56.mlp.experts.135.down_proj", "model.layers.56.mlp.experts.136.down_proj", "model.layers.56.mlp.experts.137.down_proj", "model.layers.56.mlp.experts.138.down_proj", "model.layers.56.mlp.experts.139.down_proj", "model.layers.56.mlp.experts.140.down_proj", "model.layers.56.mlp.experts.141.down_proj", "model.layers.56.mlp.experts.142.down_proj", "model.layers.56.mlp.experts.143.down_proj", "model.layers.56.mlp.experts.144.down_proj", "model.layers.56.mlp.experts.145.down_proj", "model.layers.56.mlp.experts.146.down_proj", "model.layers.56.mlp.experts.147.down_proj", "model.layers.56.mlp.experts.148.down_proj", "model.layers.56.mlp.experts.149.down_proj", "model.layers.56.mlp.experts.150.down_proj", "model.layers.56.mlp.experts.151.down_proj", "model.layers.56.mlp.experts.152.down_proj", "model.layers.56.mlp.experts.153.down_proj", "model.layers.56.mlp.experts.154.down_proj", "model.layers.56.mlp.experts.155.down_proj", "model.layers.56.mlp.experts.156.down_proj", "model.layers.56.mlp.experts.157.down_proj", "model.layers.56.mlp.experts.158.down_proj", "model.layers.56.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0005595721304416656, "dbits": 1258291200 } ] }, { "idx": 339, "layers": [ "model.layers.57.self_attn.q_proj" ], "candidates": [ { "dkld": -0.000660136342048645, "dbits": 62914560 } ] }, { "idx": 340, "layers": [ "model.layers.57.self_attn.k_proj", "model.layers.57.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0005690254271030204, "dbits": 10485760 } ] }, { "idx": 341, "layers": [ "model.layers.57.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00041518211364743873, "dbits": 62914560 } ] }, { "idx": 342, "layers": [ "model.layers.57.mlp.shared_experts.gate_proj", "model.layers.57.mlp.shared_experts.up_proj", "model.layers.57.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0002032846212386974, "dbits": 23592960 } ] }, { "idx": 343, "layers": [ "model.layers.57.mlp.experts.0.gate_proj", "model.layers.57.mlp.experts.1.gate_proj", "model.layers.57.mlp.experts.2.gate_proj", "model.layers.57.mlp.experts.3.gate_proj", "model.layers.57.mlp.experts.4.gate_proj", "model.layers.57.mlp.experts.5.gate_proj", "model.layers.57.mlp.experts.6.gate_proj", "model.layers.57.mlp.experts.7.gate_proj", "model.layers.57.mlp.experts.8.gate_proj", "model.layers.57.mlp.experts.9.gate_proj", "model.layers.57.mlp.experts.10.gate_proj", "model.layers.57.mlp.experts.11.gate_proj", "model.layers.57.mlp.experts.12.gate_proj", "model.layers.57.mlp.experts.13.gate_proj", "model.layers.57.mlp.experts.14.gate_proj", "model.layers.57.mlp.experts.15.gate_proj", "model.layers.57.mlp.experts.16.gate_proj", "model.layers.57.mlp.experts.17.gate_proj", "model.layers.57.mlp.experts.18.gate_proj", "model.layers.57.mlp.experts.19.gate_proj", "model.layers.57.mlp.experts.20.gate_proj", "model.layers.57.mlp.experts.21.gate_proj", "model.layers.57.mlp.experts.22.gate_proj", "model.layers.57.mlp.experts.23.gate_proj", "model.layers.57.mlp.experts.24.gate_proj", "model.layers.57.mlp.experts.25.gate_proj", "model.layers.57.mlp.experts.26.gate_proj", "model.layers.57.mlp.experts.27.gate_proj", "model.layers.57.mlp.experts.28.gate_proj", "model.layers.57.mlp.experts.29.gate_proj", "model.layers.57.mlp.experts.30.gate_proj", "model.layers.57.mlp.experts.31.gate_proj", "model.layers.57.mlp.experts.32.gate_proj", "model.layers.57.mlp.experts.33.gate_proj", "model.layers.57.mlp.experts.34.gate_proj", "model.layers.57.mlp.experts.35.gate_proj", "model.layers.57.mlp.experts.36.gate_proj", "model.layers.57.mlp.experts.37.gate_proj", "model.layers.57.mlp.experts.38.gate_proj", "model.layers.57.mlp.experts.39.gate_proj", "model.layers.57.mlp.experts.40.gate_proj", "model.layers.57.mlp.experts.41.gate_proj", "model.layers.57.mlp.experts.42.gate_proj", "model.layers.57.mlp.experts.43.gate_proj", "model.layers.57.mlp.experts.44.gate_proj", "model.layers.57.mlp.experts.45.gate_proj", "model.layers.57.mlp.experts.46.gate_proj", "model.layers.57.mlp.experts.47.gate_proj", "model.layers.57.mlp.experts.48.gate_proj", "model.layers.57.mlp.experts.49.gate_proj", "model.layers.57.mlp.experts.50.gate_proj", "model.layers.57.mlp.experts.51.gate_proj", "model.layers.57.mlp.experts.52.gate_proj", "model.layers.57.mlp.experts.53.gate_proj", "model.layers.57.mlp.experts.54.gate_proj", "model.layers.57.mlp.experts.55.gate_proj", "model.layers.57.mlp.experts.56.gate_proj", "model.layers.57.mlp.experts.57.gate_proj", "model.layers.57.mlp.experts.58.gate_proj", "model.layers.57.mlp.experts.59.gate_proj", "model.layers.57.mlp.experts.60.gate_proj", "model.layers.57.mlp.experts.61.gate_proj", "model.layers.57.mlp.experts.62.gate_proj", "model.layers.57.mlp.experts.63.gate_proj", "model.layers.57.mlp.experts.64.gate_proj", "model.layers.57.mlp.experts.65.gate_proj", "model.layers.57.mlp.experts.66.gate_proj", "model.layers.57.mlp.experts.67.gate_proj", "model.layers.57.mlp.experts.68.gate_proj", "model.layers.57.mlp.experts.69.gate_proj", "model.layers.57.mlp.experts.70.gate_proj", "model.layers.57.mlp.experts.71.gate_proj", "model.layers.57.mlp.experts.72.gate_proj", "model.layers.57.mlp.experts.73.gate_proj", "model.layers.57.mlp.experts.74.gate_proj", "model.layers.57.mlp.experts.75.gate_proj", "model.layers.57.mlp.experts.76.gate_proj", "model.layers.57.mlp.experts.77.gate_proj", "model.layers.57.mlp.experts.78.gate_proj", "model.layers.57.mlp.experts.79.gate_proj", "model.layers.57.mlp.experts.80.gate_proj", "model.layers.57.mlp.experts.81.gate_proj", "model.layers.57.mlp.experts.82.gate_proj", "model.layers.57.mlp.experts.83.gate_proj", "model.layers.57.mlp.experts.84.gate_proj", "model.layers.57.mlp.experts.85.gate_proj", "model.layers.57.mlp.experts.86.gate_proj", "model.layers.57.mlp.experts.87.gate_proj", "model.layers.57.mlp.experts.88.gate_proj", "model.layers.57.mlp.experts.89.gate_proj", "model.layers.57.mlp.experts.90.gate_proj", "model.layers.57.mlp.experts.91.gate_proj", "model.layers.57.mlp.experts.92.gate_proj", "model.layers.57.mlp.experts.93.gate_proj", "model.layers.57.mlp.experts.94.gate_proj", "model.layers.57.mlp.experts.95.gate_proj", "model.layers.57.mlp.experts.96.gate_proj", "model.layers.57.mlp.experts.97.gate_proj", "model.layers.57.mlp.experts.98.gate_proj", "model.layers.57.mlp.experts.99.gate_proj", "model.layers.57.mlp.experts.100.gate_proj", "model.layers.57.mlp.experts.101.gate_proj", "model.layers.57.mlp.experts.102.gate_proj", "model.layers.57.mlp.experts.103.gate_proj", "model.layers.57.mlp.experts.104.gate_proj", "model.layers.57.mlp.experts.105.gate_proj", "model.layers.57.mlp.experts.106.gate_proj", "model.layers.57.mlp.experts.107.gate_proj", "model.layers.57.mlp.experts.108.gate_proj", "model.layers.57.mlp.experts.109.gate_proj", "model.layers.57.mlp.experts.110.gate_proj", "model.layers.57.mlp.experts.111.gate_proj", "model.layers.57.mlp.experts.112.gate_proj", "model.layers.57.mlp.experts.113.gate_proj", "model.layers.57.mlp.experts.114.gate_proj", "model.layers.57.mlp.experts.115.gate_proj", "model.layers.57.mlp.experts.116.gate_proj", "model.layers.57.mlp.experts.117.gate_proj", "model.layers.57.mlp.experts.118.gate_proj", "model.layers.57.mlp.experts.119.gate_proj", "model.layers.57.mlp.experts.120.gate_proj", "model.layers.57.mlp.experts.121.gate_proj", "model.layers.57.mlp.experts.122.gate_proj", "model.layers.57.mlp.experts.123.gate_proj", "model.layers.57.mlp.experts.124.gate_proj", "model.layers.57.mlp.experts.125.gate_proj", "model.layers.57.mlp.experts.126.gate_proj", "model.layers.57.mlp.experts.127.gate_proj", "model.layers.57.mlp.experts.128.gate_proj", "model.layers.57.mlp.experts.129.gate_proj", "model.layers.57.mlp.experts.130.gate_proj", "model.layers.57.mlp.experts.131.gate_proj", "model.layers.57.mlp.experts.132.gate_proj", "model.layers.57.mlp.experts.133.gate_proj", "model.layers.57.mlp.experts.134.gate_proj", "model.layers.57.mlp.experts.135.gate_proj", "model.layers.57.mlp.experts.136.gate_proj", "model.layers.57.mlp.experts.137.gate_proj", "model.layers.57.mlp.experts.138.gate_proj", "model.layers.57.mlp.experts.139.gate_proj", "model.layers.57.mlp.experts.140.gate_proj", "model.layers.57.mlp.experts.141.gate_proj", "model.layers.57.mlp.experts.142.gate_proj", "model.layers.57.mlp.experts.143.gate_proj", "model.layers.57.mlp.experts.144.gate_proj", "model.layers.57.mlp.experts.145.gate_proj", "model.layers.57.mlp.experts.146.gate_proj", "model.layers.57.mlp.experts.147.gate_proj", "model.layers.57.mlp.experts.148.gate_proj", "model.layers.57.mlp.experts.149.gate_proj", "model.layers.57.mlp.experts.150.gate_proj", "model.layers.57.mlp.experts.151.gate_proj", "model.layers.57.mlp.experts.152.gate_proj", "model.layers.57.mlp.experts.153.gate_proj", "model.layers.57.mlp.experts.154.gate_proj", "model.layers.57.mlp.experts.155.gate_proj", "model.layers.57.mlp.experts.156.gate_proj", "model.layers.57.mlp.experts.157.gate_proj", "model.layers.57.mlp.experts.158.gate_proj", "model.layers.57.mlp.experts.159.gate_proj", "model.layers.57.mlp.experts.0.up_proj", "model.layers.57.mlp.experts.1.up_proj", "model.layers.57.mlp.experts.2.up_proj", "model.layers.57.mlp.experts.3.up_proj", "model.layers.57.mlp.experts.4.up_proj", "model.layers.57.mlp.experts.5.up_proj", "model.layers.57.mlp.experts.6.up_proj", "model.layers.57.mlp.experts.7.up_proj", "model.layers.57.mlp.experts.8.up_proj", "model.layers.57.mlp.experts.9.up_proj", "model.layers.57.mlp.experts.10.up_proj", "model.layers.57.mlp.experts.11.up_proj", "model.layers.57.mlp.experts.12.up_proj", "model.layers.57.mlp.experts.13.up_proj", "model.layers.57.mlp.experts.14.up_proj", "model.layers.57.mlp.experts.15.up_proj", "model.layers.57.mlp.experts.16.up_proj", "model.layers.57.mlp.experts.17.up_proj", "model.layers.57.mlp.experts.18.up_proj", "model.layers.57.mlp.experts.19.up_proj", "model.layers.57.mlp.experts.20.up_proj", "model.layers.57.mlp.experts.21.up_proj", "model.layers.57.mlp.experts.22.up_proj", "model.layers.57.mlp.experts.23.up_proj", "model.layers.57.mlp.experts.24.up_proj", "model.layers.57.mlp.experts.25.up_proj", "model.layers.57.mlp.experts.26.up_proj", "model.layers.57.mlp.experts.27.up_proj", "model.layers.57.mlp.experts.28.up_proj", "model.layers.57.mlp.experts.29.up_proj", "model.layers.57.mlp.experts.30.up_proj", "model.layers.57.mlp.experts.31.up_proj", "model.layers.57.mlp.experts.32.up_proj", "model.layers.57.mlp.experts.33.up_proj", "model.layers.57.mlp.experts.34.up_proj", "model.layers.57.mlp.experts.35.up_proj", "model.layers.57.mlp.experts.36.up_proj", "model.layers.57.mlp.experts.37.up_proj", "model.layers.57.mlp.experts.38.up_proj", "model.layers.57.mlp.experts.39.up_proj", "model.layers.57.mlp.experts.40.up_proj", "model.layers.57.mlp.experts.41.up_proj", "model.layers.57.mlp.experts.42.up_proj", "model.layers.57.mlp.experts.43.up_proj", "model.layers.57.mlp.experts.44.up_proj", "model.layers.57.mlp.experts.45.up_proj", "model.layers.57.mlp.experts.46.up_proj", "model.layers.57.mlp.experts.47.up_proj", "model.layers.57.mlp.experts.48.up_proj", "model.layers.57.mlp.experts.49.up_proj", "model.layers.57.mlp.experts.50.up_proj", "model.layers.57.mlp.experts.51.up_proj", "model.layers.57.mlp.experts.52.up_proj", "model.layers.57.mlp.experts.53.up_proj", "model.layers.57.mlp.experts.54.up_proj", "model.layers.57.mlp.experts.55.up_proj", "model.layers.57.mlp.experts.56.up_proj", "model.layers.57.mlp.experts.57.up_proj", "model.layers.57.mlp.experts.58.up_proj", "model.layers.57.mlp.experts.59.up_proj", "model.layers.57.mlp.experts.60.up_proj", "model.layers.57.mlp.experts.61.up_proj", "model.layers.57.mlp.experts.62.up_proj", "model.layers.57.mlp.experts.63.up_proj", "model.layers.57.mlp.experts.64.up_proj", "model.layers.57.mlp.experts.65.up_proj", "model.layers.57.mlp.experts.66.up_proj", "model.layers.57.mlp.experts.67.up_proj", "model.layers.57.mlp.experts.68.up_proj", "model.layers.57.mlp.experts.69.up_proj", "model.layers.57.mlp.experts.70.up_proj", "model.layers.57.mlp.experts.71.up_proj", "model.layers.57.mlp.experts.72.up_proj", "model.layers.57.mlp.experts.73.up_proj", "model.layers.57.mlp.experts.74.up_proj", "model.layers.57.mlp.experts.75.up_proj", "model.layers.57.mlp.experts.76.up_proj", "model.layers.57.mlp.experts.77.up_proj", "model.layers.57.mlp.experts.78.up_proj", "model.layers.57.mlp.experts.79.up_proj", "model.layers.57.mlp.experts.80.up_proj", "model.layers.57.mlp.experts.81.up_proj", "model.layers.57.mlp.experts.82.up_proj", "model.layers.57.mlp.experts.83.up_proj", "model.layers.57.mlp.experts.84.up_proj", "model.layers.57.mlp.experts.85.up_proj", "model.layers.57.mlp.experts.86.up_proj", "model.layers.57.mlp.experts.87.up_proj", "model.layers.57.mlp.experts.88.up_proj", "model.layers.57.mlp.experts.89.up_proj", "model.layers.57.mlp.experts.90.up_proj", "model.layers.57.mlp.experts.91.up_proj", "model.layers.57.mlp.experts.92.up_proj", "model.layers.57.mlp.experts.93.up_proj", "model.layers.57.mlp.experts.94.up_proj", "model.layers.57.mlp.experts.95.up_proj", "model.layers.57.mlp.experts.96.up_proj", "model.layers.57.mlp.experts.97.up_proj", "model.layers.57.mlp.experts.98.up_proj", "model.layers.57.mlp.experts.99.up_proj", "model.layers.57.mlp.experts.100.up_proj", "model.layers.57.mlp.experts.101.up_proj", "model.layers.57.mlp.experts.102.up_proj", "model.layers.57.mlp.experts.103.up_proj", "model.layers.57.mlp.experts.104.up_proj", "model.layers.57.mlp.experts.105.up_proj", "model.layers.57.mlp.experts.106.up_proj", "model.layers.57.mlp.experts.107.up_proj", "model.layers.57.mlp.experts.108.up_proj", "model.layers.57.mlp.experts.109.up_proj", "model.layers.57.mlp.experts.110.up_proj", "model.layers.57.mlp.experts.111.up_proj", "model.layers.57.mlp.experts.112.up_proj", "model.layers.57.mlp.experts.113.up_proj", "model.layers.57.mlp.experts.114.up_proj", "model.layers.57.mlp.experts.115.up_proj", "model.layers.57.mlp.experts.116.up_proj", "model.layers.57.mlp.experts.117.up_proj", "model.layers.57.mlp.experts.118.up_proj", "model.layers.57.mlp.experts.119.up_proj", "model.layers.57.mlp.experts.120.up_proj", "model.layers.57.mlp.experts.121.up_proj", "model.layers.57.mlp.experts.122.up_proj", "model.layers.57.mlp.experts.123.up_proj", "model.layers.57.mlp.experts.124.up_proj", "model.layers.57.mlp.experts.125.up_proj", "model.layers.57.mlp.experts.126.up_proj", "model.layers.57.mlp.experts.127.up_proj", "model.layers.57.mlp.experts.128.up_proj", "model.layers.57.mlp.experts.129.up_proj", "model.layers.57.mlp.experts.130.up_proj", "model.layers.57.mlp.experts.131.up_proj", "model.layers.57.mlp.experts.132.up_proj", "model.layers.57.mlp.experts.133.up_proj", "model.layers.57.mlp.experts.134.up_proj", "model.layers.57.mlp.experts.135.up_proj", "model.layers.57.mlp.experts.136.up_proj", "model.layers.57.mlp.experts.137.up_proj", "model.layers.57.mlp.experts.138.up_proj", "model.layers.57.mlp.experts.139.up_proj", "model.layers.57.mlp.experts.140.up_proj", "model.layers.57.mlp.experts.141.up_proj", "model.layers.57.mlp.experts.142.up_proj", "model.layers.57.mlp.experts.143.up_proj", "model.layers.57.mlp.experts.144.up_proj", "model.layers.57.mlp.experts.145.up_proj", "model.layers.57.mlp.experts.146.up_proj", "model.layers.57.mlp.experts.147.up_proj", "model.layers.57.mlp.experts.148.up_proj", "model.layers.57.mlp.experts.149.up_proj", "model.layers.57.mlp.experts.150.up_proj", "model.layers.57.mlp.experts.151.up_proj", "model.layers.57.mlp.experts.152.up_proj", "model.layers.57.mlp.experts.153.up_proj", "model.layers.57.mlp.experts.154.up_proj", "model.layers.57.mlp.experts.155.up_proj", "model.layers.57.mlp.experts.156.up_proj", "model.layers.57.mlp.experts.157.up_proj", "model.layers.57.mlp.experts.158.up_proj", "model.layers.57.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0010848678648471832, "dbits": 2516582400 } ] }, { "idx": 344, "layers": [ "model.layers.57.mlp.experts.0.down_proj", "model.layers.57.mlp.experts.1.down_proj", "model.layers.57.mlp.experts.2.down_proj", "model.layers.57.mlp.experts.3.down_proj", "model.layers.57.mlp.experts.4.down_proj", "model.layers.57.mlp.experts.5.down_proj", "model.layers.57.mlp.experts.6.down_proj", "model.layers.57.mlp.experts.7.down_proj", "model.layers.57.mlp.experts.8.down_proj", "model.layers.57.mlp.experts.9.down_proj", "model.layers.57.mlp.experts.10.down_proj", "model.layers.57.mlp.experts.11.down_proj", "model.layers.57.mlp.experts.12.down_proj", "model.layers.57.mlp.experts.13.down_proj", "model.layers.57.mlp.experts.14.down_proj", "model.layers.57.mlp.experts.15.down_proj", "model.layers.57.mlp.experts.16.down_proj", "model.layers.57.mlp.experts.17.down_proj", "model.layers.57.mlp.experts.18.down_proj", "model.layers.57.mlp.experts.19.down_proj", "model.layers.57.mlp.experts.20.down_proj", "model.layers.57.mlp.experts.21.down_proj", "model.layers.57.mlp.experts.22.down_proj", "model.layers.57.mlp.experts.23.down_proj", "model.layers.57.mlp.experts.24.down_proj", "model.layers.57.mlp.experts.25.down_proj", "model.layers.57.mlp.experts.26.down_proj", "model.layers.57.mlp.experts.27.down_proj", "model.layers.57.mlp.experts.28.down_proj", "model.layers.57.mlp.experts.29.down_proj", "model.layers.57.mlp.experts.30.down_proj", "model.layers.57.mlp.experts.31.down_proj", "model.layers.57.mlp.experts.32.down_proj", "model.layers.57.mlp.experts.33.down_proj", "model.layers.57.mlp.experts.34.down_proj", "model.layers.57.mlp.experts.35.down_proj", "model.layers.57.mlp.experts.36.down_proj", "model.layers.57.mlp.experts.37.down_proj", "model.layers.57.mlp.experts.38.down_proj", "model.layers.57.mlp.experts.39.down_proj", "model.layers.57.mlp.experts.40.down_proj", "model.layers.57.mlp.experts.41.down_proj", "model.layers.57.mlp.experts.42.down_proj", "model.layers.57.mlp.experts.43.down_proj", "model.layers.57.mlp.experts.44.down_proj", "model.layers.57.mlp.experts.45.down_proj", "model.layers.57.mlp.experts.46.down_proj", "model.layers.57.mlp.experts.47.down_proj", "model.layers.57.mlp.experts.48.down_proj", "model.layers.57.mlp.experts.49.down_proj", "model.layers.57.mlp.experts.50.down_proj", "model.layers.57.mlp.experts.51.down_proj", "model.layers.57.mlp.experts.52.down_proj", "model.layers.57.mlp.experts.53.down_proj", "model.layers.57.mlp.experts.54.down_proj", "model.layers.57.mlp.experts.55.down_proj", "model.layers.57.mlp.experts.56.down_proj", "model.layers.57.mlp.experts.57.down_proj", "model.layers.57.mlp.experts.58.down_proj", "model.layers.57.mlp.experts.59.down_proj", "model.layers.57.mlp.experts.60.down_proj", "model.layers.57.mlp.experts.61.down_proj", "model.layers.57.mlp.experts.62.down_proj", "model.layers.57.mlp.experts.63.down_proj", "model.layers.57.mlp.experts.64.down_proj", "model.layers.57.mlp.experts.65.down_proj", "model.layers.57.mlp.experts.66.down_proj", "model.layers.57.mlp.experts.67.down_proj", "model.layers.57.mlp.experts.68.down_proj", "model.layers.57.mlp.experts.69.down_proj", "model.layers.57.mlp.experts.70.down_proj", "model.layers.57.mlp.experts.71.down_proj", "model.layers.57.mlp.experts.72.down_proj", "model.layers.57.mlp.experts.73.down_proj", "model.layers.57.mlp.experts.74.down_proj", "model.layers.57.mlp.experts.75.down_proj", "model.layers.57.mlp.experts.76.down_proj", "model.layers.57.mlp.experts.77.down_proj", "model.layers.57.mlp.experts.78.down_proj", "model.layers.57.mlp.experts.79.down_proj", "model.layers.57.mlp.experts.80.down_proj", "model.layers.57.mlp.experts.81.down_proj", "model.layers.57.mlp.experts.82.down_proj", "model.layers.57.mlp.experts.83.down_proj", "model.layers.57.mlp.experts.84.down_proj", "model.layers.57.mlp.experts.85.down_proj", "model.layers.57.mlp.experts.86.down_proj", "model.layers.57.mlp.experts.87.down_proj", "model.layers.57.mlp.experts.88.down_proj", "model.layers.57.mlp.experts.89.down_proj", "model.layers.57.mlp.experts.90.down_proj", "model.layers.57.mlp.experts.91.down_proj", "model.layers.57.mlp.experts.92.down_proj", "model.layers.57.mlp.experts.93.down_proj", "model.layers.57.mlp.experts.94.down_proj", "model.layers.57.mlp.experts.95.down_proj", "model.layers.57.mlp.experts.96.down_proj", "model.layers.57.mlp.experts.97.down_proj", "model.layers.57.mlp.experts.98.down_proj", "model.layers.57.mlp.experts.99.down_proj", "model.layers.57.mlp.experts.100.down_proj", "model.layers.57.mlp.experts.101.down_proj", "model.layers.57.mlp.experts.102.down_proj", "model.layers.57.mlp.experts.103.down_proj", "model.layers.57.mlp.experts.104.down_proj", "model.layers.57.mlp.experts.105.down_proj", "model.layers.57.mlp.experts.106.down_proj", "model.layers.57.mlp.experts.107.down_proj", "model.layers.57.mlp.experts.108.down_proj", "model.layers.57.mlp.experts.109.down_proj", "model.layers.57.mlp.experts.110.down_proj", "model.layers.57.mlp.experts.111.down_proj", "model.layers.57.mlp.experts.112.down_proj", "model.layers.57.mlp.experts.113.down_proj", "model.layers.57.mlp.experts.114.down_proj", "model.layers.57.mlp.experts.115.down_proj", "model.layers.57.mlp.experts.116.down_proj", "model.layers.57.mlp.experts.117.down_proj", "model.layers.57.mlp.experts.118.down_proj", "model.layers.57.mlp.experts.119.down_proj", "model.layers.57.mlp.experts.120.down_proj", "model.layers.57.mlp.experts.121.down_proj", "model.layers.57.mlp.experts.122.down_proj", "model.layers.57.mlp.experts.123.down_proj", "model.layers.57.mlp.experts.124.down_proj", "model.layers.57.mlp.experts.125.down_proj", "model.layers.57.mlp.experts.126.down_proj", "model.layers.57.mlp.experts.127.down_proj", "model.layers.57.mlp.experts.128.down_proj", "model.layers.57.mlp.experts.129.down_proj", "model.layers.57.mlp.experts.130.down_proj", "model.layers.57.mlp.experts.131.down_proj", "model.layers.57.mlp.experts.132.down_proj", "model.layers.57.mlp.experts.133.down_proj", "model.layers.57.mlp.experts.134.down_proj", "model.layers.57.mlp.experts.135.down_proj", "model.layers.57.mlp.experts.136.down_proj", "model.layers.57.mlp.experts.137.down_proj", "model.layers.57.mlp.experts.138.down_proj", "model.layers.57.mlp.experts.139.down_proj", "model.layers.57.mlp.experts.140.down_proj", "model.layers.57.mlp.experts.141.down_proj", "model.layers.57.mlp.experts.142.down_proj", "model.layers.57.mlp.experts.143.down_proj", "model.layers.57.mlp.experts.144.down_proj", "model.layers.57.mlp.experts.145.down_proj", "model.layers.57.mlp.experts.146.down_proj", "model.layers.57.mlp.experts.147.down_proj", "model.layers.57.mlp.experts.148.down_proj", "model.layers.57.mlp.experts.149.down_proj", "model.layers.57.mlp.experts.150.down_proj", "model.layers.57.mlp.experts.151.down_proj", "model.layers.57.mlp.experts.152.down_proj", "model.layers.57.mlp.experts.153.down_proj", "model.layers.57.mlp.experts.154.down_proj", "model.layers.57.mlp.experts.155.down_proj", "model.layers.57.mlp.experts.156.down_proj", "model.layers.57.mlp.experts.157.down_proj", "model.layers.57.mlp.experts.158.down_proj", "model.layers.57.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0005262926220894193, "dbits": 1258291200 } ] }, { "idx": 345, "layers": [ "model.layers.58.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0007946036756038555, "dbits": 62914560 } ] }, { "idx": 346, "layers": [ "model.layers.58.self_attn.k_proj", "model.layers.58.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0014794848859310261, "dbits": 10485760 } ] }, { "idx": 347, "layers": [ "model.layers.58.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0001347079873084689, "dbits": 62914560 } ] }, { "idx": 348, "layers": [ "model.layers.58.mlp.shared_experts.gate_proj", "model.layers.58.mlp.shared_experts.up_proj", "model.layers.58.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -6.56127929687389e-05, "dbits": 23592960 } ] }, { "idx": 349, "layers": [ "model.layers.58.mlp.experts.0.gate_proj", "model.layers.58.mlp.experts.1.gate_proj", "model.layers.58.mlp.experts.2.gate_proj", "model.layers.58.mlp.experts.3.gate_proj", "model.layers.58.mlp.experts.4.gate_proj", "model.layers.58.mlp.experts.5.gate_proj", "model.layers.58.mlp.experts.6.gate_proj", "model.layers.58.mlp.experts.7.gate_proj", "model.layers.58.mlp.experts.8.gate_proj", "model.layers.58.mlp.experts.9.gate_proj", "model.layers.58.mlp.experts.10.gate_proj", "model.layers.58.mlp.experts.11.gate_proj", "model.layers.58.mlp.experts.12.gate_proj", "model.layers.58.mlp.experts.13.gate_proj", "model.layers.58.mlp.experts.14.gate_proj", "model.layers.58.mlp.experts.15.gate_proj", "model.layers.58.mlp.experts.16.gate_proj", "model.layers.58.mlp.experts.17.gate_proj", "model.layers.58.mlp.experts.18.gate_proj", "model.layers.58.mlp.experts.19.gate_proj", "model.layers.58.mlp.experts.20.gate_proj", "model.layers.58.mlp.experts.21.gate_proj", "model.layers.58.mlp.experts.22.gate_proj", "model.layers.58.mlp.experts.23.gate_proj", "model.layers.58.mlp.experts.24.gate_proj", "model.layers.58.mlp.experts.25.gate_proj", "model.layers.58.mlp.experts.26.gate_proj", "model.layers.58.mlp.experts.27.gate_proj", "model.layers.58.mlp.experts.28.gate_proj", "model.layers.58.mlp.experts.29.gate_proj", "model.layers.58.mlp.experts.30.gate_proj", "model.layers.58.mlp.experts.31.gate_proj", "model.layers.58.mlp.experts.32.gate_proj", "model.layers.58.mlp.experts.33.gate_proj", "model.layers.58.mlp.experts.34.gate_proj", "model.layers.58.mlp.experts.35.gate_proj", "model.layers.58.mlp.experts.36.gate_proj", "model.layers.58.mlp.experts.37.gate_proj", "model.layers.58.mlp.experts.38.gate_proj", "model.layers.58.mlp.experts.39.gate_proj", "model.layers.58.mlp.experts.40.gate_proj", "model.layers.58.mlp.experts.41.gate_proj", "model.layers.58.mlp.experts.42.gate_proj", "model.layers.58.mlp.experts.43.gate_proj", "model.layers.58.mlp.experts.44.gate_proj", "model.layers.58.mlp.experts.45.gate_proj", "model.layers.58.mlp.experts.46.gate_proj", "model.layers.58.mlp.experts.47.gate_proj", "model.layers.58.mlp.experts.48.gate_proj", "model.layers.58.mlp.experts.49.gate_proj", "model.layers.58.mlp.experts.50.gate_proj", "model.layers.58.mlp.experts.51.gate_proj", "model.layers.58.mlp.experts.52.gate_proj", "model.layers.58.mlp.experts.53.gate_proj", "model.layers.58.mlp.experts.54.gate_proj", "model.layers.58.mlp.experts.55.gate_proj", "model.layers.58.mlp.experts.56.gate_proj", "model.layers.58.mlp.experts.57.gate_proj", "model.layers.58.mlp.experts.58.gate_proj", "model.layers.58.mlp.experts.59.gate_proj", "model.layers.58.mlp.experts.60.gate_proj", "model.layers.58.mlp.experts.61.gate_proj", "model.layers.58.mlp.experts.62.gate_proj", "model.layers.58.mlp.experts.63.gate_proj", "model.layers.58.mlp.experts.64.gate_proj", "model.layers.58.mlp.experts.65.gate_proj", "model.layers.58.mlp.experts.66.gate_proj", "model.layers.58.mlp.experts.67.gate_proj", "model.layers.58.mlp.experts.68.gate_proj", "model.layers.58.mlp.experts.69.gate_proj", "model.layers.58.mlp.experts.70.gate_proj", "model.layers.58.mlp.experts.71.gate_proj", "model.layers.58.mlp.experts.72.gate_proj", "model.layers.58.mlp.experts.73.gate_proj", "model.layers.58.mlp.experts.74.gate_proj", "model.layers.58.mlp.experts.75.gate_proj", "model.layers.58.mlp.experts.76.gate_proj", "model.layers.58.mlp.experts.77.gate_proj", "model.layers.58.mlp.experts.78.gate_proj", "model.layers.58.mlp.experts.79.gate_proj", "model.layers.58.mlp.experts.80.gate_proj", "model.layers.58.mlp.experts.81.gate_proj", "model.layers.58.mlp.experts.82.gate_proj", "model.layers.58.mlp.experts.83.gate_proj", "model.layers.58.mlp.experts.84.gate_proj", "model.layers.58.mlp.experts.85.gate_proj", "model.layers.58.mlp.experts.86.gate_proj", "model.layers.58.mlp.experts.87.gate_proj", "model.layers.58.mlp.experts.88.gate_proj", "model.layers.58.mlp.experts.89.gate_proj", "model.layers.58.mlp.experts.90.gate_proj", "model.layers.58.mlp.experts.91.gate_proj", "model.layers.58.mlp.experts.92.gate_proj", "model.layers.58.mlp.experts.93.gate_proj", "model.layers.58.mlp.experts.94.gate_proj", "model.layers.58.mlp.experts.95.gate_proj", "model.layers.58.mlp.experts.96.gate_proj", "model.layers.58.mlp.experts.97.gate_proj", "model.layers.58.mlp.experts.98.gate_proj", "model.layers.58.mlp.experts.99.gate_proj", "model.layers.58.mlp.experts.100.gate_proj", "model.layers.58.mlp.experts.101.gate_proj", "model.layers.58.mlp.experts.102.gate_proj", "model.layers.58.mlp.experts.103.gate_proj", "model.layers.58.mlp.experts.104.gate_proj", "model.layers.58.mlp.experts.105.gate_proj", "model.layers.58.mlp.experts.106.gate_proj", "model.layers.58.mlp.experts.107.gate_proj", "model.layers.58.mlp.experts.108.gate_proj", "model.layers.58.mlp.experts.109.gate_proj", "model.layers.58.mlp.experts.110.gate_proj", "model.layers.58.mlp.experts.111.gate_proj", "model.layers.58.mlp.experts.112.gate_proj", "model.layers.58.mlp.experts.113.gate_proj", "model.layers.58.mlp.experts.114.gate_proj", "model.layers.58.mlp.experts.115.gate_proj", "model.layers.58.mlp.experts.116.gate_proj", "model.layers.58.mlp.experts.117.gate_proj", "model.layers.58.mlp.experts.118.gate_proj", "model.layers.58.mlp.experts.119.gate_proj", "model.layers.58.mlp.experts.120.gate_proj", "model.layers.58.mlp.experts.121.gate_proj", "model.layers.58.mlp.experts.122.gate_proj", "model.layers.58.mlp.experts.123.gate_proj", "model.layers.58.mlp.experts.124.gate_proj", "model.layers.58.mlp.experts.125.gate_proj", "model.layers.58.mlp.experts.126.gate_proj", "model.layers.58.mlp.experts.127.gate_proj", "model.layers.58.mlp.experts.128.gate_proj", "model.layers.58.mlp.experts.129.gate_proj", "model.layers.58.mlp.experts.130.gate_proj", "model.layers.58.mlp.experts.131.gate_proj", "model.layers.58.mlp.experts.132.gate_proj", "model.layers.58.mlp.experts.133.gate_proj", "model.layers.58.mlp.experts.134.gate_proj", "model.layers.58.mlp.experts.135.gate_proj", "model.layers.58.mlp.experts.136.gate_proj", "model.layers.58.mlp.experts.137.gate_proj", "model.layers.58.mlp.experts.138.gate_proj", "model.layers.58.mlp.experts.139.gate_proj", "model.layers.58.mlp.experts.140.gate_proj", "model.layers.58.mlp.experts.141.gate_proj", "model.layers.58.mlp.experts.142.gate_proj", "model.layers.58.mlp.experts.143.gate_proj", "model.layers.58.mlp.experts.144.gate_proj", "model.layers.58.mlp.experts.145.gate_proj", "model.layers.58.mlp.experts.146.gate_proj", "model.layers.58.mlp.experts.147.gate_proj", "model.layers.58.mlp.experts.148.gate_proj", "model.layers.58.mlp.experts.149.gate_proj", "model.layers.58.mlp.experts.150.gate_proj", "model.layers.58.mlp.experts.151.gate_proj", "model.layers.58.mlp.experts.152.gate_proj", "model.layers.58.mlp.experts.153.gate_proj", "model.layers.58.mlp.experts.154.gate_proj", "model.layers.58.mlp.experts.155.gate_proj", "model.layers.58.mlp.experts.156.gate_proj", "model.layers.58.mlp.experts.157.gate_proj", "model.layers.58.mlp.experts.158.gate_proj", "model.layers.58.mlp.experts.159.gate_proj", "model.layers.58.mlp.experts.0.up_proj", "model.layers.58.mlp.experts.1.up_proj", "model.layers.58.mlp.experts.2.up_proj", "model.layers.58.mlp.experts.3.up_proj", "model.layers.58.mlp.experts.4.up_proj", "model.layers.58.mlp.experts.5.up_proj", "model.layers.58.mlp.experts.6.up_proj", "model.layers.58.mlp.experts.7.up_proj", "model.layers.58.mlp.experts.8.up_proj", "model.layers.58.mlp.experts.9.up_proj", "model.layers.58.mlp.experts.10.up_proj", "model.layers.58.mlp.experts.11.up_proj", "model.layers.58.mlp.experts.12.up_proj", "model.layers.58.mlp.experts.13.up_proj", "model.layers.58.mlp.experts.14.up_proj", "model.layers.58.mlp.experts.15.up_proj", "model.layers.58.mlp.experts.16.up_proj", "model.layers.58.mlp.experts.17.up_proj", "model.layers.58.mlp.experts.18.up_proj", "model.layers.58.mlp.experts.19.up_proj", "model.layers.58.mlp.experts.20.up_proj", "model.layers.58.mlp.experts.21.up_proj", "model.layers.58.mlp.experts.22.up_proj", "model.layers.58.mlp.experts.23.up_proj", "model.layers.58.mlp.experts.24.up_proj", "model.layers.58.mlp.experts.25.up_proj", "model.layers.58.mlp.experts.26.up_proj", "model.layers.58.mlp.experts.27.up_proj", "model.layers.58.mlp.experts.28.up_proj", "model.layers.58.mlp.experts.29.up_proj", "model.layers.58.mlp.experts.30.up_proj", "model.layers.58.mlp.experts.31.up_proj", "model.layers.58.mlp.experts.32.up_proj", "model.layers.58.mlp.experts.33.up_proj", "model.layers.58.mlp.experts.34.up_proj", "model.layers.58.mlp.experts.35.up_proj", "model.layers.58.mlp.experts.36.up_proj", "model.layers.58.mlp.experts.37.up_proj", "model.layers.58.mlp.experts.38.up_proj", "model.layers.58.mlp.experts.39.up_proj", "model.layers.58.mlp.experts.40.up_proj", "model.layers.58.mlp.experts.41.up_proj", "model.layers.58.mlp.experts.42.up_proj", "model.layers.58.mlp.experts.43.up_proj", "model.layers.58.mlp.experts.44.up_proj", "model.layers.58.mlp.experts.45.up_proj", "model.layers.58.mlp.experts.46.up_proj", "model.layers.58.mlp.experts.47.up_proj", "model.layers.58.mlp.experts.48.up_proj", "model.layers.58.mlp.experts.49.up_proj", "model.layers.58.mlp.experts.50.up_proj", "model.layers.58.mlp.experts.51.up_proj", "model.layers.58.mlp.experts.52.up_proj", "model.layers.58.mlp.experts.53.up_proj", "model.layers.58.mlp.experts.54.up_proj", "model.layers.58.mlp.experts.55.up_proj", "model.layers.58.mlp.experts.56.up_proj", "model.layers.58.mlp.experts.57.up_proj", "model.layers.58.mlp.experts.58.up_proj", "model.layers.58.mlp.experts.59.up_proj", "model.layers.58.mlp.experts.60.up_proj", "model.layers.58.mlp.experts.61.up_proj", "model.layers.58.mlp.experts.62.up_proj", "model.layers.58.mlp.experts.63.up_proj", "model.layers.58.mlp.experts.64.up_proj", "model.layers.58.mlp.experts.65.up_proj", "model.layers.58.mlp.experts.66.up_proj", "model.layers.58.mlp.experts.67.up_proj", "model.layers.58.mlp.experts.68.up_proj", "model.layers.58.mlp.experts.69.up_proj", "model.layers.58.mlp.experts.70.up_proj", "model.layers.58.mlp.experts.71.up_proj", "model.layers.58.mlp.experts.72.up_proj", "model.layers.58.mlp.experts.73.up_proj", "model.layers.58.mlp.experts.74.up_proj", "model.layers.58.mlp.experts.75.up_proj", "model.layers.58.mlp.experts.76.up_proj", "model.layers.58.mlp.experts.77.up_proj", "model.layers.58.mlp.experts.78.up_proj", "model.layers.58.mlp.experts.79.up_proj", "model.layers.58.mlp.experts.80.up_proj", "model.layers.58.mlp.experts.81.up_proj", "model.layers.58.mlp.experts.82.up_proj", "model.layers.58.mlp.experts.83.up_proj", "model.layers.58.mlp.experts.84.up_proj", "model.layers.58.mlp.experts.85.up_proj", "model.layers.58.mlp.experts.86.up_proj", "model.layers.58.mlp.experts.87.up_proj", "model.layers.58.mlp.experts.88.up_proj", "model.layers.58.mlp.experts.89.up_proj", "model.layers.58.mlp.experts.90.up_proj", "model.layers.58.mlp.experts.91.up_proj", "model.layers.58.mlp.experts.92.up_proj", "model.layers.58.mlp.experts.93.up_proj", "model.layers.58.mlp.experts.94.up_proj", "model.layers.58.mlp.experts.95.up_proj", "model.layers.58.mlp.experts.96.up_proj", "model.layers.58.mlp.experts.97.up_proj", "model.layers.58.mlp.experts.98.up_proj", "model.layers.58.mlp.experts.99.up_proj", "model.layers.58.mlp.experts.100.up_proj", "model.layers.58.mlp.experts.101.up_proj", "model.layers.58.mlp.experts.102.up_proj", "model.layers.58.mlp.experts.103.up_proj", "model.layers.58.mlp.experts.104.up_proj", "model.layers.58.mlp.experts.105.up_proj", "model.layers.58.mlp.experts.106.up_proj", "model.layers.58.mlp.experts.107.up_proj", "model.layers.58.mlp.experts.108.up_proj", "model.layers.58.mlp.experts.109.up_proj", "model.layers.58.mlp.experts.110.up_proj", "model.layers.58.mlp.experts.111.up_proj", "model.layers.58.mlp.experts.112.up_proj", "model.layers.58.mlp.experts.113.up_proj", "model.layers.58.mlp.experts.114.up_proj", "model.layers.58.mlp.experts.115.up_proj", "model.layers.58.mlp.experts.116.up_proj", "model.layers.58.mlp.experts.117.up_proj", "model.layers.58.mlp.experts.118.up_proj", "model.layers.58.mlp.experts.119.up_proj", "model.layers.58.mlp.experts.120.up_proj", "model.layers.58.mlp.experts.121.up_proj", "model.layers.58.mlp.experts.122.up_proj", "model.layers.58.mlp.experts.123.up_proj", "model.layers.58.mlp.experts.124.up_proj", "model.layers.58.mlp.experts.125.up_proj", "model.layers.58.mlp.experts.126.up_proj", "model.layers.58.mlp.experts.127.up_proj", "model.layers.58.mlp.experts.128.up_proj", "model.layers.58.mlp.experts.129.up_proj", "model.layers.58.mlp.experts.130.up_proj", "model.layers.58.mlp.experts.131.up_proj", "model.layers.58.mlp.experts.132.up_proj", "model.layers.58.mlp.experts.133.up_proj", "model.layers.58.mlp.experts.134.up_proj", "model.layers.58.mlp.experts.135.up_proj", "model.layers.58.mlp.experts.136.up_proj", "model.layers.58.mlp.experts.137.up_proj", "model.layers.58.mlp.experts.138.up_proj", "model.layers.58.mlp.experts.139.up_proj", "model.layers.58.mlp.experts.140.up_proj", "model.layers.58.mlp.experts.141.up_proj", "model.layers.58.mlp.experts.142.up_proj", "model.layers.58.mlp.experts.143.up_proj", "model.layers.58.mlp.experts.144.up_proj", "model.layers.58.mlp.experts.145.up_proj", "model.layers.58.mlp.experts.146.up_proj", "model.layers.58.mlp.experts.147.up_proj", "model.layers.58.mlp.experts.148.up_proj", "model.layers.58.mlp.experts.149.up_proj", "model.layers.58.mlp.experts.150.up_proj", "model.layers.58.mlp.experts.151.up_proj", "model.layers.58.mlp.experts.152.up_proj", "model.layers.58.mlp.experts.153.up_proj", "model.layers.58.mlp.experts.154.up_proj", "model.layers.58.mlp.experts.155.up_proj", "model.layers.58.mlp.experts.156.up_proj", "model.layers.58.mlp.experts.157.up_proj", "model.layers.58.mlp.experts.158.up_proj", "model.layers.58.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0010825976729393338, "dbits": 2516582400 } ] }, { "idx": 350, "layers": [ "model.layers.58.mlp.experts.0.down_proj", "model.layers.58.mlp.experts.1.down_proj", "model.layers.58.mlp.experts.2.down_proj", "model.layers.58.mlp.experts.3.down_proj", "model.layers.58.mlp.experts.4.down_proj", "model.layers.58.mlp.experts.5.down_proj", "model.layers.58.mlp.experts.6.down_proj", "model.layers.58.mlp.experts.7.down_proj", "model.layers.58.mlp.experts.8.down_proj", "model.layers.58.mlp.experts.9.down_proj", "model.layers.58.mlp.experts.10.down_proj", "model.layers.58.mlp.experts.11.down_proj", "model.layers.58.mlp.experts.12.down_proj", "model.layers.58.mlp.experts.13.down_proj", "model.layers.58.mlp.experts.14.down_proj", "model.layers.58.mlp.experts.15.down_proj", "model.layers.58.mlp.experts.16.down_proj", "model.layers.58.mlp.experts.17.down_proj", "model.layers.58.mlp.experts.18.down_proj", "model.layers.58.mlp.experts.19.down_proj", "model.layers.58.mlp.experts.20.down_proj", "model.layers.58.mlp.experts.21.down_proj", "model.layers.58.mlp.experts.22.down_proj", "model.layers.58.mlp.experts.23.down_proj", "model.layers.58.mlp.experts.24.down_proj", "model.layers.58.mlp.experts.25.down_proj", "model.layers.58.mlp.experts.26.down_proj", "model.layers.58.mlp.experts.27.down_proj", "model.layers.58.mlp.experts.28.down_proj", "model.layers.58.mlp.experts.29.down_proj", "model.layers.58.mlp.experts.30.down_proj", "model.layers.58.mlp.experts.31.down_proj", "model.layers.58.mlp.experts.32.down_proj", "model.layers.58.mlp.experts.33.down_proj", "model.layers.58.mlp.experts.34.down_proj", "model.layers.58.mlp.experts.35.down_proj", "model.layers.58.mlp.experts.36.down_proj", "model.layers.58.mlp.experts.37.down_proj", "model.layers.58.mlp.experts.38.down_proj", "model.layers.58.mlp.experts.39.down_proj", "model.layers.58.mlp.experts.40.down_proj", "model.layers.58.mlp.experts.41.down_proj", "model.layers.58.mlp.experts.42.down_proj", "model.layers.58.mlp.experts.43.down_proj", "model.layers.58.mlp.experts.44.down_proj", "model.layers.58.mlp.experts.45.down_proj", "model.layers.58.mlp.experts.46.down_proj", "model.layers.58.mlp.experts.47.down_proj", "model.layers.58.mlp.experts.48.down_proj", "model.layers.58.mlp.experts.49.down_proj", "model.layers.58.mlp.experts.50.down_proj", "model.layers.58.mlp.experts.51.down_proj", "model.layers.58.mlp.experts.52.down_proj", "model.layers.58.mlp.experts.53.down_proj", "model.layers.58.mlp.experts.54.down_proj", "model.layers.58.mlp.experts.55.down_proj", "model.layers.58.mlp.experts.56.down_proj", "model.layers.58.mlp.experts.57.down_proj", "model.layers.58.mlp.experts.58.down_proj", "model.layers.58.mlp.experts.59.down_proj", "model.layers.58.mlp.experts.60.down_proj", "model.layers.58.mlp.experts.61.down_proj", "model.layers.58.mlp.experts.62.down_proj", "model.layers.58.mlp.experts.63.down_proj", "model.layers.58.mlp.experts.64.down_proj", "model.layers.58.mlp.experts.65.down_proj", "model.layers.58.mlp.experts.66.down_proj", "model.layers.58.mlp.experts.67.down_proj", "model.layers.58.mlp.experts.68.down_proj", "model.layers.58.mlp.experts.69.down_proj", "model.layers.58.mlp.experts.70.down_proj", "model.layers.58.mlp.experts.71.down_proj", "model.layers.58.mlp.experts.72.down_proj", "model.layers.58.mlp.experts.73.down_proj", "model.layers.58.mlp.experts.74.down_proj", "model.layers.58.mlp.experts.75.down_proj", "model.layers.58.mlp.experts.76.down_proj", "model.layers.58.mlp.experts.77.down_proj", "model.layers.58.mlp.experts.78.down_proj", "model.layers.58.mlp.experts.79.down_proj", "model.layers.58.mlp.experts.80.down_proj", "model.layers.58.mlp.experts.81.down_proj", "model.layers.58.mlp.experts.82.down_proj", "model.layers.58.mlp.experts.83.down_proj", "model.layers.58.mlp.experts.84.down_proj", "model.layers.58.mlp.experts.85.down_proj", "model.layers.58.mlp.experts.86.down_proj", "model.layers.58.mlp.experts.87.down_proj", "model.layers.58.mlp.experts.88.down_proj", "model.layers.58.mlp.experts.89.down_proj", "model.layers.58.mlp.experts.90.down_proj", "model.layers.58.mlp.experts.91.down_proj", "model.layers.58.mlp.experts.92.down_proj", "model.layers.58.mlp.experts.93.down_proj", "model.layers.58.mlp.experts.94.down_proj", "model.layers.58.mlp.experts.95.down_proj", "model.layers.58.mlp.experts.96.down_proj", "model.layers.58.mlp.experts.97.down_proj", "model.layers.58.mlp.experts.98.down_proj", "model.layers.58.mlp.experts.99.down_proj", "model.layers.58.mlp.experts.100.down_proj", "model.layers.58.mlp.experts.101.down_proj", "model.layers.58.mlp.experts.102.down_proj", "model.layers.58.mlp.experts.103.down_proj", "model.layers.58.mlp.experts.104.down_proj", "model.layers.58.mlp.experts.105.down_proj", "model.layers.58.mlp.experts.106.down_proj", "model.layers.58.mlp.experts.107.down_proj", "model.layers.58.mlp.experts.108.down_proj", "model.layers.58.mlp.experts.109.down_proj", "model.layers.58.mlp.experts.110.down_proj", "model.layers.58.mlp.experts.111.down_proj", "model.layers.58.mlp.experts.112.down_proj", "model.layers.58.mlp.experts.113.down_proj", "model.layers.58.mlp.experts.114.down_proj", "model.layers.58.mlp.experts.115.down_proj", "model.layers.58.mlp.experts.116.down_proj", "model.layers.58.mlp.experts.117.down_proj", "model.layers.58.mlp.experts.118.down_proj", "model.layers.58.mlp.experts.119.down_proj", "model.layers.58.mlp.experts.120.down_proj", "model.layers.58.mlp.experts.121.down_proj", "model.layers.58.mlp.experts.122.down_proj", "model.layers.58.mlp.experts.123.down_proj", "model.layers.58.mlp.experts.124.down_proj", "model.layers.58.mlp.experts.125.down_proj", "model.layers.58.mlp.experts.126.down_proj", "model.layers.58.mlp.experts.127.down_proj", "model.layers.58.mlp.experts.128.down_proj", "model.layers.58.mlp.experts.129.down_proj", "model.layers.58.mlp.experts.130.down_proj", "model.layers.58.mlp.experts.131.down_proj", "model.layers.58.mlp.experts.132.down_proj", "model.layers.58.mlp.experts.133.down_proj", "model.layers.58.mlp.experts.134.down_proj", "model.layers.58.mlp.experts.135.down_proj", "model.layers.58.mlp.experts.136.down_proj", "model.layers.58.mlp.experts.137.down_proj", "model.layers.58.mlp.experts.138.down_proj", "model.layers.58.mlp.experts.139.down_proj", "model.layers.58.mlp.experts.140.down_proj", "model.layers.58.mlp.experts.141.down_proj", "model.layers.58.mlp.experts.142.down_proj", "model.layers.58.mlp.experts.143.down_proj", "model.layers.58.mlp.experts.144.down_proj", "model.layers.58.mlp.experts.145.down_proj", "model.layers.58.mlp.experts.146.down_proj", "model.layers.58.mlp.experts.147.down_proj", "model.layers.58.mlp.experts.148.down_proj", "model.layers.58.mlp.experts.149.down_proj", "model.layers.58.mlp.experts.150.down_proj", "model.layers.58.mlp.experts.151.down_proj", "model.layers.58.mlp.experts.152.down_proj", "model.layers.58.mlp.experts.153.down_proj", "model.layers.58.mlp.experts.154.down_proj", "model.layers.58.mlp.experts.155.down_proj", "model.layers.58.mlp.experts.156.down_proj", "model.layers.58.mlp.experts.157.down_proj", "model.layers.58.mlp.experts.158.down_proj", "model.layers.58.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 6.712973117828369e-06, "dbits": 1258291200 } ] }, { "idx": 351, "layers": [ "model.layers.59.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0006814561784267426, "dbits": 62914560 } ] }, { "idx": 352, "layers": [ "model.layers.59.self_attn.k_proj", "model.layers.59.self_attn.v_proj" ], "candidates": [ { "dkld": -0.001195570081472419, "dbits": 10485760 } ] }, { "idx": 353, "layers": [ "model.layers.59.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00034132376313211754, "dbits": 62914560 } ] }, { "idx": 354, "layers": [ "model.layers.59.mlp.shared_experts.gate_proj", "model.layers.59.mlp.shared_experts.up_proj", "model.layers.59.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0007945224642753601, "dbits": 23592960 } ] }, { "idx": 355, "layers": [ "model.layers.59.mlp.experts.0.gate_proj", "model.layers.59.mlp.experts.1.gate_proj", "model.layers.59.mlp.experts.2.gate_proj", "model.layers.59.mlp.experts.3.gate_proj", "model.layers.59.mlp.experts.4.gate_proj", "model.layers.59.mlp.experts.5.gate_proj", "model.layers.59.mlp.experts.6.gate_proj", "model.layers.59.mlp.experts.7.gate_proj", "model.layers.59.mlp.experts.8.gate_proj", "model.layers.59.mlp.experts.9.gate_proj", "model.layers.59.mlp.experts.10.gate_proj", "model.layers.59.mlp.experts.11.gate_proj", "model.layers.59.mlp.experts.12.gate_proj", "model.layers.59.mlp.experts.13.gate_proj", "model.layers.59.mlp.experts.14.gate_proj", "model.layers.59.mlp.experts.15.gate_proj", "model.layers.59.mlp.experts.16.gate_proj", "model.layers.59.mlp.experts.17.gate_proj", "model.layers.59.mlp.experts.18.gate_proj", "model.layers.59.mlp.experts.19.gate_proj", "model.layers.59.mlp.experts.20.gate_proj", "model.layers.59.mlp.experts.21.gate_proj", "model.layers.59.mlp.experts.22.gate_proj", "model.layers.59.mlp.experts.23.gate_proj", "model.layers.59.mlp.experts.24.gate_proj", "model.layers.59.mlp.experts.25.gate_proj", "model.layers.59.mlp.experts.26.gate_proj", "model.layers.59.mlp.experts.27.gate_proj", "model.layers.59.mlp.experts.28.gate_proj", "model.layers.59.mlp.experts.29.gate_proj", "model.layers.59.mlp.experts.30.gate_proj", "model.layers.59.mlp.experts.31.gate_proj", "model.layers.59.mlp.experts.32.gate_proj", "model.layers.59.mlp.experts.33.gate_proj", "model.layers.59.mlp.experts.34.gate_proj", "model.layers.59.mlp.experts.35.gate_proj", "model.layers.59.mlp.experts.36.gate_proj", "model.layers.59.mlp.experts.37.gate_proj", "model.layers.59.mlp.experts.38.gate_proj", "model.layers.59.mlp.experts.39.gate_proj", "model.layers.59.mlp.experts.40.gate_proj", "model.layers.59.mlp.experts.41.gate_proj", "model.layers.59.mlp.experts.42.gate_proj", "model.layers.59.mlp.experts.43.gate_proj", "model.layers.59.mlp.experts.44.gate_proj", "model.layers.59.mlp.experts.45.gate_proj", "model.layers.59.mlp.experts.46.gate_proj", "model.layers.59.mlp.experts.47.gate_proj", "model.layers.59.mlp.experts.48.gate_proj", "model.layers.59.mlp.experts.49.gate_proj", "model.layers.59.mlp.experts.50.gate_proj", "model.layers.59.mlp.experts.51.gate_proj", "model.layers.59.mlp.experts.52.gate_proj", "model.layers.59.mlp.experts.53.gate_proj", "model.layers.59.mlp.experts.54.gate_proj", "model.layers.59.mlp.experts.55.gate_proj", "model.layers.59.mlp.experts.56.gate_proj", "model.layers.59.mlp.experts.57.gate_proj", "model.layers.59.mlp.experts.58.gate_proj", "model.layers.59.mlp.experts.59.gate_proj", "model.layers.59.mlp.experts.60.gate_proj", "model.layers.59.mlp.experts.61.gate_proj", "model.layers.59.mlp.experts.62.gate_proj", "model.layers.59.mlp.experts.63.gate_proj", "model.layers.59.mlp.experts.64.gate_proj", "model.layers.59.mlp.experts.65.gate_proj", "model.layers.59.mlp.experts.66.gate_proj", "model.layers.59.mlp.experts.67.gate_proj", "model.layers.59.mlp.experts.68.gate_proj", "model.layers.59.mlp.experts.69.gate_proj", "model.layers.59.mlp.experts.70.gate_proj", "model.layers.59.mlp.experts.71.gate_proj", "model.layers.59.mlp.experts.72.gate_proj", "model.layers.59.mlp.experts.73.gate_proj", "model.layers.59.mlp.experts.74.gate_proj", "model.layers.59.mlp.experts.75.gate_proj", "model.layers.59.mlp.experts.76.gate_proj", "model.layers.59.mlp.experts.77.gate_proj", "model.layers.59.mlp.experts.78.gate_proj", "model.layers.59.mlp.experts.79.gate_proj", "model.layers.59.mlp.experts.80.gate_proj", "model.layers.59.mlp.experts.81.gate_proj", "model.layers.59.mlp.experts.82.gate_proj", "model.layers.59.mlp.experts.83.gate_proj", "model.layers.59.mlp.experts.84.gate_proj", "model.layers.59.mlp.experts.85.gate_proj", "model.layers.59.mlp.experts.86.gate_proj", "model.layers.59.mlp.experts.87.gate_proj", "model.layers.59.mlp.experts.88.gate_proj", "model.layers.59.mlp.experts.89.gate_proj", "model.layers.59.mlp.experts.90.gate_proj", "model.layers.59.mlp.experts.91.gate_proj", "model.layers.59.mlp.experts.92.gate_proj", "model.layers.59.mlp.experts.93.gate_proj", "model.layers.59.mlp.experts.94.gate_proj", "model.layers.59.mlp.experts.95.gate_proj", "model.layers.59.mlp.experts.96.gate_proj", "model.layers.59.mlp.experts.97.gate_proj", "model.layers.59.mlp.experts.98.gate_proj", "model.layers.59.mlp.experts.99.gate_proj", "model.layers.59.mlp.experts.100.gate_proj", "model.layers.59.mlp.experts.101.gate_proj", "model.layers.59.mlp.experts.102.gate_proj", "model.layers.59.mlp.experts.103.gate_proj", "model.layers.59.mlp.experts.104.gate_proj", "model.layers.59.mlp.experts.105.gate_proj", "model.layers.59.mlp.experts.106.gate_proj", "model.layers.59.mlp.experts.107.gate_proj", "model.layers.59.mlp.experts.108.gate_proj", "model.layers.59.mlp.experts.109.gate_proj", "model.layers.59.mlp.experts.110.gate_proj", "model.layers.59.mlp.experts.111.gate_proj", "model.layers.59.mlp.experts.112.gate_proj", "model.layers.59.mlp.experts.113.gate_proj", "model.layers.59.mlp.experts.114.gate_proj", "model.layers.59.mlp.experts.115.gate_proj", "model.layers.59.mlp.experts.116.gate_proj", "model.layers.59.mlp.experts.117.gate_proj", "model.layers.59.mlp.experts.118.gate_proj", "model.layers.59.mlp.experts.119.gate_proj", "model.layers.59.mlp.experts.120.gate_proj", "model.layers.59.mlp.experts.121.gate_proj", "model.layers.59.mlp.experts.122.gate_proj", "model.layers.59.mlp.experts.123.gate_proj", "model.layers.59.mlp.experts.124.gate_proj", "model.layers.59.mlp.experts.125.gate_proj", "model.layers.59.mlp.experts.126.gate_proj", "model.layers.59.mlp.experts.127.gate_proj", "model.layers.59.mlp.experts.128.gate_proj", "model.layers.59.mlp.experts.129.gate_proj", "model.layers.59.mlp.experts.130.gate_proj", "model.layers.59.mlp.experts.131.gate_proj", "model.layers.59.mlp.experts.132.gate_proj", "model.layers.59.mlp.experts.133.gate_proj", "model.layers.59.mlp.experts.134.gate_proj", "model.layers.59.mlp.experts.135.gate_proj", "model.layers.59.mlp.experts.136.gate_proj", "model.layers.59.mlp.experts.137.gate_proj", "model.layers.59.mlp.experts.138.gate_proj", "model.layers.59.mlp.experts.139.gate_proj", "model.layers.59.mlp.experts.140.gate_proj", "model.layers.59.mlp.experts.141.gate_proj", "model.layers.59.mlp.experts.142.gate_proj", "model.layers.59.mlp.experts.143.gate_proj", "model.layers.59.mlp.experts.144.gate_proj", "model.layers.59.mlp.experts.145.gate_proj", "model.layers.59.mlp.experts.146.gate_proj", "model.layers.59.mlp.experts.147.gate_proj", "model.layers.59.mlp.experts.148.gate_proj", "model.layers.59.mlp.experts.149.gate_proj", "model.layers.59.mlp.experts.150.gate_proj", "model.layers.59.mlp.experts.151.gate_proj", "model.layers.59.mlp.experts.152.gate_proj", "model.layers.59.mlp.experts.153.gate_proj", "model.layers.59.mlp.experts.154.gate_proj", "model.layers.59.mlp.experts.155.gate_proj", "model.layers.59.mlp.experts.156.gate_proj", "model.layers.59.mlp.experts.157.gate_proj", "model.layers.59.mlp.experts.158.gate_proj", "model.layers.59.mlp.experts.159.gate_proj", "model.layers.59.mlp.experts.0.up_proj", "model.layers.59.mlp.experts.1.up_proj", "model.layers.59.mlp.experts.2.up_proj", "model.layers.59.mlp.experts.3.up_proj", "model.layers.59.mlp.experts.4.up_proj", "model.layers.59.mlp.experts.5.up_proj", "model.layers.59.mlp.experts.6.up_proj", "model.layers.59.mlp.experts.7.up_proj", "model.layers.59.mlp.experts.8.up_proj", "model.layers.59.mlp.experts.9.up_proj", "model.layers.59.mlp.experts.10.up_proj", "model.layers.59.mlp.experts.11.up_proj", "model.layers.59.mlp.experts.12.up_proj", "model.layers.59.mlp.experts.13.up_proj", "model.layers.59.mlp.experts.14.up_proj", "model.layers.59.mlp.experts.15.up_proj", "model.layers.59.mlp.experts.16.up_proj", "model.layers.59.mlp.experts.17.up_proj", "model.layers.59.mlp.experts.18.up_proj", "model.layers.59.mlp.experts.19.up_proj", "model.layers.59.mlp.experts.20.up_proj", "model.layers.59.mlp.experts.21.up_proj", "model.layers.59.mlp.experts.22.up_proj", "model.layers.59.mlp.experts.23.up_proj", "model.layers.59.mlp.experts.24.up_proj", "model.layers.59.mlp.experts.25.up_proj", "model.layers.59.mlp.experts.26.up_proj", "model.layers.59.mlp.experts.27.up_proj", "model.layers.59.mlp.experts.28.up_proj", "model.layers.59.mlp.experts.29.up_proj", "model.layers.59.mlp.experts.30.up_proj", "model.layers.59.mlp.experts.31.up_proj", "model.layers.59.mlp.experts.32.up_proj", "model.layers.59.mlp.experts.33.up_proj", "model.layers.59.mlp.experts.34.up_proj", "model.layers.59.mlp.experts.35.up_proj", "model.layers.59.mlp.experts.36.up_proj", "model.layers.59.mlp.experts.37.up_proj", "model.layers.59.mlp.experts.38.up_proj", "model.layers.59.mlp.experts.39.up_proj", "model.layers.59.mlp.experts.40.up_proj", "model.layers.59.mlp.experts.41.up_proj", "model.layers.59.mlp.experts.42.up_proj", "model.layers.59.mlp.experts.43.up_proj", "model.layers.59.mlp.experts.44.up_proj", "model.layers.59.mlp.experts.45.up_proj", "model.layers.59.mlp.experts.46.up_proj", "model.layers.59.mlp.experts.47.up_proj", "model.layers.59.mlp.experts.48.up_proj", "model.layers.59.mlp.experts.49.up_proj", "model.layers.59.mlp.experts.50.up_proj", "model.layers.59.mlp.experts.51.up_proj", "model.layers.59.mlp.experts.52.up_proj", "model.layers.59.mlp.experts.53.up_proj", "model.layers.59.mlp.experts.54.up_proj", "model.layers.59.mlp.experts.55.up_proj", "model.layers.59.mlp.experts.56.up_proj", "model.layers.59.mlp.experts.57.up_proj", "model.layers.59.mlp.experts.58.up_proj", "model.layers.59.mlp.experts.59.up_proj", "model.layers.59.mlp.experts.60.up_proj", "model.layers.59.mlp.experts.61.up_proj", "model.layers.59.mlp.experts.62.up_proj", "model.layers.59.mlp.experts.63.up_proj", "model.layers.59.mlp.experts.64.up_proj", "model.layers.59.mlp.experts.65.up_proj", "model.layers.59.mlp.experts.66.up_proj", "model.layers.59.mlp.experts.67.up_proj", "model.layers.59.mlp.experts.68.up_proj", "model.layers.59.mlp.experts.69.up_proj", "model.layers.59.mlp.experts.70.up_proj", "model.layers.59.mlp.experts.71.up_proj", "model.layers.59.mlp.experts.72.up_proj", "model.layers.59.mlp.experts.73.up_proj", "model.layers.59.mlp.experts.74.up_proj", "model.layers.59.mlp.experts.75.up_proj", "model.layers.59.mlp.experts.76.up_proj", "model.layers.59.mlp.experts.77.up_proj", "model.layers.59.mlp.experts.78.up_proj", "model.layers.59.mlp.experts.79.up_proj", "model.layers.59.mlp.experts.80.up_proj", "model.layers.59.mlp.experts.81.up_proj", "model.layers.59.mlp.experts.82.up_proj", "model.layers.59.mlp.experts.83.up_proj", "model.layers.59.mlp.experts.84.up_proj", "model.layers.59.mlp.experts.85.up_proj", "model.layers.59.mlp.experts.86.up_proj", "model.layers.59.mlp.experts.87.up_proj", "model.layers.59.mlp.experts.88.up_proj", "model.layers.59.mlp.experts.89.up_proj", "model.layers.59.mlp.experts.90.up_proj", "model.layers.59.mlp.experts.91.up_proj", "model.layers.59.mlp.experts.92.up_proj", "model.layers.59.mlp.experts.93.up_proj", "model.layers.59.mlp.experts.94.up_proj", "model.layers.59.mlp.experts.95.up_proj", "model.layers.59.mlp.experts.96.up_proj", "model.layers.59.mlp.experts.97.up_proj", "model.layers.59.mlp.experts.98.up_proj", "model.layers.59.mlp.experts.99.up_proj", "model.layers.59.mlp.experts.100.up_proj", "model.layers.59.mlp.experts.101.up_proj", "model.layers.59.mlp.experts.102.up_proj", "model.layers.59.mlp.experts.103.up_proj", "model.layers.59.mlp.experts.104.up_proj", "model.layers.59.mlp.experts.105.up_proj", "model.layers.59.mlp.experts.106.up_proj", "model.layers.59.mlp.experts.107.up_proj", "model.layers.59.mlp.experts.108.up_proj", "model.layers.59.mlp.experts.109.up_proj", "model.layers.59.mlp.experts.110.up_proj", "model.layers.59.mlp.experts.111.up_proj", "model.layers.59.mlp.experts.112.up_proj", "model.layers.59.mlp.experts.113.up_proj", "model.layers.59.mlp.experts.114.up_proj", "model.layers.59.mlp.experts.115.up_proj", "model.layers.59.mlp.experts.116.up_proj", "model.layers.59.mlp.experts.117.up_proj", "model.layers.59.mlp.experts.118.up_proj", "model.layers.59.mlp.experts.119.up_proj", "model.layers.59.mlp.experts.120.up_proj", "model.layers.59.mlp.experts.121.up_proj", "model.layers.59.mlp.experts.122.up_proj", "model.layers.59.mlp.experts.123.up_proj", "model.layers.59.mlp.experts.124.up_proj", "model.layers.59.mlp.experts.125.up_proj", "model.layers.59.mlp.experts.126.up_proj", "model.layers.59.mlp.experts.127.up_proj", "model.layers.59.mlp.experts.128.up_proj", "model.layers.59.mlp.experts.129.up_proj", "model.layers.59.mlp.experts.130.up_proj", "model.layers.59.mlp.experts.131.up_proj", "model.layers.59.mlp.experts.132.up_proj", "model.layers.59.mlp.experts.133.up_proj", "model.layers.59.mlp.experts.134.up_proj", "model.layers.59.mlp.experts.135.up_proj", "model.layers.59.mlp.experts.136.up_proj", "model.layers.59.mlp.experts.137.up_proj", "model.layers.59.mlp.experts.138.up_proj", "model.layers.59.mlp.experts.139.up_proj", "model.layers.59.mlp.experts.140.up_proj", "model.layers.59.mlp.experts.141.up_proj", "model.layers.59.mlp.experts.142.up_proj", "model.layers.59.mlp.experts.143.up_proj", "model.layers.59.mlp.experts.144.up_proj", "model.layers.59.mlp.experts.145.up_proj", "model.layers.59.mlp.experts.146.up_proj", "model.layers.59.mlp.experts.147.up_proj", "model.layers.59.mlp.experts.148.up_proj", "model.layers.59.mlp.experts.149.up_proj", "model.layers.59.mlp.experts.150.up_proj", "model.layers.59.mlp.experts.151.up_proj", "model.layers.59.mlp.experts.152.up_proj", "model.layers.59.mlp.experts.153.up_proj", "model.layers.59.mlp.experts.154.up_proj", "model.layers.59.mlp.experts.155.up_proj", "model.layers.59.mlp.experts.156.up_proj", "model.layers.59.mlp.experts.157.up_proj", "model.layers.59.mlp.experts.158.up_proj", "model.layers.59.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00045992955565454796, "dbits": 2516582400 } ] }, { "idx": 356, "layers": [ "model.layers.59.mlp.experts.0.down_proj", "model.layers.59.mlp.experts.1.down_proj", "model.layers.59.mlp.experts.2.down_proj", "model.layers.59.mlp.experts.3.down_proj", "model.layers.59.mlp.experts.4.down_proj", "model.layers.59.mlp.experts.5.down_proj", "model.layers.59.mlp.experts.6.down_proj", "model.layers.59.mlp.experts.7.down_proj", "model.layers.59.mlp.experts.8.down_proj", "model.layers.59.mlp.experts.9.down_proj", "model.layers.59.mlp.experts.10.down_proj", "model.layers.59.mlp.experts.11.down_proj", "model.layers.59.mlp.experts.12.down_proj", "model.layers.59.mlp.experts.13.down_proj", "model.layers.59.mlp.experts.14.down_proj", "model.layers.59.mlp.experts.15.down_proj", "model.layers.59.mlp.experts.16.down_proj", "model.layers.59.mlp.experts.17.down_proj", "model.layers.59.mlp.experts.18.down_proj", "model.layers.59.mlp.experts.19.down_proj", "model.layers.59.mlp.experts.20.down_proj", "model.layers.59.mlp.experts.21.down_proj", "model.layers.59.mlp.experts.22.down_proj", "model.layers.59.mlp.experts.23.down_proj", "model.layers.59.mlp.experts.24.down_proj", "model.layers.59.mlp.experts.25.down_proj", "model.layers.59.mlp.experts.26.down_proj", "model.layers.59.mlp.experts.27.down_proj", "model.layers.59.mlp.experts.28.down_proj", "model.layers.59.mlp.experts.29.down_proj", "model.layers.59.mlp.experts.30.down_proj", "model.layers.59.mlp.experts.31.down_proj", "model.layers.59.mlp.experts.32.down_proj", "model.layers.59.mlp.experts.33.down_proj", "model.layers.59.mlp.experts.34.down_proj", "model.layers.59.mlp.experts.35.down_proj", "model.layers.59.mlp.experts.36.down_proj", "model.layers.59.mlp.experts.37.down_proj", "model.layers.59.mlp.experts.38.down_proj", "model.layers.59.mlp.experts.39.down_proj", "model.layers.59.mlp.experts.40.down_proj", "model.layers.59.mlp.experts.41.down_proj", "model.layers.59.mlp.experts.42.down_proj", "model.layers.59.mlp.experts.43.down_proj", "model.layers.59.mlp.experts.44.down_proj", "model.layers.59.mlp.experts.45.down_proj", "model.layers.59.mlp.experts.46.down_proj", "model.layers.59.mlp.experts.47.down_proj", "model.layers.59.mlp.experts.48.down_proj", "model.layers.59.mlp.experts.49.down_proj", "model.layers.59.mlp.experts.50.down_proj", "model.layers.59.mlp.experts.51.down_proj", "model.layers.59.mlp.experts.52.down_proj", "model.layers.59.mlp.experts.53.down_proj", "model.layers.59.mlp.experts.54.down_proj", "model.layers.59.mlp.experts.55.down_proj", "model.layers.59.mlp.experts.56.down_proj", "model.layers.59.mlp.experts.57.down_proj", "model.layers.59.mlp.experts.58.down_proj", "model.layers.59.mlp.experts.59.down_proj", "model.layers.59.mlp.experts.60.down_proj", "model.layers.59.mlp.experts.61.down_proj", "model.layers.59.mlp.experts.62.down_proj", "model.layers.59.mlp.experts.63.down_proj", "model.layers.59.mlp.experts.64.down_proj", "model.layers.59.mlp.experts.65.down_proj", "model.layers.59.mlp.experts.66.down_proj", "model.layers.59.mlp.experts.67.down_proj", "model.layers.59.mlp.experts.68.down_proj", "model.layers.59.mlp.experts.69.down_proj", "model.layers.59.mlp.experts.70.down_proj", "model.layers.59.mlp.experts.71.down_proj", "model.layers.59.mlp.experts.72.down_proj", "model.layers.59.mlp.experts.73.down_proj", "model.layers.59.mlp.experts.74.down_proj", "model.layers.59.mlp.experts.75.down_proj", "model.layers.59.mlp.experts.76.down_proj", "model.layers.59.mlp.experts.77.down_proj", "model.layers.59.mlp.experts.78.down_proj", "model.layers.59.mlp.experts.79.down_proj", "model.layers.59.mlp.experts.80.down_proj", "model.layers.59.mlp.experts.81.down_proj", "model.layers.59.mlp.experts.82.down_proj", "model.layers.59.mlp.experts.83.down_proj", "model.layers.59.mlp.experts.84.down_proj", "model.layers.59.mlp.experts.85.down_proj", "model.layers.59.mlp.experts.86.down_proj", "model.layers.59.mlp.experts.87.down_proj", "model.layers.59.mlp.experts.88.down_proj", "model.layers.59.mlp.experts.89.down_proj", "model.layers.59.mlp.experts.90.down_proj", "model.layers.59.mlp.experts.91.down_proj", "model.layers.59.mlp.experts.92.down_proj", "model.layers.59.mlp.experts.93.down_proj", "model.layers.59.mlp.experts.94.down_proj", "model.layers.59.mlp.experts.95.down_proj", "model.layers.59.mlp.experts.96.down_proj", "model.layers.59.mlp.experts.97.down_proj", "model.layers.59.mlp.experts.98.down_proj", "model.layers.59.mlp.experts.99.down_proj", "model.layers.59.mlp.experts.100.down_proj", "model.layers.59.mlp.experts.101.down_proj", "model.layers.59.mlp.experts.102.down_proj", "model.layers.59.mlp.experts.103.down_proj", "model.layers.59.mlp.experts.104.down_proj", "model.layers.59.mlp.experts.105.down_proj", "model.layers.59.mlp.experts.106.down_proj", "model.layers.59.mlp.experts.107.down_proj", "model.layers.59.mlp.experts.108.down_proj", "model.layers.59.mlp.experts.109.down_proj", "model.layers.59.mlp.experts.110.down_proj", "model.layers.59.mlp.experts.111.down_proj", "model.layers.59.mlp.experts.112.down_proj", "model.layers.59.mlp.experts.113.down_proj", "model.layers.59.mlp.experts.114.down_proj", "model.layers.59.mlp.experts.115.down_proj", "model.layers.59.mlp.experts.116.down_proj", "model.layers.59.mlp.experts.117.down_proj", "model.layers.59.mlp.experts.118.down_proj", "model.layers.59.mlp.experts.119.down_proj", "model.layers.59.mlp.experts.120.down_proj", "model.layers.59.mlp.experts.121.down_proj", "model.layers.59.mlp.experts.122.down_proj", "model.layers.59.mlp.experts.123.down_proj", "model.layers.59.mlp.experts.124.down_proj", "model.layers.59.mlp.experts.125.down_proj", "model.layers.59.mlp.experts.126.down_proj", "model.layers.59.mlp.experts.127.down_proj", "model.layers.59.mlp.experts.128.down_proj", "model.layers.59.mlp.experts.129.down_proj", "model.layers.59.mlp.experts.130.down_proj", "model.layers.59.mlp.experts.131.down_proj", "model.layers.59.mlp.experts.132.down_proj", "model.layers.59.mlp.experts.133.down_proj", "model.layers.59.mlp.experts.134.down_proj", "model.layers.59.mlp.experts.135.down_proj", "model.layers.59.mlp.experts.136.down_proj", "model.layers.59.mlp.experts.137.down_proj", "model.layers.59.mlp.experts.138.down_proj", "model.layers.59.mlp.experts.139.down_proj", "model.layers.59.mlp.experts.140.down_proj", "model.layers.59.mlp.experts.141.down_proj", "model.layers.59.mlp.experts.142.down_proj", "model.layers.59.mlp.experts.143.down_proj", "model.layers.59.mlp.experts.144.down_proj", "model.layers.59.mlp.experts.145.down_proj", "model.layers.59.mlp.experts.146.down_proj", "model.layers.59.mlp.experts.147.down_proj", "model.layers.59.mlp.experts.148.down_proj", "model.layers.59.mlp.experts.149.down_proj", "model.layers.59.mlp.experts.150.down_proj", "model.layers.59.mlp.experts.151.down_proj", "model.layers.59.mlp.experts.152.down_proj", "model.layers.59.mlp.experts.153.down_proj", "model.layers.59.mlp.experts.154.down_proj", "model.layers.59.mlp.experts.155.down_proj", "model.layers.59.mlp.experts.156.down_proj", "model.layers.59.mlp.experts.157.down_proj", "model.layers.59.mlp.experts.158.down_proj", "model.layers.59.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0005811765789985879, "dbits": 1258291200 } ] }, { "idx": 357, "layers": [ "model.layers.60.self_attn.q_proj" ], "candidates": [ { "dkld": -1.7374753952248412e-06, "dbits": 62914560 } ] }, { "idx": 358, "layers": [ "model.layers.60.self_attn.k_proj", "model.layers.60.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0008133895695209836, "dbits": 10485760 } ] }, { "idx": 359, "layers": [ "model.layers.60.self_attn.o_proj" ], "candidates": [ { "dkld": 4.271417856216431e-05, "dbits": 62914560 } ] }, { "idx": 360, "layers": [ "model.layers.60.mlp.shared_experts.gate_proj", "model.layers.60.mlp.shared_experts.up_proj", "model.layers.60.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00041336193680763245, "dbits": 23592960 } ] }, { "idx": 361, "layers": [ "model.layers.60.mlp.experts.0.gate_proj", "model.layers.60.mlp.experts.1.gate_proj", "model.layers.60.mlp.experts.2.gate_proj", "model.layers.60.mlp.experts.3.gate_proj", "model.layers.60.mlp.experts.4.gate_proj", "model.layers.60.mlp.experts.5.gate_proj", "model.layers.60.mlp.experts.6.gate_proj", "model.layers.60.mlp.experts.7.gate_proj", "model.layers.60.mlp.experts.8.gate_proj", "model.layers.60.mlp.experts.9.gate_proj", "model.layers.60.mlp.experts.10.gate_proj", "model.layers.60.mlp.experts.11.gate_proj", "model.layers.60.mlp.experts.12.gate_proj", "model.layers.60.mlp.experts.13.gate_proj", "model.layers.60.mlp.experts.14.gate_proj", "model.layers.60.mlp.experts.15.gate_proj", "model.layers.60.mlp.experts.16.gate_proj", "model.layers.60.mlp.experts.17.gate_proj", "model.layers.60.mlp.experts.18.gate_proj", "model.layers.60.mlp.experts.19.gate_proj", "model.layers.60.mlp.experts.20.gate_proj", "model.layers.60.mlp.experts.21.gate_proj", "model.layers.60.mlp.experts.22.gate_proj", "model.layers.60.mlp.experts.23.gate_proj", "model.layers.60.mlp.experts.24.gate_proj", "model.layers.60.mlp.experts.25.gate_proj", "model.layers.60.mlp.experts.26.gate_proj", "model.layers.60.mlp.experts.27.gate_proj", "model.layers.60.mlp.experts.28.gate_proj", "model.layers.60.mlp.experts.29.gate_proj", "model.layers.60.mlp.experts.30.gate_proj", "model.layers.60.mlp.experts.31.gate_proj", "model.layers.60.mlp.experts.32.gate_proj", "model.layers.60.mlp.experts.33.gate_proj", "model.layers.60.mlp.experts.34.gate_proj", "model.layers.60.mlp.experts.35.gate_proj", "model.layers.60.mlp.experts.36.gate_proj", "model.layers.60.mlp.experts.37.gate_proj", "model.layers.60.mlp.experts.38.gate_proj", "model.layers.60.mlp.experts.39.gate_proj", "model.layers.60.mlp.experts.40.gate_proj", "model.layers.60.mlp.experts.41.gate_proj", "model.layers.60.mlp.experts.42.gate_proj", "model.layers.60.mlp.experts.43.gate_proj", "model.layers.60.mlp.experts.44.gate_proj", "model.layers.60.mlp.experts.45.gate_proj", "model.layers.60.mlp.experts.46.gate_proj", "model.layers.60.mlp.experts.47.gate_proj", "model.layers.60.mlp.experts.48.gate_proj", "model.layers.60.mlp.experts.49.gate_proj", "model.layers.60.mlp.experts.50.gate_proj", "model.layers.60.mlp.experts.51.gate_proj", "model.layers.60.mlp.experts.52.gate_proj", "model.layers.60.mlp.experts.53.gate_proj", "model.layers.60.mlp.experts.54.gate_proj", "model.layers.60.mlp.experts.55.gate_proj", "model.layers.60.mlp.experts.56.gate_proj", "model.layers.60.mlp.experts.57.gate_proj", "model.layers.60.mlp.experts.58.gate_proj", "model.layers.60.mlp.experts.59.gate_proj", "model.layers.60.mlp.experts.60.gate_proj", "model.layers.60.mlp.experts.61.gate_proj", "model.layers.60.mlp.experts.62.gate_proj", "model.layers.60.mlp.experts.63.gate_proj", "model.layers.60.mlp.experts.64.gate_proj", "model.layers.60.mlp.experts.65.gate_proj", "model.layers.60.mlp.experts.66.gate_proj", "model.layers.60.mlp.experts.67.gate_proj", "model.layers.60.mlp.experts.68.gate_proj", "model.layers.60.mlp.experts.69.gate_proj", "model.layers.60.mlp.experts.70.gate_proj", "model.layers.60.mlp.experts.71.gate_proj", "model.layers.60.mlp.experts.72.gate_proj", "model.layers.60.mlp.experts.73.gate_proj", "model.layers.60.mlp.experts.74.gate_proj", "model.layers.60.mlp.experts.75.gate_proj", "model.layers.60.mlp.experts.76.gate_proj", "model.layers.60.mlp.experts.77.gate_proj", "model.layers.60.mlp.experts.78.gate_proj", "model.layers.60.mlp.experts.79.gate_proj", "model.layers.60.mlp.experts.80.gate_proj", "model.layers.60.mlp.experts.81.gate_proj", "model.layers.60.mlp.experts.82.gate_proj", "model.layers.60.mlp.experts.83.gate_proj", "model.layers.60.mlp.experts.84.gate_proj", "model.layers.60.mlp.experts.85.gate_proj", "model.layers.60.mlp.experts.86.gate_proj", "model.layers.60.mlp.experts.87.gate_proj", "model.layers.60.mlp.experts.88.gate_proj", "model.layers.60.mlp.experts.89.gate_proj", "model.layers.60.mlp.experts.90.gate_proj", "model.layers.60.mlp.experts.91.gate_proj", "model.layers.60.mlp.experts.92.gate_proj", "model.layers.60.mlp.experts.93.gate_proj", "model.layers.60.mlp.experts.94.gate_proj", "model.layers.60.mlp.experts.95.gate_proj", "model.layers.60.mlp.experts.96.gate_proj", "model.layers.60.mlp.experts.97.gate_proj", "model.layers.60.mlp.experts.98.gate_proj", "model.layers.60.mlp.experts.99.gate_proj", "model.layers.60.mlp.experts.100.gate_proj", "model.layers.60.mlp.experts.101.gate_proj", "model.layers.60.mlp.experts.102.gate_proj", "model.layers.60.mlp.experts.103.gate_proj", "model.layers.60.mlp.experts.104.gate_proj", "model.layers.60.mlp.experts.105.gate_proj", "model.layers.60.mlp.experts.106.gate_proj", "model.layers.60.mlp.experts.107.gate_proj", "model.layers.60.mlp.experts.108.gate_proj", "model.layers.60.mlp.experts.109.gate_proj", "model.layers.60.mlp.experts.110.gate_proj", "model.layers.60.mlp.experts.111.gate_proj", "model.layers.60.mlp.experts.112.gate_proj", "model.layers.60.mlp.experts.113.gate_proj", "model.layers.60.mlp.experts.114.gate_proj", "model.layers.60.mlp.experts.115.gate_proj", "model.layers.60.mlp.experts.116.gate_proj", "model.layers.60.mlp.experts.117.gate_proj", "model.layers.60.mlp.experts.118.gate_proj", "model.layers.60.mlp.experts.119.gate_proj", "model.layers.60.mlp.experts.120.gate_proj", "model.layers.60.mlp.experts.121.gate_proj", "model.layers.60.mlp.experts.122.gate_proj", "model.layers.60.mlp.experts.123.gate_proj", "model.layers.60.mlp.experts.124.gate_proj", "model.layers.60.mlp.experts.125.gate_proj", "model.layers.60.mlp.experts.126.gate_proj", "model.layers.60.mlp.experts.127.gate_proj", "model.layers.60.mlp.experts.128.gate_proj", "model.layers.60.mlp.experts.129.gate_proj", "model.layers.60.mlp.experts.130.gate_proj", "model.layers.60.mlp.experts.131.gate_proj", "model.layers.60.mlp.experts.132.gate_proj", "model.layers.60.mlp.experts.133.gate_proj", "model.layers.60.mlp.experts.134.gate_proj", "model.layers.60.mlp.experts.135.gate_proj", "model.layers.60.mlp.experts.136.gate_proj", "model.layers.60.mlp.experts.137.gate_proj", "model.layers.60.mlp.experts.138.gate_proj", "model.layers.60.mlp.experts.139.gate_proj", "model.layers.60.mlp.experts.140.gate_proj", "model.layers.60.mlp.experts.141.gate_proj", "model.layers.60.mlp.experts.142.gate_proj", "model.layers.60.mlp.experts.143.gate_proj", "model.layers.60.mlp.experts.144.gate_proj", "model.layers.60.mlp.experts.145.gate_proj", "model.layers.60.mlp.experts.146.gate_proj", "model.layers.60.mlp.experts.147.gate_proj", "model.layers.60.mlp.experts.148.gate_proj", "model.layers.60.mlp.experts.149.gate_proj", "model.layers.60.mlp.experts.150.gate_proj", "model.layers.60.mlp.experts.151.gate_proj", "model.layers.60.mlp.experts.152.gate_proj", "model.layers.60.mlp.experts.153.gate_proj", "model.layers.60.mlp.experts.154.gate_proj", "model.layers.60.mlp.experts.155.gate_proj", "model.layers.60.mlp.experts.156.gate_proj", "model.layers.60.mlp.experts.157.gate_proj", "model.layers.60.mlp.experts.158.gate_proj", "model.layers.60.mlp.experts.159.gate_proj", "model.layers.60.mlp.experts.0.up_proj", "model.layers.60.mlp.experts.1.up_proj", "model.layers.60.mlp.experts.2.up_proj", "model.layers.60.mlp.experts.3.up_proj", "model.layers.60.mlp.experts.4.up_proj", "model.layers.60.mlp.experts.5.up_proj", "model.layers.60.mlp.experts.6.up_proj", "model.layers.60.mlp.experts.7.up_proj", "model.layers.60.mlp.experts.8.up_proj", "model.layers.60.mlp.experts.9.up_proj", "model.layers.60.mlp.experts.10.up_proj", "model.layers.60.mlp.experts.11.up_proj", "model.layers.60.mlp.experts.12.up_proj", "model.layers.60.mlp.experts.13.up_proj", "model.layers.60.mlp.experts.14.up_proj", "model.layers.60.mlp.experts.15.up_proj", "model.layers.60.mlp.experts.16.up_proj", "model.layers.60.mlp.experts.17.up_proj", "model.layers.60.mlp.experts.18.up_proj", "model.layers.60.mlp.experts.19.up_proj", "model.layers.60.mlp.experts.20.up_proj", "model.layers.60.mlp.experts.21.up_proj", "model.layers.60.mlp.experts.22.up_proj", "model.layers.60.mlp.experts.23.up_proj", "model.layers.60.mlp.experts.24.up_proj", "model.layers.60.mlp.experts.25.up_proj", "model.layers.60.mlp.experts.26.up_proj", "model.layers.60.mlp.experts.27.up_proj", "model.layers.60.mlp.experts.28.up_proj", "model.layers.60.mlp.experts.29.up_proj", "model.layers.60.mlp.experts.30.up_proj", "model.layers.60.mlp.experts.31.up_proj", "model.layers.60.mlp.experts.32.up_proj", "model.layers.60.mlp.experts.33.up_proj", "model.layers.60.mlp.experts.34.up_proj", "model.layers.60.mlp.experts.35.up_proj", "model.layers.60.mlp.experts.36.up_proj", "model.layers.60.mlp.experts.37.up_proj", "model.layers.60.mlp.experts.38.up_proj", "model.layers.60.mlp.experts.39.up_proj", "model.layers.60.mlp.experts.40.up_proj", "model.layers.60.mlp.experts.41.up_proj", "model.layers.60.mlp.experts.42.up_proj", "model.layers.60.mlp.experts.43.up_proj", "model.layers.60.mlp.experts.44.up_proj", "model.layers.60.mlp.experts.45.up_proj", "model.layers.60.mlp.experts.46.up_proj", "model.layers.60.mlp.experts.47.up_proj", "model.layers.60.mlp.experts.48.up_proj", "model.layers.60.mlp.experts.49.up_proj", "model.layers.60.mlp.experts.50.up_proj", "model.layers.60.mlp.experts.51.up_proj", "model.layers.60.mlp.experts.52.up_proj", "model.layers.60.mlp.experts.53.up_proj", "model.layers.60.mlp.experts.54.up_proj", "model.layers.60.mlp.experts.55.up_proj", "model.layers.60.mlp.experts.56.up_proj", "model.layers.60.mlp.experts.57.up_proj", "model.layers.60.mlp.experts.58.up_proj", "model.layers.60.mlp.experts.59.up_proj", "model.layers.60.mlp.experts.60.up_proj", "model.layers.60.mlp.experts.61.up_proj", "model.layers.60.mlp.experts.62.up_proj", "model.layers.60.mlp.experts.63.up_proj", "model.layers.60.mlp.experts.64.up_proj", "model.layers.60.mlp.experts.65.up_proj", "model.layers.60.mlp.experts.66.up_proj", "model.layers.60.mlp.experts.67.up_proj", "model.layers.60.mlp.experts.68.up_proj", "model.layers.60.mlp.experts.69.up_proj", "model.layers.60.mlp.experts.70.up_proj", "model.layers.60.mlp.experts.71.up_proj", "model.layers.60.mlp.experts.72.up_proj", "model.layers.60.mlp.experts.73.up_proj", "model.layers.60.mlp.experts.74.up_proj", "model.layers.60.mlp.experts.75.up_proj", "model.layers.60.mlp.experts.76.up_proj", "model.layers.60.mlp.experts.77.up_proj", "model.layers.60.mlp.experts.78.up_proj", "model.layers.60.mlp.experts.79.up_proj", "model.layers.60.mlp.experts.80.up_proj", "model.layers.60.mlp.experts.81.up_proj", "model.layers.60.mlp.experts.82.up_proj", "model.layers.60.mlp.experts.83.up_proj", "model.layers.60.mlp.experts.84.up_proj", "model.layers.60.mlp.experts.85.up_proj", "model.layers.60.mlp.experts.86.up_proj", "model.layers.60.mlp.experts.87.up_proj", "model.layers.60.mlp.experts.88.up_proj", "model.layers.60.mlp.experts.89.up_proj", "model.layers.60.mlp.experts.90.up_proj", "model.layers.60.mlp.experts.91.up_proj", "model.layers.60.mlp.experts.92.up_proj", "model.layers.60.mlp.experts.93.up_proj", "model.layers.60.mlp.experts.94.up_proj", "model.layers.60.mlp.experts.95.up_proj", "model.layers.60.mlp.experts.96.up_proj", "model.layers.60.mlp.experts.97.up_proj", "model.layers.60.mlp.experts.98.up_proj", "model.layers.60.mlp.experts.99.up_proj", "model.layers.60.mlp.experts.100.up_proj", "model.layers.60.mlp.experts.101.up_proj", "model.layers.60.mlp.experts.102.up_proj", "model.layers.60.mlp.experts.103.up_proj", "model.layers.60.mlp.experts.104.up_proj", "model.layers.60.mlp.experts.105.up_proj", "model.layers.60.mlp.experts.106.up_proj", "model.layers.60.mlp.experts.107.up_proj", "model.layers.60.mlp.experts.108.up_proj", "model.layers.60.mlp.experts.109.up_proj", "model.layers.60.mlp.experts.110.up_proj", "model.layers.60.mlp.experts.111.up_proj", "model.layers.60.mlp.experts.112.up_proj", "model.layers.60.mlp.experts.113.up_proj", "model.layers.60.mlp.experts.114.up_proj", "model.layers.60.mlp.experts.115.up_proj", "model.layers.60.mlp.experts.116.up_proj", "model.layers.60.mlp.experts.117.up_proj", "model.layers.60.mlp.experts.118.up_proj", "model.layers.60.mlp.experts.119.up_proj", "model.layers.60.mlp.experts.120.up_proj", "model.layers.60.mlp.experts.121.up_proj", "model.layers.60.mlp.experts.122.up_proj", "model.layers.60.mlp.experts.123.up_proj", "model.layers.60.mlp.experts.124.up_proj", "model.layers.60.mlp.experts.125.up_proj", "model.layers.60.mlp.experts.126.up_proj", "model.layers.60.mlp.experts.127.up_proj", "model.layers.60.mlp.experts.128.up_proj", "model.layers.60.mlp.experts.129.up_proj", "model.layers.60.mlp.experts.130.up_proj", "model.layers.60.mlp.experts.131.up_proj", "model.layers.60.mlp.experts.132.up_proj", "model.layers.60.mlp.experts.133.up_proj", "model.layers.60.mlp.experts.134.up_proj", "model.layers.60.mlp.experts.135.up_proj", "model.layers.60.mlp.experts.136.up_proj", "model.layers.60.mlp.experts.137.up_proj", "model.layers.60.mlp.experts.138.up_proj", "model.layers.60.mlp.experts.139.up_proj", "model.layers.60.mlp.experts.140.up_proj", "model.layers.60.mlp.experts.141.up_proj", "model.layers.60.mlp.experts.142.up_proj", "model.layers.60.mlp.experts.143.up_proj", "model.layers.60.mlp.experts.144.up_proj", "model.layers.60.mlp.experts.145.up_proj", "model.layers.60.mlp.experts.146.up_proj", "model.layers.60.mlp.experts.147.up_proj", "model.layers.60.mlp.experts.148.up_proj", "model.layers.60.mlp.experts.149.up_proj", "model.layers.60.mlp.experts.150.up_proj", "model.layers.60.mlp.experts.151.up_proj", "model.layers.60.mlp.experts.152.up_proj", "model.layers.60.mlp.experts.153.up_proj", "model.layers.60.mlp.experts.154.up_proj", "model.layers.60.mlp.experts.155.up_proj", "model.layers.60.mlp.experts.156.up_proj", "model.layers.60.mlp.experts.157.up_proj", "model.layers.60.mlp.experts.158.up_proj", "model.layers.60.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -2.9744952917132306e-05, "dbits": 2516582400 } ] }, { "idx": 362, "layers": [ "model.layers.60.mlp.experts.0.down_proj", "model.layers.60.mlp.experts.1.down_proj", "model.layers.60.mlp.experts.2.down_proj", "model.layers.60.mlp.experts.3.down_proj", "model.layers.60.mlp.experts.4.down_proj", "model.layers.60.mlp.experts.5.down_proj", "model.layers.60.mlp.experts.6.down_proj", "model.layers.60.mlp.experts.7.down_proj", "model.layers.60.mlp.experts.8.down_proj", "model.layers.60.mlp.experts.9.down_proj", "model.layers.60.mlp.experts.10.down_proj", "model.layers.60.mlp.experts.11.down_proj", "model.layers.60.mlp.experts.12.down_proj", "model.layers.60.mlp.experts.13.down_proj", "model.layers.60.mlp.experts.14.down_proj", "model.layers.60.mlp.experts.15.down_proj", "model.layers.60.mlp.experts.16.down_proj", "model.layers.60.mlp.experts.17.down_proj", "model.layers.60.mlp.experts.18.down_proj", "model.layers.60.mlp.experts.19.down_proj", "model.layers.60.mlp.experts.20.down_proj", "model.layers.60.mlp.experts.21.down_proj", "model.layers.60.mlp.experts.22.down_proj", "model.layers.60.mlp.experts.23.down_proj", "model.layers.60.mlp.experts.24.down_proj", "model.layers.60.mlp.experts.25.down_proj", "model.layers.60.mlp.experts.26.down_proj", "model.layers.60.mlp.experts.27.down_proj", "model.layers.60.mlp.experts.28.down_proj", "model.layers.60.mlp.experts.29.down_proj", "model.layers.60.mlp.experts.30.down_proj", "model.layers.60.mlp.experts.31.down_proj", "model.layers.60.mlp.experts.32.down_proj", "model.layers.60.mlp.experts.33.down_proj", "model.layers.60.mlp.experts.34.down_proj", "model.layers.60.mlp.experts.35.down_proj", "model.layers.60.mlp.experts.36.down_proj", "model.layers.60.mlp.experts.37.down_proj", "model.layers.60.mlp.experts.38.down_proj", "model.layers.60.mlp.experts.39.down_proj", "model.layers.60.mlp.experts.40.down_proj", "model.layers.60.mlp.experts.41.down_proj", "model.layers.60.mlp.experts.42.down_proj", "model.layers.60.mlp.experts.43.down_proj", "model.layers.60.mlp.experts.44.down_proj", "model.layers.60.mlp.experts.45.down_proj", "model.layers.60.mlp.experts.46.down_proj", "model.layers.60.mlp.experts.47.down_proj", "model.layers.60.mlp.experts.48.down_proj", "model.layers.60.mlp.experts.49.down_proj", "model.layers.60.mlp.experts.50.down_proj", "model.layers.60.mlp.experts.51.down_proj", "model.layers.60.mlp.experts.52.down_proj", "model.layers.60.mlp.experts.53.down_proj", "model.layers.60.mlp.experts.54.down_proj", "model.layers.60.mlp.experts.55.down_proj", "model.layers.60.mlp.experts.56.down_proj", "model.layers.60.mlp.experts.57.down_proj", "model.layers.60.mlp.experts.58.down_proj", "model.layers.60.mlp.experts.59.down_proj", "model.layers.60.mlp.experts.60.down_proj", "model.layers.60.mlp.experts.61.down_proj", "model.layers.60.mlp.experts.62.down_proj", "model.layers.60.mlp.experts.63.down_proj", "model.layers.60.mlp.experts.64.down_proj", "model.layers.60.mlp.experts.65.down_proj", "model.layers.60.mlp.experts.66.down_proj", "model.layers.60.mlp.experts.67.down_proj", "model.layers.60.mlp.experts.68.down_proj", "model.layers.60.mlp.experts.69.down_proj", "model.layers.60.mlp.experts.70.down_proj", "model.layers.60.mlp.experts.71.down_proj", "model.layers.60.mlp.experts.72.down_proj", "model.layers.60.mlp.experts.73.down_proj", "model.layers.60.mlp.experts.74.down_proj", "model.layers.60.mlp.experts.75.down_proj", "model.layers.60.mlp.experts.76.down_proj", "model.layers.60.mlp.experts.77.down_proj", "model.layers.60.mlp.experts.78.down_proj", "model.layers.60.mlp.experts.79.down_proj", "model.layers.60.mlp.experts.80.down_proj", "model.layers.60.mlp.experts.81.down_proj", "model.layers.60.mlp.experts.82.down_proj", "model.layers.60.mlp.experts.83.down_proj", "model.layers.60.mlp.experts.84.down_proj", "model.layers.60.mlp.experts.85.down_proj", "model.layers.60.mlp.experts.86.down_proj", "model.layers.60.mlp.experts.87.down_proj", "model.layers.60.mlp.experts.88.down_proj", "model.layers.60.mlp.experts.89.down_proj", "model.layers.60.mlp.experts.90.down_proj", "model.layers.60.mlp.experts.91.down_proj", "model.layers.60.mlp.experts.92.down_proj", "model.layers.60.mlp.experts.93.down_proj", "model.layers.60.mlp.experts.94.down_proj", "model.layers.60.mlp.experts.95.down_proj", "model.layers.60.mlp.experts.96.down_proj", "model.layers.60.mlp.experts.97.down_proj", "model.layers.60.mlp.experts.98.down_proj", "model.layers.60.mlp.experts.99.down_proj", "model.layers.60.mlp.experts.100.down_proj", "model.layers.60.mlp.experts.101.down_proj", "model.layers.60.mlp.experts.102.down_proj", "model.layers.60.mlp.experts.103.down_proj", "model.layers.60.mlp.experts.104.down_proj", "model.layers.60.mlp.experts.105.down_proj", "model.layers.60.mlp.experts.106.down_proj", "model.layers.60.mlp.experts.107.down_proj", "model.layers.60.mlp.experts.108.down_proj", "model.layers.60.mlp.experts.109.down_proj", "model.layers.60.mlp.experts.110.down_proj", "model.layers.60.mlp.experts.111.down_proj", "model.layers.60.mlp.experts.112.down_proj", "model.layers.60.mlp.experts.113.down_proj", "model.layers.60.mlp.experts.114.down_proj", "model.layers.60.mlp.experts.115.down_proj", "model.layers.60.mlp.experts.116.down_proj", "model.layers.60.mlp.experts.117.down_proj", "model.layers.60.mlp.experts.118.down_proj", "model.layers.60.mlp.experts.119.down_proj", "model.layers.60.mlp.experts.120.down_proj", "model.layers.60.mlp.experts.121.down_proj", "model.layers.60.mlp.experts.122.down_proj", "model.layers.60.mlp.experts.123.down_proj", "model.layers.60.mlp.experts.124.down_proj", "model.layers.60.mlp.experts.125.down_proj", "model.layers.60.mlp.experts.126.down_proj", "model.layers.60.mlp.experts.127.down_proj", "model.layers.60.mlp.experts.128.down_proj", "model.layers.60.mlp.experts.129.down_proj", "model.layers.60.mlp.experts.130.down_proj", "model.layers.60.mlp.experts.131.down_proj", "model.layers.60.mlp.experts.132.down_proj", "model.layers.60.mlp.experts.133.down_proj", "model.layers.60.mlp.experts.134.down_proj", "model.layers.60.mlp.experts.135.down_proj", "model.layers.60.mlp.experts.136.down_proj", "model.layers.60.mlp.experts.137.down_proj", "model.layers.60.mlp.experts.138.down_proj", "model.layers.60.mlp.experts.139.down_proj", "model.layers.60.mlp.experts.140.down_proj", "model.layers.60.mlp.experts.141.down_proj", "model.layers.60.mlp.experts.142.down_proj", "model.layers.60.mlp.experts.143.down_proj", "model.layers.60.mlp.experts.144.down_proj", "model.layers.60.mlp.experts.145.down_proj", "model.layers.60.mlp.experts.146.down_proj", "model.layers.60.mlp.experts.147.down_proj", "model.layers.60.mlp.experts.148.down_proj", "model.layers.60.mlp.experts.149.down_proj", "model.layers.60.mlp.experts.150.down_proj", "model.layers.60.mlp.experts.151.down_proj", "model.layers.60.mlp.experts.152.down_proj", "model.layers.60.mlp.experts.153.down_proj", "model.layers.60.mlp.experts.154.down_proj", "model.layers.60.mlp.experts.155.down_proj", "model.layers.60.mlp.experts.156.down_proj", "model.layers.60.mlp.experts.157.down_proj", "model.layers.60.mlp.experts.158.down_proj", "model.layers.60.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0002992391586303933, "dbits": 1258291200 } ] }, { "idx": 363, "layers": [ "model.layers.61.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0011672593653201946, "dbits": 62914560 } ] }, { "idx": 364, "layers": [ "model.layers.61.self_attn.k_proj", "model.layers.61.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0001747123897075653, "dbits": 10485760 } ] }, { "idx": 365, "layers": [ "model.layers.61.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0014454208314418904, "dbits": 62914560 } ] }, { "idx": 366, "layers": [ "model.layers.61.mlp.shared_experts.gate_proj", "model.layers.61.mlp.shared_experts.up_proj", "model.layers.61.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0005098171532154083, "dbits": 23592960 } ] }, { "idx": 367, "layers": [ "model.layers.61.mlp.experts.0.gate_proj", "model.layers.61.mlp.experts.1.gate_proj", "model.layers.61.mlp.experts.2.gate_proj", "model.layers.61.mlp.experts.3.gate_proj", "model.layers.61.mlp.experts.4.gate_proj", "model.layers.61.mlp.experts.5.gate_proj", "model.layers.61.mlp.experts.6.gate_proj", "model.layers.61.mlp.experts.7.gate_proj", "model.layers.61.mlp.experts.8.gate_proj", "model.layers.61.mlp.experts.9.gate_proj", "model.layers.61.mlp.experts.10.gate_proj", "model.layers.61.mlp.experts.11.gate_proj", "model.layers.61.mlp.experts.12.gate_proj", "model.layers.61.mlp.experts.13.gate_proj", "model.layers.61.mlp.experts.14.gate_proj", "model.layers.61.mlp.experts.15.gate_proj", "model.layers.61.mlp.experts.16.gate_proj", "model.layers.61.mlp.experts.17.gate_proj", "model.layers.61.mlp.experts.18.gate_proj", "model.layers.61.mlp.experts.19.gate_proj", "model.layers.61.mlp.experts.20.gate_proj", "model.layers.61.mlp.experts.21.gate_proj", "model.layers.61.mlp.experts.22.gate_proj", "model.layers.61.mlp.experts.23.gate_proj", "model.layers.61.mlp.experts.24.gate_proj", "model.layers.61.mlp.experts.25.gate_proj", "model.layers.61.mlp.experts.26.gate_proj", "model.layers.61.mlp.experts.27.gate_proj", "model.layers.61.mlp.experts.28.gate_proj", "model.layers.61.mlp.experts.29.gate_proj", "model.layers.61.mlp.experts.30.gate_proj", "model.layers.61.mlp.experts.31.gate_proj", "model.layers.61.mlp.experts.32.gate_proj", "model.layers.61.mlp.experts.33.gate_proj", "model.layers.61.mlp.experts.34.gate_proj", "model.layers.61.mlp.experts.35.gate_proj", "model.layers.61.mlp.experts.36.gate_proj", "model.layers.61.mlp.experts.37.gate_proj", "model.layers.61.mlp.experts.38.gate_proj", "model.layers.61.mlp.experts.39.gate_proj", "model.layers.61.mlp.experts.40.gate_proj", "model.layers.61.mlp.experts.41.gate_proj", "model.layers.61.mlp.experts.42.gate_proj", "model.layers.61.mlp.experts.43.gate_proj", "model.layers.61.mlp.experts.44.gate_proj", "model.layers.61.mlp.experts.45.gate_proj", "model.layers.61.mlp.experts.46.gate_proj", "model.layers.61.mlp.experts.47.gate_proj", "model.layers.61.mlp.experts.48.gate_proj", "model.layers.61.mlp.experts.49.gate_proj", "model.layers.61.mlp.experts.50.gate_proj", "model.layers.61.mlp.experts.51.gate_proj", "model.layers.61.mlp.experts.52.gate_proj", "model.layers.61.mlp.experts.53.gate_proj", "model.layers.61.mlp.experts.54.gate_proj", "model.layers.61.mlp.experts.55.gate_proj", "model.layers.61.mlp.experts.56.gate_proj", "model.layers.61.mlp.experts.57.gate_proj", "model.layers.61.mlp.experts.58.gate_proj", "model.layers.61.mlp.experts.59.gate_proj", "model.layers.61.mlp.experts.60.gate_proj", "model.layers.61.mlp.experts.61.gate_proj", "model.layers.61.mlp.experts.62.gate_proj", "model.layers.61.mlp.experts.63.gate_proj", "model.layers.61.mlp.experts.64.gate_proj", "model.layers.61.mlp.experts.65.gate_proj", "model.layers.61.mlp.experts.66.gate_proj", "model.layers.61.mlp.experts.67.gate_proj", "model.layers.61.mlp.experts.68.gate_proj", "model.layers.61.mlp.experts.69.gate_proj", "model.layers.61.mlp.experts.70.gate_proj", "model.layers.61.mlp.experts.71.gate_proj", "model.layers.61.mlp.experts.72.gate_proj", "model.layers.61.mlp.experts.73.gate_proj", "model.layers.61.mlp.experts.74.gate_proj", "model.layers.61.mlp.experts.75.gate_proj", "model.layers.61.mlp.experts.76.gate_proj", "model.layers.61.mlp.experts.77.gate_proj", "model.layers.61.mlp.experts.78.gate_proj", "model.layers.61.mlp.experts.79.gate_proj", "model.layers.61.mlp.experts.80.gate_proj", "model.layers.61.mlp.experts.81.gate_proj", "model.layers.61.mlp.experts.82.gate_proj", "model.layers.61.mlp.experts.83.gate_proj", "model.layers.61.mlp.experts.84.gate_proj", "model.layers.61.mlp.experts.85.gate_proj", "model.layers.61.mlp.experts.86.gate_proj", "model.layers.61.mlp.experts.87.gate_proj", "model.layers.61.mlp.experts.88.gate_proj", "model.layers.61.mlp.experts.89.gate_proj", "model.layers.61.mlp.experts.90.gate_proj", "model.layers.61.mlp.experts.91.gate_proj", "model.layers.61.mlp.experts.92.gate_proj", "model.layers.61.mlp.experts.93.gate_proj", "model.layers.61.mlp.experts.94.gate_proj", "model.layers.61.mlp.experts.95.gate_proj", "model.layers.61.mlp.experts.96.gate_proj", "model.layers.61.mlp.experts.97.gate_proj", "model.layers.61.mlp.experts.98.gate_proj", "model.layers.61.mlp.experts.99.gate_proj", "model.layers.61.mlp.experts.100.gate_proj", "model.layers.61.mlp.experts.101.gate_proj", "model.layers.61.mlp.experts.102.gate_proj", "model.layers.61.mlp.experts.103.gate_proj", "model.layers.61.mlp.experts.104.gate_proj", "model.layers.61.mlp.experts.105.gate_proj", "model.layers.61.mlp.experts.106.gate_proj", "model.layers.61.mlp.experts.107.gate_proj", "model.layers.61.mlp.experts.108.gate_proj", "model.layers.61.mlp.experts.109.gate_proj", "model.layers.61.mlp.experts.110.gate_proj", "model.layers.61.mlp.experts.111.gate_proj", "model.layers.61.mlp.experts.112.gate_proj", "model.layers.61.mlp.experts.113.gate_proj", "model.layers.61.mlp.experts.114.gate_proj", "model.layers.61.mlp.experts.115.gate_proj", "model.layers.61.mlp.experts.116.gate_proj", "model.layers.61.mlp.experts.117.gate_proj", "model.layers.61.mlp.experts.118.gate_proj", "model.layers.61.mlp.experts.119.gate_proj", "model.layers.61.mlp.experts.120.gate_proj", "model.layers.61.mlp.experts.121.gate_proj", "model.layers.61.mlp.experts.122.gate_proj", "model.layers.61.mlp.experts.123.gate_proj", "model.layers.61.mlp.experts.124.gate_proj", "model.layers.61.mlp.experts.125.gate_proj", "model.layers.61.mlp.experts.126.gate_proj", "model.layers.61.mlp.experts.127.gate_proj", "model.layers.61.mlp.experts.128.gate_proj", "model.layers.61.mlp.experts.129.gate_proj", "model.layers.61.mlp.experts.130.gate_proj", "model.layers.61.mlp.experts.131.gate_proj", "model.layers.61.mlp.experts.132.gate_proj", "model.layers.61.mlp.experts.133.gate_proj", "model.layers.61.mlp.experts.134.gate_proj", "model.layers.61.mlp.experts.135.gate_proj", "model.layers.61.mlp.experts.136.gate_proj", "model.layers.61.mlp.experts.137.gate_proj", "model.layers.61.mlp.experts.138.gate_proj", "model.layers.61.mlp.experts.139.gate_proj", "model.layers.61.mlp.experts.140.gate_proj", "model.layers.61.mlp.experts.141.gate_proj", "model.layers.61.mlp.experts.142.gate_proj", "model.layers.61.mlp.experts.143.gate_proj", "model.layers.61.mlp.experts.144.gate_proj", "model.layers.61.mlp.experts.145.gate_proj", "model.layers.61.mlp.experts.146.gate_proj", "model.layers.61.mlp.experts.147.gate_proj", "model.layers.61.mlp.experts.148.gate_proj", "model.layers.61.mlp.experts.149.gate_proj", "model.layers.61.mlp.experts.150.gate_proj", "model.layers.61.mlp.experts.151.gate_proj", "model.layers.61.mlp.experts.152.gate_proj", "model.layers.61.mlp.experts.153.gate_proj", "model.layers.61.mlp.experts.154.gate_proj", "model.layers.61.mlp.experts.155.gate_proj", "model.layers.61.mlp.experts.156.gate_proj", "model.layers.61.mlp.experts.157.gate_proj", "model.layers.61.mlp.experts.158.gate_proj", "model.layers.61.mlp.experts.159.gate_proj", "model.layers.61.mlp.experts.0.up_proj", "model.layers.61.mlp.experts.1.up_proj", "model.layers.61.mlp.experts.2.up_proj", "model.layers.61.mlp.experts.3.up_proj", "model.layers.61.mlp.experts.4.up_proj", "model.layers.61.mlp.experts.5.up_proj", "model.layers.61.mlp.experts.6.up_proj", "model.layers.61.mlp.experts.7.up_proj", "model.layers.61.mlp.experts.8.up_proj", "model.layers.61.mlp.experts.9.up_proj", "model.layers.61.mlp.experts.10.up_proj", "model.layers.61.mlp.experts.11.up_proj", "model.layers.61.mlp.experts.12.up_proj", "model.layers.61.mlp.experts.13.up_proj", "model.layers.61.mlp.experts.14.up_proj", "model.layers.61.mlp.experts.15.up_proj", "model.layers.61.mlp.experts.16.up_proj", "model.layers.61.mlp.experts.17.up_proj", "model.layers.61.mlp.experts.18.up_proj", "model.layers.61.mlp.experts.19.up_proj", "model.layers.61.mlp.experts.20.up_proj", "model.layers.61.mlp.experts.21.up_proj", "model.layers.61.mlp.experts.22.up_proj", "model.layers.61.mlp.experts.23.up_proj", "model.layers.61.mlp.experts.24.up_proj", "model.layers.61.mlp.experts.25.up_proj", "model.layers.61.mlp.experts.26.up_proj", "model.layers.61.mlp.experts.27.up_proj", "model.layers.61.mlp.experts.28.up_proj", "model.layers.61.mlp.experts.29.up_proj", "model.layers.61.mlp.experts.30.up_proj", "model.layers.61.mlp.experts.31.up_proj", "model.layers.61.mlp.experts.32.up_proj", "model.layers.61.mlp.experts.33.up_proj", "model.layers.61.mlp.experts.34.up_proj", "model.layers.61.mlp.experts.35.up_proj", "model.layers.61.mlp.experts.36.up_proj", "model.layers.61.mlp.experts.37.up_proj", "model.layers.61.mlp.experts.38.up_proj", "model.layers.61.mlp.experts.39.up_proj", "model.layers.61.mlp.experts.40.up_proj", "model.layers.61.mlp.experts.41.up_proj", "model.layers.61.mlp.experts.42.up_proj", "model.layers.61.mlp.experts.43.up_proj", "model.layers.61.mlp.experts.44.up_proj", "model.layers.61.mlp.experts.45.up_proj", "model.layers.61.mlp.experts.46.up_proj", "model.layers.61.mlp.experts.47.up_proj", "model.layers.61.mlp.experts.48.up_proj", "model.layers.61.mlp.experts.49.up_proj", "model.layers.61.mlp.experts.50.up_proj", "model.layers.61.mlp.experts.51.up_proj", "model.layers.61.mlp.experts.52.up_proj", "model.layers.61.mlp.experts.53.up_proj", "model.layers.61.mlp.experts.54.up_proj", "model.layers.61.mlp.experts.55.up_proj", "model.layers.61.mlp.experts.56.up_proj", "model.layers.61.mlp.experts.57.up_proj", "model.layers.61.mlp.experts.58.up_proj", "model.layers.61.mlp.experts.59.up_proj", "model.layers.61.mlp.experts.60.up_proj", "model.layers.61.mlp.experts.61.up_proj", "model.layers.61.mlp.experts.62.up_proj", "model.layers.61.mlp.experts.63.up_proj", "model.layers.61.mlp.experts.64.up_proj", "model.layers.61.mlp.experts.65.up_proj", "model.layers.61.mlp.experts.66.up_proj", "model.layers.61.mlp.experts.67.up_proj", "model.layers.61.mlp.experts.68.up_proj", "model.layers.61.mlp.experts.69.up_proj", "model.layers.61.mlp.experts.70.up_proj", "model.layers.61.mlp.experts.71.up_proj", "model.layers.61.mlp.experts.72.up_proj", "model.layers.61.mlp.experts.73.up_proj", "model.layers.61.mlp.experts.74.up_proj", "model.layers.61.mlp.experts.75.up_proj", "model.layers.61.mlp.experts.76.up_proj", "model.layers.61.mlp.experts.77.up_proj", "model.layers.61.mlp.experts.78.up_proj", "model.layers.61.mlp.experts.79.up_proj", "model.layers.61.mlp.experts.80.up_proj", "model.layers.61.mlp.experts.81.up_proj", "model.layers.61.mlp.experts.82.up_proj", "model.layers.61.mlp.experts.83.up_proj", "model.layers.61.mlp.experts.84.up_proj", "model.layers.61.mlp.experts.85.up_proj", "model.layers.61.mlp.experts.86.up_proj", "model.layers.61.mlp.experts.87.up_proj", "model.layers.61.mlp.experts.88.up_proj", "model.layers.61.mlp.experts.89.up_proj", "model.layers.61.mlp.experts.90.up_proj", "model.layers.61.mlp.experts.91.up_proj", "model.layers.61.mlp.experts.92.up_proj", "model.layers.61.mlp.experts.93.up_proj", "model.layers.61.mlp.experts.94.up_proj", "model.layers.61.mlp.experts.95.up_proj", "model.layers.61.mlp.experts.96.up_proj", "model.layers.61.mlp.experts.97.up_proj", "model.layers.61.mlp.experts.98.up_proj", "model.layers.61.mlp.experts.99.up_proj", "model.layers.61.mlp.experts.100.up_proj", "model.layers.61.mlp.experts.101.up_proj", "model.layers.61.mlp.experts.102.up_proj", "model.layers.61.mlp.experts.103.up_proj", "model.layers.61.mlp.experts.104.up_proj", "model.layers.61.mlp.experts.105.up_proj", "model.layers.61.mlp.experts.106.up_proj", "model.layers.61.mlp.experts.107.up_proj", "model.layers.61.mlp.experts.108.up_proj", "model.layers.61.mlp.experts.109.up_proj", "model.layers.61.mlp.experts.110.up_proj", "model.layers.61.mlp.experts.111.up_proj", "model.layers.61.mlp.experts.112.up_proj", "model.layers.61.mlp.experts.113.up_proj", "model.layers.61.mlp.experts.114.up_proj", "model.layers.61.mlp.experts.115.up_proj", "model.layers.61.mlp.experts.116.up_proj", "model.layers.61.mlp.experts.117.up_proj", "model.layers.61.mlp.experts.118.up_proj", "model.layers.61.mlp.experts.119.up_proj", "model.layers.61.mlp.experts.120.up_proj", "model.layers.61.mlp.experts.121.up_proj", "model.layers.61.mlp.experts.122.up_proj", "model.layers.61.mlp.experts.123.up_proj", "model.layers.61.mlp.experts.124.up_proj", "model.layers.61.mlp.experts.125.up_proj", "model.layers.61.mlp.experts.126.up_proj", "model.layers.61.mlp.experts.127.up_proj", "model.layers.61.mlp.experts.128.up_proj", "model.layers.61.mlp.experts.129.up_proj", "model.layers.61.mlp.experts.130.up_proj", "model.layers.61.mlp.experts.131.up_proj", "model.layers.61.mlp.experts.132.up_proj", "model.layers.61.mlp.experts.133.up_proj", "model.layers.61.mlp.experts.134.up_proj", "model.layers.61.mlp.experts.135.up_proj", "model.layers.61.mlp.experts.136.up_proj", "model.layers.61.mlp.experts.137.up_proj", "model.layers.61.mlp.experts.138.up_proj", "model.layers.61.mlp.experts.139.up_proj", "model.layers.61.mlp.experts.140.up_proj", "model.layers.61.mlp.experts.141.up_proj", "model.layers.61.mlp.experts.142.up_proj", "model.layers.61.mlp.experts.143.up_proj", "model.layers.61.mlp.experts.144.up_proj", "model.layers.61.mlp.experts.145.up_proj", "model.layers.61.mlp.experts.146.up_proj", "model.layers.61.mlp.experts.147.up_proj", "model.layers.61.mlp.experts.148.up_proj", "model.layers.61.mlp.experts.149.up_proj", "model.layers.61.mlp.experts.150.up_proj", "model.layers.61.mlp.experts.151.up_proj", "model.layers.61.mlp.experts.152.up_proj", "model.layers.61.mlp.experts.153.up_proj", "model.layers.61.mlp.experts.154.up_proj", "model.layers.61.mlp.experts.155.up_proj", "model.layers.61.mlp.experts.156.up_proj", "model.layers.61.mlp.experts.157.up_proj", "model.layers.61.mlp.experts.158.up_proj", "model.layers.61.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0003510415554046631, "dbits": 2516582400 } ] }, { "idx": 368, "layers": [ "model.layers.61.mlp.experts.0.down_proj", "model.layers.61.mlp.experts.1.down_proj", "model.layers.61.mlp.experts.2.down_proj", "model.layers.61.mlp.experts.3.down_proj", "model.layers.61.mlp.experts.4.down_proj", "model.layers.61.mlp.experts.5.down_proj", "model.layers.61.mlp.experts.6.down_proj", "model.layers.61.mlp.experts.7.down_proj", "model.layers.61.mlp.experts.8.down_proj", "model.layers.61.mlp.experts.9.down_proj", "model.layers.61.mlp.experts.10.down_proj", "model.layers.61.mlp.experts.11.down_proj", "model.layers.61.mlp.experts.12.down_proj", "model.layers.61.mlp.experts.13.down_proj", "model.layers.61.mlp.experts.14.down_proj", "model.layers.61.mlp.experts.15.down_proj", "model.layers.61.mlp.experts.16.down_proj", "model.layers.61.mlp.experts.17.down_proj", "model.layers.61.mlp.experts.18.down_proj", "model.layers.61.mlp.experts.19.down_proj", "model.layers.61.mlp.experts.20.down_proj", "model.layers.61.mlp.experts.21.down_proj", "model.layers.61.mlp.experts.22.down_proj", "model.layers.61.mlp.experts.23.down_proj", "model.layers.61.mlp.experts.24.down_proj", "model.layers.61.mlp.experts.25.down_proj", "model.layers.61.mlp.experts.26.down_proj", "model.layers.61.mlp.experts.27.down_proj", "model.layers.61.mlp.experts.28.down_proj", "model.layers.61.mlp.experts.29.down_proj", "model.layers.61.mlp.experts.30.down_proj", "model.layers.61.mlp.experts.31.down_proj", "model.layers.61.mlp.experts.32.down_proj", "model.layers.61.mlp.experts.33.down_proj", "model.layers.61.mlp.experts.34.down_proj", "model.layers.61.mlp.experts.35.down_proj", "model.layers.61.mlp.experts.36.down_proj", "model.layers.61.mlp.experts.37.down_proj", "model.layers.61.mlp.experts.38.down_proj", "model.layers.61.mlp.experts.39.down_proj", "model.layers.61.mlp.experts.40.down_proj", "model.layers.61.mlp.experts.41.down_proj", "model.layers.61.mlp.experts.42.down_proj", "model.layers.61.mlp.experts.43.down_proj", "model.layers.61.mlp.experts.44.down_proj", "model.layers.61.mlp.experts.45.down_proj", "model.layers.61.mlp.experts.46.down_proj", "model.layers.61.mlp.experts.47.down_proj", "model.layers.61.mlp.experts.48.down_proj", "model.layers.61.mlp.experts.49.down_proj", "model.layers.61.mlp.experts.50.down_proj", "model.layers.61.mlp.experts.51.down_proj", "model.layers.61.mlp.experts.52.down_proj", "model.layers.61.mlp.experts.53.down_proj", "model.layers.61.mlp.experts.54.down_proj", "model.layers.61.mlp.experts.55.down_proj", "model.layers.61.mlp.experts.56.down_proj", "model.layers.61.mlp.experts.57.down_proj", "model.layers.61.mlp.experts.58.down_proj", "model.layers.61.mlp.experts.59.down_proj", "model.layers.61.mlp.experts.60.down_proj", "model.layers.61.mlp.experts.61.down_proj", "model.layers.61.mlp.experts.62.down_proj", "model.layers.61.mlp.experts.63.down_proj", "model.layers.61.mlp.experts.64.down_proj", "model.layers.61.mlp.experts.65.down_proj", "model.layers.61.mlp.experts.66.down_proj", "model.layers.61.mlp.experts.67.down_proj", "model.layers.61.mlp.experts.68.down_proj", "model.layers.61.mlp.experts.69.down_proj", "model.layers.61.mlp.experts.70.down_proj", "model.layers.61.mlp.experts.71.down_proj", "model.layers.61.mlp.experts.72.down_proj", "model.layers.61.mlp.experts.73.down_proj", "model.layers.61.mlp.experts.74.down_proj", "model.layers.61.mlp.experts.75.down_proj", "model.layers.61.mlp.experts.76.down_proj", "model.layers.61.mlp.experts.77.down_proj", "model.layers.61.mlp.experts.78.down_proj", "model.layers.61.mlp.experts.79.down_proj", "model.layers.61.mlp.experts.80.down_proj", "model.layers.61.mlp.experts.81.down_proj", "model.layers.61.mlp.experts.82.down_proj", "model.layers.61.mlp.experts.83.down_proj", "model.layers.61.mlp.experts.84.down_proj", "model.layers.61.mlp.experts.85.down_proj", "model.layers.61.mlp.experts.86.down_proj", "model.layers.61.mlp.experts.87.down_proj", "model.layers.61.mlp.experts.88.down_proj", "model.layers.61.mlp.experts.89.down_proj", "model.layers.61.mlp.experts.90.down_proj", "model.layers.61.mlp.experts.91.down_proj", "model.layers.61.mlp.experts.92.down_proj", "model.layers.61.mlp.experts.93.down_proj", "model.layers.61.mlp.experts.94.down_proj", "model.layers.61.mlp.experts.95.down_proj", "model.layers.61.mlp.experts.96.down_proj", "model.layers.61.mlp.experts.97.down_proj", "model.layers.61.mlp.experts.98.down_proj", "model.layers.61.mlp.experts.99.down_proj", "model.layers.61.mlp.experts.100.down_proj", "model.layers.61.mlp.experts.101.down_proj", "model.layers.61.mlp.experts.102.down_proj", "model.layers.61.mlp.experts.103.down_proj", "model.layers.61.mlp.experts.104.down_proj", "model.layers.61.mlp.experts.105.down_proj", "model.layers.61.mlp.experts.106.down_proj", "model.layers.61.mlp.experts.107.down_proj", "model.layers.61.mlp.experts.108.down_proj", "model.layers.61.mlp.experts.109.down_proj", "model.layers.61.mlp.experts.110.down_proj", "model.layers.61.mlp.experts.111.down_proj", "model.layers.61.mlp.experts.112.down_proj", "model.layers.61.mlp.experts.113.down_proj", "model.layers.61.mlp.experts.114.down_proj", "model.layers.61.mlp.experts.115.down_proj", "model.layers.61.mlp.experts.116.down_proj", "model.layers.61.mlp.experts.117.down_proj", "model.layers.61.mlp.experts.118.down_proj", "model.layers.61.mlp.experts.119.down_proj", "model.layers.61.mlp.experts.120.down_proj", "model.layers.61.mlp.experts.121.down_proj", "model.layers.61.mlp.experts.122.down_proj", "model.layers.61.mlp.experts.123.down_proj", "model.layers.61.mlp.experts.124.down_proj", "model.layers.61.mlp.experts.125.down_proj", "model.layers.61.mlp.experts.126.down_proj", "model.layers.61.mlp.experts.127.down_proj", "model.layers.61.mlp.experts.128.down_proj", "model.layers.61.mlp.experts.129.down_proj", "model.layers.61.mlp.experts.130.down_proj", "model.layers.61.mlp.experts.131.down_proj", "model.layers.61.mlp.experts.132.down_proj", "model.layers.61.mlp.experts.133.down_proj", "model.layers.61.mlp.experts.134.down_proj", "model.layers.61.mlp.experts.135.down_proj", "model.layers.61.mlp.experts.136.down_proj", "model.layers.61.mlp.experts.137.down_proj", "model.layers.61.mlp.experts.138.down_proj", "model.layers.61.mlp.experts.139.down_proj", "model.layers.61.mlp.experts.140.down_proj", "model.layers.61.mlp.experts.141.down_proj", "model.layers.61.mlp.experts.142.down_proj", "model.layers.61.mlp.experts.143.down_proj", "model.layers.61.mlp.experts.144.down_proj", "model.layers.61.mlp.experts.145.down_proj", "model.layers.61.mlp.experts.146.down_proj", "model.layers.61.mlp.experts.147.down_proj", "model.layers.61.mlp.experts.148.down_proj", "model.layers.61.mlp.experts.149.down_proj", "model.layers.61.mlp.experts.150.down_proj", "model.layers.61.mlp.experts.151.down_proj", "model.layers.61.mlp.experts.152.down_proj", "model.layers.61.mlp.experts.153.down_proj", "model.layers.61.mlp.experts.154.down_proj", "model.layers.61.mlp.experts.155.down_proj", "model.layers.61.mlp.experts.156.down_proj", "model.layers.61.mlp.experts.157.down_proj", "model.layers.61.mlp.experts.158.down_proj", "model.layers.61.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006620429456233867, "dbits": 1258291200 } ] }, { "idx": 369, "layers": [ "model.layers.62.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00024027898907663658, "dbits": 62914560 } ] }, { "idx": 370, "layers": [ "model.layers.62.self_attn.k_proj", "model.layers.62.self_attn.v_proj" ], "candidates": [ { "dkld": -5.483254790306091e-05, "dbits": 10485760 } ] }, { "idx": 371, "layers": [ "model.layers.62.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0005622796714305767, "dbits": 62914560 } ] }, { "idx": 372, "layers": [ "model.layers.62.mlp.shared_experts.gate_proj", "model.layers.62.mlp.shared_experts.up_proj", "model.layers.62.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0004636809229850991, "dbits": 23592960 } ] }, { "idx": 373, "layers": [ "model.layers.62.mlp.experts.0.gate_proj", "model.layers.62.mlp.experts.1.gate_proj", "model.layers.62.mlp.experts.2.gate_proj", "model.layers.62.mlp.experts.3.gate_proj", "model.layers.62.mlp.experts.4.gate_proj", "model.layers.62.mlp.experts.5.gate_proj", "model.layers.62.mlp.experts.6.gate_proj", "model.layers.62.mlp.experts.7.gate_proj", "model.layers.62.mlp.experts.8.gate_proj", "model.layers.62.mlp.experts.9.gate_proj", "model.layers.62.mlp.experts.10.gate_proj", "model.layers.62.mlp.experts.11.gate_proj", "model.layers.62.mlp.experts.12.gate_proj", "model.layers.62.mlp.experts.13.gate_proj", "model.layers.62.mlp.experts.14.gate_proj", "model.layers.62.mlp.experts.15.gate_proj", "model.layers.62.mlp.experts.16.gate_proj", "model.layers.62.mlp.experts.17.gate_proj", "model.layers.62.mlp.experts.18.gate_proj", "model.layers.62.mlp.experts.19.gate_proj", "model.layers.62.mlp.experts.20.gate_proj", "model.layers.62.mlp.experts.21.gate_proj", "model.layers.62.mlp.experts.22.gate_proj", "model.layers.62.mlp.experts.23.gate_proj", "model.layers.62.mlp.experts.24.gate_proj", "model.layers.62.mlp.experts.25.gate_proj", "model.layers.62.mlp.experts.26.gate_proj", "model.layers.62.mlp.experts.27.gate_proj", "model.layers.62.mlp.experts.28.gate_proj", "model.layers.62.mlp.experts.29.gate_proj", "model.layers.62.mlp.experts.30.gate_proj", "model.layers.62.mlp.experts.31.gate_proj", "model.layers.62.mlp.experts.32.gate_proj", "model.layers.62.mlp.experts.33.gate_proj", "model.layers.62.mlp.experts.34.gate_proj", "model.layers.62.mlp.experts.35.gate_proj", "model.layers.62.mlp.experts.36.gate_proj", "model.layers.62.mlp.experts.37.gate_proj", "model.layers.62.mlp.experts.38.gate_proj", "model.layers.62.mlp.experts.39.gate_proj", "model.layers.62.mlp.experts.40.gate_proj", "model.layers.62.mlp.experts.41.gate_proj", "model.layers.62.mlp.experts.42.gate_proj", "model.layers.62.mlp.experts.43.gate_proj", "model.layers.62.mlp.experts.44.gate_proj", "model.layers.62.mlp.experts.45.gate_proj", "model.layers.62.mlp.experts.46.gate_proj", "model.layers.62.mlp.experts.47.gate_proj", "model.layers.62.mlp.experts.48.gate_proj", "model.layers.62.mlp.experts.49.gate_proj", "model.layers.62.mlp.experts.50.gate_proj", "model.layers.62.mlp.experts.51.gate_proj", "model.layers.62.mlp.experts.52.gate_proj", "model.layers.62.mlp.experts.53.gate_proj", "model.layers.62.mlp.experts.54.gate_proj", "model.layers.62.mlp.experts.55.gate_proj", "model.layers.62.mlp.experts.56.gate_proj", "model.layers.62.mlp.experts.57.gate_proj", "model.layers.62.mlp.experts.58.gate_proj", "model.layers.62.mlp.experts.59.gate_proj", "model.layers.62.mlp.experts.60.gate_proj", "model.layers.62.mlp.experts.61.gate_proj", "model.layers.62.mlp.experts.62.gate_proj", "model.layers.62.mlp.experts.63.gate_proj", "model.layers.62.mlp.experts.64.gate_proj", "model.layers.62.mlp.experts.65.gate_proj", "model.layers.62.mlp.experts.66.gate_proj", "model.layers.62.mlp.experts.67.gate_proj", "model.layers.62.mlp.experts.68.gate_proj", "model.layers.62.mlp.experts.69.gate_proj", "model.layers.62.mlp.experts.70.gate_proj", "model.layers.62.mlp.experts.71.gate_proj", "model.layers.62.mlp.experts.72.gate_proj", "model.layers.62.mlp.experts.73.gate_proj", "model.layers.62.mlp.experts.74.gate_proj", "model.layers.62.mlp.experts.75.gate_proj", "model.layers.62.mlp.experts.76.gate_proj", "model.layers.62.mlp.experts.77.gate_proj", "model.layers.62.mlp.experts.78.gate_proj", "model.layers.62.mlp.experts.79.gate_proj", "model.layers.62.mlp.experts.80.gate_proj", "model.layers.62.mlp.experts.81.gate_proj", "model.layers.62.mlp.experts.82.gate_proj", "model.layers.62.mlp.experts.83.gate_proj", "model.layers.62.mlp.experts.84.gate_proj", "model.layers.62.mlp.experts.85.gate_proj", "model.layers.62.mlp.experts.86.gate_proj", "model.layers.62.mlp.experts.87.gate_proj", "model.layers.62.mlp.experts.88.gate_proj", "model.layers.62.mlp.experts.89.gate_proj", "model.layers.62.mlp.experts.90.gate_proj", "model.layers.62.mlp.experts.91.gate_proj", "model.layers.62.mlp.experts.92.gate_proj", "model.layers.62.mlp.experts.93.gate_proj", "model.layers.62.mlp.experts.94.gate_proj", "model.layers.62.mlp.experts.95.gate_proj", "model.layers.62.mlp.experts.96.gate_proj", "model.layers.62.mlp.experts.97.gate_proj", "model.layers.62.mlp.experts.98.gate_proj", "model.layers.62.mlp.experts.99.gate_proj", "model.layers.62.mlp.experts.100.gate_proj", "model.layers.62.mlp.experts.101.gate_proj", "model.layers.62.mlp.experts.102.gate_proj", "model.layers.62.mlp.experts.103.gate_proj", "model.layers.62.mlp.experts.104.gate_proj", "model.layers.62.mlp.experts.105.gate_proj", "model.layers.62.mlp.experts.106.gate_proj", "model.layers.62.mlp.experts.107.gate_proj", "model.layers.62.mlp.experts.108.gate_proj", "model.layers.62.mlp.experts.109.gate_proj", "model.layers.62.mlp.experts.110.gate_proj", "model.layers.62.mlp.experts.111.gate_proj", "model.layers.62.mlp.experts.112.gate_proj", "model.layers.62.mlp.experts.113.gate_proj", "model.layers.62.mlp.experts.114.gate_proj", "model.layers.62.mlp.experts.115.gate_proj", "model.layers.62.mlp.experts.116.gate_proj", "model.layers.62.mlp.experts.117.gate_proj", "model.layers.62.mlp.experts.118.gate_proj", "model.layers.62.mlp.experts.119.gate_proj", "model.layers.62.mlp.experts.120.gate_proj", "model.layers.62.mlp.experts.121.gate_proj", "model.layers.62.mlp.experts.122.gate_proj", "model.layers.62.mlp.experts.123.gate_proj", "model.layers.62.mlp.experts.124.gate_proj", "model.layers.62.mlp.experts.125.gate_proj", "model.layers.62.mlp.experts.126.gate_proj", "model.layers.62.mlp.experts.127.gate_proj", "model.layers.62.mlp.experts.128.gate_proj", "model.layers.62.mlp.experts.129.gate_proj", "model.layers.62.mlp.experts.130.gate_proj", "model.layers.62.mlp.experts.131.gate_proj", "model.layers.62.mlp.experts.132.gate_proj", "model.layers.62.mlp.experts.133.gate_proj", "model.layers.62.mlp.experts.134.gate_proj", "model.layers.62.mlp.experts.135.gate_proj", "model.layers.62.mlp.experts.136.gate_proj", "model.layers.62.mlp.experts.137.gate_proj", "model.layers.62.mlp.experts.138.gate_proj", "model.layers.62.mlp.experts.139.gate_proj", "model.layers.62.mlp.experts.140.gate_proj", "model.layers.62.mlp.experts.141.gate_proj", "model.layers.62.mlp.experts.142.gate_proj", "model.layers.62.mlp.experts.143.gate_proj", "model.layers.62.mlp.experts.144.gate_proj", "model.layers.62.mlp.experts.145.gate_proj", "model.layers.62.mlp.experts.146.gate_proj", "model.layers.62.mlp.experts.147.gate_proj", "model.layers.62.mlp.experts.148.gate_proj", "model.layers.62.mlp.experts.149.gate_proj", "model.layers.62.mlp.experts.150.gate_proj", "model.layers.62.mlp.experts.151.gate_proj", "model.layers.62.mlp.experts.152.gate_proj", "model.layers.62.mlp.experts.153.gate_proj", "model.layers.62.mlp.experts.154.gate_proj", "model.layers.62.mlp.experts.155.gate_proj", "model.layers.62.mlp.experts.156.gate_proj", "model.layers.62.mlp.experts.157.gate_proj", "model.layers.62.mlp.experts.158.gate_proj", "model.layers.62.mlp.experts.159.gate_proj", "model.layers.62.mlp.experts.0.up_proj", "model.layers.62.mlp.experts.1.up_proj", "model.layers.62.mlp.experts.2.up_proj", "model.layers.62.mlp.experts.3.up_proj", "model.layers.62.mlp.experts.4.up_proj", "model.layers.62.mlp.experts.5.up_proj", "model.layers.62.mlp.experts.6.up_proj", "model.layers.62.mlp.experts.7.up_proj", "model.layers.62.mlp.experts.8.up_proj", "model.layers.62.mlp.experts.9.up_proj", "model.layers.62.mlp.experts.10.up_proj", "model.layers.62.mlp.experts.11.up_proj", "model.layers.62.mlp.experts.12.up_proj", "model.layers.62.mlp.experts.13.up_proj", "model.layers.62.mlp.experts.14.up_proj", "model.layers.62.mlp.experts.15.up_proj", "model.layers.62.mlp.experts.16.up_proj", "model.layers.62.mlp.experts.17.up_proj", "model.layers.62.mlp.experts.18.up_proj", "model.layers.62.mlp.experts.19.up_proj", "model.layers.62.mlp.experts.20.up_proj", "model.layers.62.mlp.experts.21.up_proj", "model.layers.62.mlp.experts.22.up_proj", "model.layers.62.mlp.experts.23.up_proj", "model.layers.62.mlp.experts.24.up_proj", "model.layers.62.mlp.experts.25.up_proj", "model.layers.62.mlp.experts.26.up_proj", "model.layers.62.mlp.experts.27.up_proj", "model.layers.62.mlp.experts.28.up_proj", "model.layers.62.mlp.experts.29.up_proj", "model.layers.62.mlp.experts.30.up_proj", "model.layers.62.mlp.experts.31.up_proj", "model.layers.62.mlp.experts.32.up_proj", "model.layers.62.mlp.experts.33.up_proj", "model.layers.62.mlp.experts.34.up_proj", "model.layers.62.mlp.experts.35.up_proj", "model.layers.62.mlp.experts.36.up_proj", "model.layers.62.mlp.experts.37.up_proj", "model.layers.62.mlp.experts.38.up_proj", "model.layers.62.mlp.experts.39.up_proj", "model.layers.62.mlp.experts.40.up_proj", "model.layers.62.mlp.experts.41.up_proj", "model.layers.62.mlp.experts.42.up_proj", "model.layers.62.mlp.experts.43.up_proj", "model.layers.62.mlp.experts.44.up_proj", "model.layers.62.mlp.experts.45.up_proj", "model.layers.62.mlp.experts.46.up_proj", "model.layers.62.mlp.experts.47.up_proj", "model.layers.62.mlp.experts.48.up_proj", "model.layers.62.mlp.experts.49.up_proj", "model.layers.62.mlp.experts.50.up_proj", "model.layers.62.mlp.experts.51.up_proj", "model.layers.62.mlp.experts.52.up_proj", "model.layers.62.mlp.experts.53.up_proj", "model.layers.62.mlp.experts.54.up_proj", "model.layers.62.mlp.experts.55.up_proj", "model.layers.62.mlp.experts.56.up_proj", "model.layers.62.mlp.experts.57.up_proj", "model.layers.62.mlp.experts.58.up_proj", "model.layers.62.mlp.experts.59.up_proj", "model.layers.62.mlp.experts.60.up_proj", "model.layers.62.mlp.experts.61.up_proj", "model.layers.62.mlp.experts.62.up_proj", "model.layers.62.mlp.experts.63.up_proj", "model.layers.62.mlp.experts.64.up_proj", "model.layers.62.mlp.experts.65.up_proj", "model.layers.62.mlp.experts.66.up_proj", "model.layers.62.mlp.experts.67.up_proj", "model.layers.62.mlp.experts.68.up_proj", "model.layers.62.mlp.experts.69.up_proj", "model.layers.62.mlp.experts.70.up_proj", "model.layers.62.mlp.experts.71.up_proj", "model.layers.62.mlp.experts.72.up_proj", "model.layers.62.mlp.experts.73.up_proj", "model.layers.62.mlp.experts.74.up_proj", "model.layers.62.mlp.experts.75.up_proj", "model.layers.62.mlp.experts.76.up_proj", "model.layers.62.mlp.experts.77.up_proj", "model.layers.62.mlp.experts.78.up_proj", "model.layers.62.mlp.experts.79.up_proj", "model.layers.62.mlp.experts.80.up_proj", "model.layers.62.mlp.experts.81.up_proj", "model.layers.62.mlp.experts.82.up_proj", "model.layers.62.mlp.experts.83.up_proj", "model.layers.62.mlp.experts.84.up_proj", "model.layers.62.mlp.experts.85.up_proj", "model.layers.62.mlp.experts.86.up_proj", "model.layers.62.mlp.experts.87.up_proj", "model.layers.62.mlp.experts.88.up_proj", "model.layers.62.mlp.experts.89.up_proj", "model.layers.62.mlp.experts.90.up_proj", "model.layers.62.mlp.experts.91.up_proj", "model.layers.62.mlp.experts.92.up_proj", "model.layers.62.mlp.experts.93.up_proj", "model.layers.62.mlp.experts.94.up_proj", "model.layers.62.mlp.experts.95.up_proj", "model.layers.62.mlp.experts.96.up_proj", "model.layers.62.mlp.experts.97.up_proj", "model.layers.62.mlp.experts.98.up_proj", "model.layers.62.mlp.experts.99.up_proj", "model.layers.62.mlp.experts.100.up_proj", "model.layers.62.mlp.experts.101.up_proj", "model.layers.62.mlp.experts.102.up_proj", "model.layers.62.mlp.experts.103.up_proj", "model.layers.62.mlp.experts.104.up_proj", "model.layers.62.mlp.experts.105.up_proj", "model.layers.62.mlp.experts.106.up_proj", "model.layers.62.mlp.experts.107.up_proj", "model.layers.62.mlp.experts.108.up_proj", "model.layers.62.mlp.experts.109.up_proj", "model.layers.62.mlp.experts.110.up_proj", "model.layers.62.mlp.experts.111.up_proj", "model.layers.62.mlp.experts.112.up_proj", "model.layers.62.mlp.experts.113.up_proj", "model.layers.62.mlp.experts.114.up_proj", "model.layers.62.mlp.experts.115.up_proj", "model.layers.62.mlp.experts.116.up_proj", "model.layers.62.mlp.experts.117.up_proj", "model.layers.62.mlp.experts.118.up_proj", "model.layers.62.mlp.experts.119.up_proj", "model.layers.62.mlp.experts.120.up_proj", "model.layers.62.mlp.experts.121.up_proj", "model.layers.62.mlp.experts.122.up_proj", "model.layers.62.mlp.experts.123.up_proj", "model.layers.62.mlp.experts.124.up_proj", "model.layers.62.mlp.experts.125.up_proj", "model.layers.62.mlp.experts.126.up_proj", "model.layers.62.mlp.experts.127.up_proj", "model.layers.62.mlp.experts.128.up_proj", "model.layers.62.mlp.experts.129.up_proj", "model.layers.62.mlp.experts.130.up_proj", "model.layers.62.mlp.experts.131.up_proj", "model.layers.62.mlp.experts.132.up_proj", "model.layers.62.mlp.experts.133.up_proj", "model.layers.62.mlp.experts.134.up_proj", "model.layers.62.mlp.experts.135.up_proj", "model.layers.62.mlp.experts.136.up_proj", "model.layers.62.mlp.experts.137.up_proj", "model.layers.62.mlp.experts.138.up_proj", "model.layers.62.mlp.experts.139.up_proj", "model.layers.62.mlp.experts.140.up_proj", "model.layers.62.mlp.experts.141.up_proj", "model.layers.62.mlp.experts.142.up_proj", "model.layers.62.mlp.experts.143.up_proj", "model.layers.62.mlp.experts.144.up_proj", "model.layers.62.mlp.experts.145.up_proj", "model.layers.62.mlp.experts.146.up_proj", "model.layers.62.mlp.experts.147.up_proj", "model.layers.62.mlp.experts.148.up_proj", "model.layers.62.mlp.experts.149.up_proj", "model.layers.62.mlp.experts.150.up_proj", "model.layers.62.mlp.experts.151.up_proj", "model.layers.62.mlp.experts.152.up_proj", "model.layers.62.mlp.experts.153.up_proj", "model.layers.62.mlp.experts.154.up_proj", "model.layers.62.mlp.experts.155.up_proj", "model.layers.62.mlp.experts.156.up_proj", "model.layers.62.mlp.experts.157.up_proj", "model.layers.62.mlp.experts.158.up_proj", "model.layers.62.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0005050912499428017, "dbits": 2516582400 } ] }, { "idx": 374, "layers": [ "model.layers.62.mlp.experts.0.down_proj", "model.layers.62.mlp.experts.1.down_proj", "model.layers.62.mlp.experts.2.down_proj", "model.layers.62.mlp.experts.3.down_proj", "model.layers.62.mlp.experts.4.down_proj", "model.layers.62.mlp.experts.5.down_proj", "model.layers.62.mlp.experts.6.down_proj", "model.layers.62.mlp.experts.7.down_proj", "model.layers.62.mlp.experts.8.down_proj", "model.layers.62.mlp.experts.9.down_proj", "model.layers.62.mlp.experts.10.down_proj", "model.layers.62.mlp.experts.11.down_proj", "model.layers.62.mlp.experts.12.down_proj", "model.layers.62.mlp.experts.13.down_proj", "model.layers.62.mlp.experts.14.down_proj", "model.layers.62.mlp.experts.15.down_proj", "model.layers.62.mlp.experts.16.down_proj", "model.layers.62.mlp.experts.17.down_proj", "model.layers.62.mlp.experts.18.down_proj", "model.layers.62.mlp.experts.19.down_proj", "model.layers.62.mlp.experts.20.down_proj", "model.layers.62.mlp.experts.21.down_proj", "model.layers.62.mlp.experts.22.down_proj", "model.layers.62.mlp.experts.23.down_proj", "model.layers.62.mlp.experts.24.down_proj", "model.layers.62.mlp.experts.25.down_proj", "model.layers.62.mlp.experts.26.down_proj", "model.layers.62.mlp.experts.27.down_proj", "model.layers.62.mlp.experts.28.down_proj", "model.layers.62.mlp.experts.29.down_proj", "model.layers.62.mlp.experts.30.down_proj", "model.layers.62.mlp.experts.31.down_proj", "model.layers.62.mlp.experts.32.down_proj", "model.layers.62.mlp.experts.33.down_proj", "model.layers.62.mlp.experts.34.down_proj", "model.layers.62.mlp.experts.35.down_proj", "model.layers.62.mlp.experts.36.down_proj", "model.layers.62.mlp.experts.37.down_proj", "model.layers.62.mlp.experts.38.down_proj", "model.layers.62.mlp.experts.39.down_proj", "model.layers.62.mlp.experts.40.down_proj", "model.layers.62.mlp.experts.41.down_proj", "model.layers.62.mlp.experts.42.down_proj", "model.layers.62.mlp.experts.43.down_proj", "model.layers.62.mlp.experts.44.down_proj", "model.layers.62.mlp.experts.45.down_proj", "model.layers.62.mlp.experts.46.down_proj", "model.layers.62.mlp.experts.47.down_proj", "model.layers.62.mlp.experts.48.down_proj", "model.layers.62.mlp.experts.49.down_proj", "model.layers.62.mlp.experts.50.down_proj", "model.layers.62.mlp.experts.51.down_proj", "model.layers.62.mlp.experts.52.down_proj", "model.layers.62.mlp.experts.53.down_proj", "model.layers.62.mlp.experts.54.down_proj", "model.layers.62.mlp.experts.55.down_proj", "model.layers.62.mlp.experts.56.down_proj", "model.layers.62.mlp.experts.57.down_proj", "model.layers.62.mlp.experts.58.down_proj", "model.layers.62.mlp.experts.59.down_proj", "model.layers.62.mlp.experts.60.down_proj", "model.layers.62.mlp.experts.61.down_proj", "model.layers.62.mlp.experts.62.down_proj", "model.layers.62.mlp.experts.63.down_proj", "model.layers.62.mlp.experts.64.down_proj", "model.layers.62.mlp.experts.65.down_proj", "model.layers.62.mlp.experts.66.down_proj", "model.layers.62.mlp.experts.67.down_proj", "model.layers.62.mlp.experts.68.down_proj", "model.layers.62.mlp.experts.69.down_proj", "model.layers.62.mlp.experts.70.down_proj", "model.layers.62.mlp.experts.71.down_proj", "model.layers.62.mlp.experts.72.down_proj", "model.layers.62.mlp.experts.73.down_proj", "model.layers.62.mlp.experts.74.down_proj", "model.layers.62.mlp.experts.75.down_proj", "model.layers.62.mlp.experts.76.down_proj", "model.layers.62.mlp.experts.77.down_proj", "model.layers.62.mlp.experts.78.down_proj", "model.layers.62.mlp.experts.79.down_proj", "model.layers.62.mlp.experts.80.down_proj", "model.layers.62.mlp.experts.81.down_proj", "model.layers.62.mlp.experts.82.down_proj", "model.layers.62.mlp.experts.83.down_proj", "model.layers.62.mlp.experts.84.down_proj", "model.layers.62.mlp.experts.85.down_proj", "model.layers.62.mlp.experts.86.down_proj", "model.layers.62.mlp.experts.87.down_proj", "model.layers.62.mlp.experts.88.down_proj", "model.layers.62.mlp.experts.89.down_proj", "model.layers.62.mlp.experts.90.down_proj", "model.layers.62.mlp.experts.91.down_proj", "model.layers.62.mlp.experts.92.down_proj", "model.layers.62.mlp.experts.93.down_proj", "model.layers.62.mlp.experts.94.down_proj", "model.layers.62.mlp.experts.95.down_proj", "model.layers.62.mlp.experts.96.down_proj", "model.layers.62.mlp.experts.97.down_proj", "model.layers.62.mlp.experts.98.down_proj", "model.layers.62.mlp.experts.99.down_proj", "model.layers.62.mlp.experts.100.down_proj", "model.layers.62.mlp.experts.101.down_proj", "model.layers.62.mlp.experts.102.down_proj", "model.layers.62.mlp.experts.103.down_proj", "model.layers.62.mlp.experts.104.down_proj", "model.layers.62.mlp.experts.105.down_proj", "model.layers.62.mlp.experts.106.down_proj", "model.layers.62.mlp.experts.107.down_proj", "model.layers.62.mlp.experts.108.down_proj", "model.layers.62.mlp.experts.109.down_proj", "model.layers.62.mlp.experts.110.down_proj", "model.layers.62.mlp.experts.111.down_proj", "model.layers.62.mlp.experts.112.down_proj", "model.layers.62.mlp.experts.113.down_proj", "model.layers.62.mlp.experts.114.down_proj", "model.layers.62.mlp.experts.115.down_proj", "model.layers.62.mlp.experts.116.down_proj", "model.layers.62.mlp.experts.117.down_proj", "model.layers.62.mlp.experts.118.down_proj", "model.layers.62.mlp.experts.119.down_proj", "model.layers.62.mlp.experts.120.down_proj", "model.layers.62.mlp.experts.121.down_proj", "model.layers.62.mlp.experts.122.down_proj", "model.layers.62.mlp.experts.123.down_proj", "model.layers.62.mlp.experts.124.down_proj", "model.layers.62.mlp.experts.125.down_proj", "model.layers.62.mlp.experts.126.down_proj", "model.layers.62.mlp.experts.127.down_proj", "model.layers.62.mlp.experts.128.down_proj", "model.layers.62.mlp.experts.129.down_proj", "model.layers.62.mlp.experts.130.down_proj", "model.layers.62.mlp.experts.131.down_proj", "model.layers.62.mlp.experts.132.down_proj", "model.layers.62.mlp.experts.133.down_proj", "model.layers.62.mlp.experts.134.down_proj", "model.layers.62.mlp.experts.135.down_proj", "model.layers.62.mlp.experts.136.down_proj", "model.layers.62.mlp.experts.137.down_proj", "model.layers.62.mlp.experts.138.down_proj", "model.layers.62.mlp.experts.139.down_proj", "model.layers.62.mlp.experts.140.down_proj", "model.layers.62.mlp.experts.141.down_proj", "model.layers.62.mlp.experts.142.down_proj", "model.layers.62.mlp.experts.143.down_proj", "model.layers.62.mlp.experts.144.down_proj", "model.layers.62.mlp.experts.145.down_proj", "model.layers.62.mlp.experts.146.down_proj", "model.layers.62.mlp.experts.147.down_proj", "model.layers.62.mlp.experts.148.down_proj", "model.layers.62.mlp.experts.149.down_proj", "model.layers.62.mlp.experts.150.down_proj", "model.layers.62.mlp.experts.151.down_proj", "model.layers.62.mlp.experts.152.down_proj", "model.layers.62.mlp.experts.153.down_proj", "model.layers.62.mlp.experts.154.down_proj", "model.layers.62.mlp.experts.155.down_proj", "model.layers.62.mlp.experts.156.down_proj", "model.layers.62.mlp.experts.157.down_proj", "model.layers.62.mlp.experts.158.down_proj", "model.layers.62.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0005263462662696838, "dbits": 1258291200 } ] }, { "idx": 375, "layers": [ "model.layers.63.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0005340576171875, "dbits": 62914560 } ] }, { "idx": 376, "layers": [ "model.layers.63.self_attn.k_proj", "model.layers.63.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0005402654409408791, "dbits": 10485760 } ] }, { "idx": 377, "layers": [ "model.layers.63.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0006351426243781932, "dbits": 62914560 } ] }, { "idx": 378, "layers": [ "model.layers.63.mlp.shared_experts.gate_proj", "model.layers.63.mlp.shared_experts.up_proj", "model.layers.63.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0009195722639560588, "dbits": 23592960 } ] }, { "idx": 379, "layers": [ "model.layers.63.mlp.experts.0.gate_proj", "model.layers.63.mlp.experts.1.gate_proj", "model.layers.63.mlp.experts.2.gate_proj", "model.layers.63.mlp.experts.3.gate_proj", "model.layers.63.mlp.experts.4.gate_proj", "model.layers.63.mlp.experts.5.gate_proj", "model.layers.63.mlp.experts.6.gate_proj", "model.layers.63.mlp.experts.7.gate_proj", "model.layers.63.mlp.experts.8.gate_proj", "model.layers.63.mlp.experts.9.gate_proj", "model.layers.63.mlp.experts.10.gate_proj", "model.layers.63.mlp.experts.11.gate_proj", "model.layers.63.mlp.experts.12.gate_proj", "model.layers.63.mlp.experts.13.gate_proj", "model.layers.63.mlp.experts.14.gate_proj", "model.layers.63.mlp.experts.15.gate_proj", "model.layers.63.mlp.experts.16.gate_proj", "model.layers.63.mlp.experts.17.gate_proj", "model.layers.63.mlp.experts.18.gate_proj", "model.layers.63.mlp.experts.19.gate_proj", "model.layers.63.mlp.experts.20.gate_proj", "model.layers.63.mlp.experts.21.gate_proj", "model.layers.63.mlp.experts.22.gate_proj", "model.layers.63.mlp.experts.23.gate_proj", "model.layers.63.mlp.experts.24.gate_proj", "model.layers.63.mlp.experts.25.gate_proj", "model.layers.63.mlp.experts.26.gate_proj", "model.layers.63.mlp.experts.27.gate_proj", "model.layers.63.mlp.experts.28.gate_proj", "model.layers.63.mlp.experts.29.gate_proj", "model.layers.63.mlp.experts.30.gate_proj", "model.layers.63.mlp.experts.31.gate_proj", "model.layers.63.mlp.experts.32.gate_proj", "model.layers.63.mlp.experts.33.gate_proj", "model.layers.63.mlp.experts.34.gate_proj", "model.layers.63.mlp.experts.35.gate_proj", "model.layers.63.mlp.experts.36.gate_proj", "model.layers.63.mlp.experts.37.gate_proj", "model.layers.63.mlp.experts.38.gate_proj", "model.layers.63.mlp.experts.39.gate_proj", "model.layers.63.mlp.experts.40.gate_proj", "model.layers.63.mlp.experts.41.gate_proj", "model.layers.63.mlp.experts.42.gate_proj", "model.layers.63.mlp.experts.43.gate_proj", "model.layers.63.mlp.experts.44.gate_proj", "model.layers.63.mlp.experts.45.gate_proj", "model.layers.63.mlp.experts.46.gate_proj", "model.layers.63.mlp.experts.47.gate_proj", "model.layers.63.mlp.experts.48.gate_proj", "model.layers.63.mlp.experts.49.gate_proj", "model.layers.63.mlp.experts.50.gate_proj", "model.layers.63.mlp.experts.51.gate_proj", "model.layers.63.mlp.experts.52.gate_proj", "model.layers.63.mlp.experts.53.gate_proj", "model.layers.63.mlp.experts.54.gate_proj", "model.layers.63.mlp.experts.55.gate_proj", "model.layers.63.mlp.experts.56.gate_proj", "model.layers.63.mlp.experts.57.gate_proj", "model.layers.63.mlp.experts.58.gate_proj", "model.layers.63.mlp.experts.59.gate_proj", "model.layers.63.mlp.experts.60.gate_proj", "model.layers.63.mlp.experts.61.gate_proj", "model.layers.63.mlp.experts.62.gate_proj", "model.layers.63.mlp.experts.63.gate_proj", "model.layers.63.mlp.experts.64.gate_proj", "model.layers.63.mlp.experts.65.gate_proj", "model.layers.63.mlp.experts.66.gate_proj", "model.layers.63.mlp.experts.67.gate_proj", "model.layers.63.mlp.experts.68.gate_proj", "model.layers.63.mlp.experts.69.gate_proj", "model.layers.63.mlp.experts.70.gate_proj", "model.layers.63.mlp.experts.71.gate_proj", "model.layers.63.mlp.experts.72.gate_proj", "model.layers.63.mlp.experts.73.gate_proj", "model.layers.63.mlp.experts.74.gate_proj", "model.layers.63.mlp.experts.75.gate_proj", "model.layers.63.mlp.experts.76.gate_proj", "model.layers.63.mlp.experts.77.gate_proj", "model.layers.63.mlp.experts.78.gate_proj", "model.layers.63.mlp.experts.79.gate_proj", "model.layers.63.mlp.experts.80.gate_proj", "model.layers.63.mlp.experts.81.gate_proj", "model.layers.63.mlp.experts.82.gate_proj", "model.layers.63.mlp.experts.83.gate_proj", "model.layers.63.mlp.experts.84.gate_proj", "model.layers.63.mlp.experts.85.gate_proj", "model.layers.63.mlp.experts.86.gate_proj", "model.layers.63.mlp.experts.87.gate_proj", "model.layers.63.mlp.experts.88.gate_proj", "model.layers.63.mlp.experts.89.gate_proj", "model.layers.63.mlp.experts.90.gate_proj", "model.layers.63.mlp.experts.91.gate_proj", "model.layers.63.mlp.experts.92.gate_proj", "model.layers.63.mlp.experts.93.gate_proj", "model.layers.63.mlp.experts.94.gate_proj", "model.layers.63.mlp.experts.95.gate_proj", "model.layers.63.mlp.experts.96.gate_proj", "model.layers.63.mlp.experts.97.gate_proj", "model.layers.63.mlp.experts.98.gate_proj", "model.layers.63.mlp.experts.99.gate_proj", "model.layers.63.mlp.experts.100.gate_proj", "model.layers.63.mlp.experts.101.gate_proj", "model.layers.63.mlp.experts.102.gate_proj", "model.layers.63.mlp.experts.103.gate_proj", "model.layers.63.mlp.experts.104.gate_proj", "model.layers.63.mlp.experts.105.gate_proj", "model.layers.63.mlp.experts.106.gate_proj", "model.layers.63.mlp.experts.107.gate_proj", "model.layers.63.mlp.experts.108.gate_proj", "model.layers.63.mlp.experts.109.gate_proj", "model.layers.63.mlp.experts.110.gate_proj", "model.layers.63.mlp.experts.111.gate_proj", "model.layers.63.mlp.experts.112.gate_proj", "model.layers.63.mlp.experts.113.gate_proj", "model.layers.63.mlp.experts.114.gate_proj", "model.layers.63.mlp.experts.115.gate_proj", "model.layers.63.mlp.experts.116.gate_proj", "model.layers.63.mlp.experts.117.gate_proj", "model.layers.63.mlp.experts.118.gate_proj", "model.layers.63.mlp.experts.119.gate_proj", "model.layers.63.mlp.experts.120.gate_proj", "model.layers.63.mlp.experts.121.gate_proj", "model.layers.63.mlp.experts.122.gate_proj", "model.layers.63.mlp.experts.123.gate_proj", "model.layers.63.mlp.experts.124.gate_proj", "model.layers.63.mlp.experts.125.gate_proj", "model.layers.63.mlp.experts.126.gate_proj", "model.layers.63.mlp.experts.127.gate_proj", "model.layers.63.mlp.experts.128.gate_proj", "model.layers.63.mlp.experts.129.gate_proj", "model.layers.63.mlp.experts.130.gate_proj", "model.layers.63.mlp.experts.131.gate_proj", "model.layers.63.mlp.experts.132.gate_proj", "model.layers.63.mlp.experts.133.gate_proj", "model.layers.63.mlp.experts.134.gate_proj", "model.layers.63.mlp.experts.135.gate_proj", "model.layers.63.mlp.experts.136.gate_proj", "model.layers.63.mlp.experts.137.gate_proj", "model.layers.63.mlp.experts.138.gate_proj", "model.layers.63.mlp.experts.139.gate_proj", "model.layers.63.mlp.experts.140.gate_proj", "model.layers.63.mlp.experts.141.gate_proj", "model.layers.63.mlp.experts.142.gate_proj", "model.layers.63.mlp.experts.143.gate_proj", "model.layers.63.mlp.experts.144.gate_proj", "model.layers.63.mlp.experts.145.gate_proj", "model.layers.63.mlp.experts.146.gate_proj", "model.layers.63.mlp.experts.147.gate_proj", "model.layers.63.mlp.experts.148.gate_proj", "model.layers.63.mlp.experts.149.gate_proj", "model.layers.63.mlp.experts.150.gate_proj", "model.layers.63.mlp.experts.151.gate_proj", "model.layers.63.mlp.experts.152.gate_proj", "model.layers.63.mlp.experts.153.gate_proj", "model.layers.63.mlp.experts.154.gate_proj", "model.layers.63.mlp.experts.155.gate_proj", "model.layers.63.mlp.experts.156.gate_proj", "model.layers.63.mlp.experts.157.gate_proj", "model.layers.63.mlp.experts.158.gate_proj", "model.layers.63.mlp.experts.159.gate_proj", "model.layers.63.mlp.experts.0.up_proj", "model.layers.63.mlp.experts.1.up_proj", "model.layers.63.mlp.experts.2.up_proj", "model.layers.63.mlp.experts.3.up_proj", "model.layers.63.mlp.experts.4.up_proj", "model.layers.63.mlp.experts.5.up_proj", "model.layers.63.mlp.experts.6.up_proj", "model.layers.63.mlp.experts.7.up_proj", "model.layers.63.mlp.experts.8.up_proj", "model.layers.63.mlp.experts.9.up_proj", "model.layers.63.mlp.experts.10.up_proj", "model.layers.63.mlp.experts.11.up_proj", "model.layers.63.mlp.experts.12.up_proj", "model.layers.63.mlp.experts.13.up_proj", "model.layers.63.mlp.experts.14.up_proj", "model.layers.63.mlp.experts.15.up_proj", "model.layers.63.mlp.experts.16.up_proj", "model.layers.63.mlp.experts.17.up_proj", "model.layers.63.mlp.experts.18.up_proj", "model.layers.63.mlp.experts.19.up_proj", "model.layers.63.mlp.experts.20.up_proj", "model.layers.63.mlp.experts.21.up_proj", "model.layers.63.mlp.experts.22.up_proj", "model.layers.63.mlp.experts.23.up_proj", "model.layers.63.mlp.experts.24.up_proj", "model.layers.63.mlp.experts.25.up_proj", "model.layers.63.mlp.experts.26.up_proj", "model.layers.63.mlp.experts.27.up_proj", "model.layers.63.mlp.experts.28.up_proj", "model.layers.63.mlp.experts.29.up_proj", "model.layers.63.mlp.experts.30.up_proj", "model.layers.63.mlp.experts.31.up_proj", "model.layers.63.mlp.experts.32.up_proj", "model.layers.63.mlp.experts.33.up_proj", "model.layers.63.mlp.experts.34.up_proj", "model.layers.63.mlp.experts.35.up_proj", "model.layers.63.mlp.experts.36.up_proj", "model.layers.63.mlp.experts.37.up_proj", "model.layers.63.mlp.experts.38.up_proj", "model.layers.63.mlp.experts.39.up_proj", "model.layers.63.mlp.experts.40.up_proj", "model.layers.63.mlp.experts.41.up_proj", "model.layers.63.mlp.experts.42.up_proj", "model.layers.63.mlp.experts.43.up_proj", "model.layers.63.mlp.experts.44.up_proj", "model.layers.63.mlp.experts.45.up_proj", "model.layers.63.mlp.experts.46.up_proj", "model.layers.63.mlp.experts.47.up_proj", "model.layers.63.mlp.experts.48.up_proj", "model.layers.63.mlp.experts.49.up_proj", "model.layers.63.mlp.experts.50.up_proj", "model.layers.63.mlp.experts.51.up_proj", "model.layers.63.mlp.experts.52.up_proj", "model.layers.63.mlp.experts.53.up_proj", "model.layers.63.mlp.experts.54.up_proj", "model.layers.63.mlp.experts.55.up_proj", "model.layers.63.mlp.experts.56.up_proj", "model.layers.63.mlp.experts.57.up_proj", "model.layers.63.mlp.experts.58.up_proj", "model.layers.63.mlp.experts.59.up_proj", "model.layers.63.mlp.experts.60.up_proj", "model.layers.63.mlp.experts.61.up_proj", "model.layers.63.mlp.experts.62.up_proj", "model.layers.63.mlp.experts.63.up_proj", "model.layers.63.mlp.experts.64.up_proj", "model.layers.63.mlp.experts.65.up_proj", "model.layers.63.mlp.experts.66.up_proj", "model.layers.63.mlp.experts.67.up_proj", "model.layers.63.mlp.experts.68.up_proj", "model.layers.63.mlp.experts.69.up_proj", "model.layers.63.mlp.experts.70.up_proj", "model.layers.63.mlp.experts.71.up_proj", "model.layers.63.mlp.experts.72.up_proj", "model.layers.63.mlp.experts.73.up_proj", "model.layers.63.mlp.experts.74.up_proj", "model.layers.63.mlp.experts.75.up_proj", "model.layers.63.mlp.experts.76.up_proj", "model.layers.63.mlp.experts.77.up_proj", "model.layers.63.mlp.experts.78.up_proj", "model.layers.63.mlp.experts.79.up_proj", "model.layers.63.mlp.experts.80.up_proj", "model.layers.63.mlp.experts.81.up_proj", "model.layers.63.mlp.experts.82.up_proj", "model.layers.63.mlp.experts.83.up_proj", "model.layers.63.mlp.experts.84.up_proj", "model.layers.63.mlp.experts.85.up_proj", "model.layers.63.mlp.experts.86.up_proj", "model.layers.63.mlp.experts.87.up_proj", "model.layers.63.mlp.experts.88.up_proj", "model.layers.63.mlp.experts.89.up_proj", "model.layers.63.mlp.experts.90.up_proj", "model.layers.63.mlp.experts.91.up_proj", "model.layers.63.mlp.experts.92.up_proj", "model.layers.63.mlp.experts.93.up_proj", "model.layers.63.mlp.experts.94.up_proj", "model.layers.63.mlp.experts.95.up_proj", "model.layers.63.mlp.experts.96.up_proj", "model.layers.63.mlp.experts.97.up_proj", "model.layers.63.mlp.experts.98.up_proj", "model.layers.63.mlp.experts.99.up_proj", "model.layers.63.mlp.experts.100.up_proj", "model.layers.63.mlp.experts.101.up_proj", "model.layers.63.mlp.experts.102.up_proj", "model.layers.63.mlp.experts.103.up_proj", "model.layers.63.mlp.experts.104.up_proj", "model.layers.63.mlp.experts.105.up_proj", "model.layers.63.mlp.experts.106.up_proj", "model.layers.63.mlp.experts.107.up_proj", "model.layers.63.mlp.experts.108.up_proj", "model.layers.63.mlp.experts.109.up_proj", "model.layers.63.mlp.experts.110.up_proj", "model.layers.63.mlp.experts.111.up_proj", "model.layers.63.mlp.experts.112.up_proj", "model.layers.63.mlp.experts.113.up_proj", "model.layers.63.mlp.experts.114.up_proj", "model.layers.63.mlp.experts.115.up_proj", "model.layers.63.mlp.experts.116.up_proj", "model.layers.63.mlp.experts.117.up_proj", "model.layers.63.mlp.experts.118.up_proj", "model.layers.63.mlp.experts.119.up_proj", "model.layers.63.mlp.experts.120.up_proj", "model.layers.63.mlp.experts.121.up_proj", "model.layers.63.mlp.experts.122.up_proj", "model.layers.63.mlp.experts.123.up_proj", "model.layers.63.mlp.experts.124.up_proj", "model.layers.63.mlp.experts.125.up_proj", "model.layers.63.mlp.experts.126.up_proj", "model.layers.63.mlp.experts.127.up_proj", "model.layers.63.mlp.experts.128.up_proj", "model.layers.63.mlp.experts.129.up_proj", "model.layers.63.mlp.experts.130.up_proj", "model.layers.63.mlp.experts.131.up_proj", "model.layers.63.mlp.experts.132.up_proj", "model.layers.63.mlp.experts.133.up_proj", "model.layers.63.mlp.experts.134.up_proj", "model.layers.63.mlp.experts.135.up_proj", "model.layers.63.mlp.experts.136.up_proj", "model.layers.63.mlp.experts.137.up_proj", "model.layers.63.mlp.experts.138.up_proj", "model.layers.63.mlp.experts.139.up_proj", "model.layers.63.mlp.experts.140.up_proj", "model.layers.63.mlp.experts.141.up_proj", "model.layers.63.mlp.experts.142.up_proj", "model.layers.63.mlp.experts.143.up_proj", "model.layers.63.mlp.experts.144.up_proj", "model.layers.63.mlp.experts.145.up_proj", "model.layers.63.mlp.experts.146.up_proj", "model.layers.63.mlp.experts.147.up_proj", "model.layers.63.mlp.experts.148.up_proj", "model.layers.63.mlp.experts.149.up_proj", "model.layers.63.mlp.experts.150.up_proj", "model.layers.63.mlp.experts.151.up_proj", "model.layers.63.mlp.experts.152.up_proj", "model.layers.63.mlp.experts.153.up_proj", "model.layers.63.mlp.experts.154.up_proj", "model.layers.63.mlp.experts.155.up_proj", "model.layers.63.mlp.experts.156.up_proj", "model.layers.63.mlp.experts.157.up_proj", "model.layers.63.mlp.experts.158.up_proj", "model.layers.63.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0005719877779483684, "dbits": 2516582400 } ] }, { "idx": 380, "layers": [ "model.layers.63.mlp.experts.0.down_proj", "model.layers.63.mlp.experts.1.down_proj", "model.layers.63.mlp.experts.2.down_proj", "model.layers.63.mlp.experts.3.down_proj", "model.layers.63.mlp.experts.4.down_proj", "model.layers.63.mlp.experts.5.down_proj", "model.layers.63.mlp.experts.6.down_proj", "model.layers.63.mlp.experts.7.down_proj", "model.layers.63.mlp.experts.8.down_proj", "model.layers.63.mlp.experts.9.down_proj", "model.layers.63.mlp.experts.10.down_proj", "model.layers.63.mlp.experts.11.down_proj", "model.layers.63.mlp.experts.12.down_proj", "model.layers.63.mlp.experts.13.down_proj", "model.layers.63.mlp.experts.14.down_proj", "model.layers.63.mlp.experts.15.down_proj", "model.layers.63.mlp.experts.16.down_proj", "model.layers.63.mlp.experts.17.down_proj", "model.layers.63.mlp.experts.18.down_proj", "model.layers.63.mlp.experts.19.down_proj", "model.layers.63.mlp.experts.20.down_proj", "model.layers.63.mlp.experts.21.down_proj", "model.layers.63.mlp.experts.22.down_proj", "model.layers.63.mlp.experts.23.down_proj", "model.layers.63.mlp.experts.24.down_proj", "model.layers.63.mlp.experts.25.down_proj", "model.layers.63.mlp.experts.26.down_proj", "model.layers.63.mlp.experts.27.down_proj", "model.layers.63.mlp.experts.28.down_proj", "model.layers.63.mlp.experts.29.down_proj", "model.layers.63.mlp.experts.30.down_proj", "model.layers.63.mlp.experts.31.down_proj", "model.layers.63.mlp.experts.32.down_proj", "model.layers.63.mlp.experts.33.down_proj", "model.layers.63.mlp.experts.34.down_proj", "model.layers.63.mlp.experts.35.down_proj", "model.layers.63.mlp.experts.36.down_proj", "model.layers.63.mlp.experts.37.down_proj", "model.layers.63.mlp.experts.38.down_proj", "model.layers.63.mlp.experts.39.down_proj", "model.layers.63.mlp.experts.40.down_proj", "model.layers.63.mlp.experts.41.down_proj", "model.layers.63.mlp.experts.42.down_proj", "model.layers.63.mlp.experts.43.down_proj", "model.layers.63.mlp.experts.44.down_proj", "model.layers.63.mlp.experts.45.down_proj", "model.layers.63.mlp.experts.46.down_proj", "model.layers.63.mlp.experts.47.down_proj", "model.layers.63.mlp.experts.48.down_proj", "model.layers.63.mlp.experts.49.down_proj", "model.layers.63.mlp.experts.50.down_proj", "model.layers.63.mlp.experts.51.down_proj", "model.layers.63.mlp.experts.52.down_proj", "model.layers.63.mlp.experts.53.down_proj", "model.layers.63.mlp.experts.54.down_proj", "model.layers.63.mlp.experts.55.down_proj", "model.layers.63.mlp.experts.56.down_proj", "model.layers.63.mlp.experts.57.down_proj", "model.layers.63.mlp.experts.58.down_proj", "model.layers.63.mlp.experts.59.down_proj", "model.layers.63.mlp.experts.60.down_proj", "model.layers.63.mlp.experts.61.down_proj", "model.layers.63.mlp.experts.62.down_proj", "model.layers.63.mlp.experts.63.down_proj", "model.layers.63.mlp.experts.64.down_proj", "model.layers.63.mlp.experts.65.down_proj", "model.layers.63.mlp.experts.66.down_proj", "model.layers.63.mlp.experts.67.down_proj", "model.layers.63.mlp.experts.68.down_proj", "model.layers.63.mlp.experts.69.down_proj", "model.layers.63.mlp.experts.70.down_proj", "model.layers.63.mlp.experts.71.down_proj", "model.layers.63.mlp.experts.72.down_proj", "model.layers.63.mlp.experts.73.down_proj", "model.layers.63.mlp.experts.74.down_proj", "model.layers.63.mlp.experts.75.down_proj", "model.layers.63.mlp.experts.76.down_proj", "model.layers.63.mlp.experts.77.down_proj", "model.layers.63.mlp.experts.78.down_proj", "model.layers.63.mlp.experts.79.down_proj", "model.layers.63.mlp.experts.80.down_proj", "model.layers.63.mlp.experts.81.down_proj", "model.layers.63.mlp.experts.82.down_proj", "model.layers.63.mlp.experts.83.down_proj", "model.layers.63.mlp.experts.84.down_proj", "model.layers.63.mlp.experts.85.down_proj", "model.layers.63.mlp.experts.86.down_proj", "model.layers.63.mlp.experts.87.down_proj", "model.layers.63.mlp.experts.88.down_proj", "model.layers.63.mlp.experts.89.down_proj", "model.layers.63.mlp.experts.90.down_proj", "model.layers.63.mlp.experts.91.down_proj", "model.layers.63.mlp.experts.92.down_proj", "model.layers.63.mlp.experts.93.down_proj", "model.layers.63.mlp.experts.94.down_proj", "model.layers.63.mlp.experts.95.down_proj", "model.layers.63.mlp.experts.96.down_proj", "model.layers.63.mlp.experts.97.down_proj", "model.layers.63.mlp.experts.98.down_proj", "model.layers.63.mlp.experts.99.down_proj", "model.layers.63.mlp.experts.100.down_proj", "model.layers.63.mlp.experts.101.down_proj", "model.layers.63.mlp.experts.102.down_proj", "model.layers.63.mlp.experts.103.down_proj", "model.layers.63.mlp.experts.104.down_proj", "model.layers.63.mlp.experts.105.down_proj", "model.layers.63.mlp.experts.106.down_proj", "model.layers.63.mlp.experts.107.down_proj", "model.layers.63.mlp.experts.108.down_proj", "model.layers.63.mlp.experts.109.down_proj", "model.layers.63.mlp.experts.110.down_proj", "model.layers.63.mlp.experts.111.down_proj", "model.layers.63.mlp.experts.112.down_proj", "model.layers.63.mlp.experts.113.down_proj", "model.layers.63.mlp.experts.114.down_proj", "model.layers.63.mlp.experts.115.down_proj", "model.layers.63.mlp.experts.116.down_proj", "model.layers.63.mlp.experts.117.down_proj", "model.layers.63.mlp.experts.118.down_proj", "model.layers.63.mlp.experts.119.down_proj", "model.layers.63.mlp.experts.120.down_proj", "model.layers.63.mlp.experts.121.down_proj", "model.layers.63.mlp.experts.122.down_proj", "model.layers.63.mlp.experts.123.down_proj", "model.layers.63.mlp.experts.124.down_proj", "model.layers.63.mlp.experts.125.down_proj", "model.layers.63.mlp.experts.126.down_proj", "model.layers.63.mlp.experts.127.down_proj", "model.layers.63.mlp.experts.128.down_proj", "model.layers.63.mlp.experts.129.down_proj", "model.layers.63.mlp.experts.130.down_proj", "model.layers.63.mlp.experts.131.down_proj", "model.layers.63.mlp.experts.132.down_proj", "model.layers.63.mlp.experts.133.down_proj", "model.layers.63.mlp.experts.134.down_proj", "model.layers.63.mlp.experts.135.down_proj", "model.layers.63.mlp.experts.136.down_proj", "model.layers.63.mlp.experts.137.down_proj", "model.layers.63.mlp.experts.138.down_proj", "model.layers.63.mlp.experts.139.down_proj", "model.layers.63.mlp.experts.140.down_proj", "model.layers.63.mlp.experts.141.down_proj", "model.layers.63.mlp.experts.142.down_proj", "model.layers.63.mlp.experts.143.down_proj", "model.layers.63.mlp.experts.144.down_proj", "model.layers.63.mlp.experts.145.down_proj", "model.layers.63.mlp.experts.146.down_proj", "model.layers.63.mlp.experts.147.down_proj", "model.layers.63.mlp.experts.148.down_proj", "model.layers.63.mlp.experts.149.down_proj", "model.layers.63.mlp.experts.150.down_proj", "model.layers.63.mlp.experts.151.down_proj", "model.layers.63.mlp.experts.152.down_proj", "model.layers.63.mlp.experts.153.down_proj", "model.layers.63.mlp.experts.154.down_proj", "model.layers.63.mlp.experts.155.down_proj", "model.layers.63.mlp.experts.156.down_proj", "model.layers.63.mlp.experts.157.down_proj", "model.layers.63.mlp.experts.158.down_proj", "model.layers.63.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0007122673094272503, "dbits": 1258291200 } ] }, { "idx": 381, "layers": [ "model.layers.64.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00046290531754494824, "dbits": 62914560 } ] }, { "idx": 382, "layers": [ "model.layers.64.self_attn.k_proj", "model.layers.64.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0004899151623248832, "dbits": 10485760 } ] }, { "idx": 383, "layers": [ "model.layers.64.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0001062646508216969, "dbits": 62914560 } ] }, { "idx": 384, "layers": [ "model.layers.64.mlp.shared_experts.gate_proj", "model.layers.64.mlp.shared_experts.up_proj", "model.layers.64.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0003522433340549247, "dbits": 23592960 } ] }, { "idx": 385, "layers": [ "model.layers.64.mlp.experts.0.gate_proj", "model.layers.64.mlp.experts.1.gate_proj", "model.layers.64.mlp.experts.2.gate_proj", "model.layers.64.mlp.experts.3.gate_proj", "model.layers.64.mlp.experts.4.gate_proj", "model.layers.64.mlp.experts.5.gate_proj", "model.layers.64.mlp.experts.6.gate_proj", "model.layers.64.mlp.experts.7.gate_proj", "model.layers.64.mlp.experts.8.gate_proj", "model.layers.64.mlp.experts.9.gate_proj", "model.layers.64.mlp.experts.10.gate_proj", "model.layers.64.mlp.experts.11.gate_proj", "model.layers.64.mlp.experts.12.gate_proj", "model.layers.64.mlp.experts.13.gate_proj", "model.layers.64.mlp.experts.14.gate_proj", "model.layers.64.mlp.experts.15.gate_proj", "model.layers.64.mlp.experts.16.gate_proj", "model.layers.64.mlp.experts.17.gate_proj", "model.layers.64.mlp.experts.18.gate_proj", "model.layers.64.mlp.experts.19.gate_proj", "model.layers.64.mlp.experts.20.gate_proj", "model.layers.64.mlp.experts.21.gate_proj", "model.layers.64.mlp.experts.22.gate_proj", "model.layers.64.mlp.experts.23.gate_proj", "model.layers.64.mlp.experts.24.gate_proj", "model.layers.64.mlp.experts.25.gate_proj", "model.layers.64.mlp.experts.26.gate_proj", "model.layers.64.mlp.experts.27.gate_proj", "model.layers.64.mlp.experts.28.gate_proj", "model.layers.64.mlp.experts.29.gate_proj", "model.layers.64.mlp.experts.30.gate_proj", "model.layers.64.mlp.experts.31.gate_proj", "model.layers.64.mlp.experts.32.gate_proj", "model.layers.64.mlp.experts.33.gate_proj", "model.layers.64.mlp.experts.34.gate_proj", "model.layers.64.mlp.experts.35.gate_proj", "model.layers.64.mlp.experts.36.gate_proj", "model.layers.64.mlp.experts.37.gate_proj", "model.layers.64.mlp.experts.38.gate_proj", "model.layers.64.mlp.experts.39.gate_proj", "model.layers.64.mlp.experts.40.gate_proj", "model.layers.64.mlp.experts.41.gate_proj", "model.layers.64.mlp.experts.42.gate_proj", "model.layers.64.mlp.experts.43.gate_proj", "model.layers.64.mlp.experts.44.gate_proj", "model.layers.64.mlp.experts.45.gate_proj", "model.layers.64.mlp.experts.46.gate_proj", "model.layers.64.mlp.experts.47.gate_proj", "model.layers.64.mlp.experts.48.gate_proj", "model.layers.64.mlp.experts.49.gate_proj", "model.layers.64.mlp.experts.50.gate_proj", "model.layers.64.mlp.experts.51.gate_proj", "model.layers.64.mlp.experts.52.gate_proj", "model.layers.64.mlp.experts.53.gate_proj", "model.layers.64.mlp.experts.54.gate_proj", "model.layers.64.mlp.experts.55.gate_proj", "model.layers.64.mlp.experts.56.gate_proj", "model.layers.64.mlp.experts.57.gate_proj", "model.layers.64.mlp.experts.58.gate_proj", "model.layers.64.mlp.experts.59.gate_proj", "model.layers.64.mlp.experts.60.gate_proj", "model.layers.64.mlp.experts.61.gate_proj", "model.layers.64.mlp.experts.62.gate_proj", "model.layers.64.mlp.experts.63.gate_proj", "model.layers.64.mlp.experts.64.gate_proj", "model.layers.64.mlp.experts.65.gate_proj", "model.layers.64.mlp.experts.66.gate_proj", "model.layers.64.mlp.experts.67.gate_proj", "model.layers.64.mlp.experts.68.gate_proj", "model.layers.64.mlp.experts.69.gate_proj", "model.layers.64.mlp.experts.70.gate_proj", "model.layers.64.mlp.experts.71.gate_proj", "model.layers.64.mlp.experts.72.gate_proj", "model.layers.64.mlp.experts.73.gate_proj", "model.layers.64.mlp.experts.74.gate_proj", "model.layers.64.mlp.experts.75.gate_proj", "model.layers.64.mlp.experts.76.gate_proj", "model.layers.64.mlp.experts.77.gate_proj", "model.layers.64.mlp.experts.78.gate_proj", "model.layers.64.mlp.experts.79.gate_proj", "model.layers.64.mlp.experts.80.gate_proj", "model.layers.64.mlp.experts.81.gate_proj", "model.layers.64.mlp.experts.82.gate_proj", "model.layers.64.mlp.experts.83.gate_proj", "model.layers.64.mlp.experts.84.gate_proj", "model.layers.64.mlp.experts.85.gate_proj", "model.layers.64.mlp.experts.86.gate_proj", "model.layers.64.mlp.experts.87.gate_proj", "model.layers.64.mlp.experts.88.gate_proj", "model.layers.64.mlp.experts.89.gate_proj", "model.layers.64.mlp.experts.90.gate_proj", "model.layers.64.mlp.experts.91.gate_proj", "model.layers.64.mlp.experts.92.gate_proj", "model.layers.64.mlp.experts.93.gate_proj", "model.layers.64.mlp.experts.94.gate_proj", "model.layers.64.mlp.experts.95.gate_proj", "model.layers.64.mlp.experts.96.gate_proj", "model.layers.64.mlp.experts.97.gate_proj", "model.layers.64.mlp.experts.98.gate_proj", "model.layers.64.mlp.experts.99.gate_proj", "model.layers.64.mlp.experts.100.gate_proj", "model.layers.64.mlp.experts.101.gate_proj", "model.layers.64.mlp.experts.102.gate_proj", "model.layers.64.mlp.experts.103.gate_proj", "model.layers.64.mlp.experts.104.gate_proj", "model.layers.64.mlp.experts.105.gate_proj", "model.layers.64.mlp.experts.106.gate_proj", "model.layers.64.mlp.experts.107.gate_proj", "model.layers.64.mlp.experts.108.gate_proj", "model.layers.64.mlp.experts.109.gate_proj", "model.layers.64.mlp.experts.110.gate_proj", "model.layers.64.mlp.experts.111.gate_proj", "model.layers.64.mlp.experts.112.gate_proj", "model.layers.64.mlp.experts.113.gate_proj", "model.layers.64.mlp.experts.114.gate_proj", "model.layers.64.mlp.experts.115.gate_proj", "model.layers.64.mlp.experts.116.gate_proj", "model.layers.64.mlp.experts.117.gate_proj", "model.layers.64.mlp.experts.118.gate_proj", "model.layers.64.mlp.experts.119.gate_proj", "model.layers.64.mlp.experts.120.gate_proj", "model.layers.64.mlp.experts.121.gate_proj", "model.layers.64.mlp.experts.122.gate_proj", "model.layers.64.mlp.experts.123.gate_proj", "model.layers.64.mlp.experts.124.gate_proj", "model.layers.64.mlp.experts.125.gate_proj", "model.layers.64.mlp.experts.126.gate_proj", "model.layers.64.mlp.experts.127.gate_proj", "model.layers.64.mlp.experts.128.gate_proj", "model.layers.64.mlp.experts.129.gate_proj", "model.layers.64.mlp.experts.130.gate_proj", "model.layers.64.mlp.experts.131.gate_proj", "model.layers.64.mlp.experts.132.gate_proj", "model.layers.64.mlp.experts.133.gate_proj", "model.layers.64.mlp.experts.134.gate_proj", "model.layers.64.mlp.experts.135.gate_proj", "model.layers.64.mlp.experts.136.gate_proj", "model.layers.64.mlp.experts.137.gate_proj", "model.layers.64.mlp.experts.138.gate_proj", "model.layers.64.mlp.experts.139.gate_proj", "model.layers.64.mlp.experts.140.gate_proj", "model.layers.64.mlp.experts.141.gate_proj", "model.layers.64.mlp.experts.142.gate_proj", "model.layers.64.mlp.experts.143.gate_proj", "model.layers.64.mlp.experts.144.gate_proj", "model.layers.64.mlp.experts.145.gate_proj", "model.layers.64.mlp.experts.146.gate_proj", "model.layers.64.mlp.experts.147.gate_proj", "model.layers.64.mlp.experts.148.gate_proj", "model.layers.64.mlp.experts.149.gate_proj", "model.layers.64.mlp.experts.150.gate_proj", "model.layers.64.mlp.experts.151.gate_proj", "model.layers.64.mlp.experts.152.gate_proj", "model.layers.64.mlp.experts.153.gate_proj", "model.layers.64.mlp.experts.154.gate_proj", "model.layers.64.mlp.experts.155.gate_proj", "model.layers.64.mlp.experts.156.gate_proj", "model.layers.64.mlp.experts.157.gate_proj", "model.layers.64.mlp.experts.158.gate_proj", "model.layers.64.mlp.experts.159.gate_proj", "model.layers.64.mlp.experts.0.up_proj", "model.layers.64.mlp.experts.1.up_proj", "model.layers.64.mlp.experts.2.up_proj", "model.layers.64.mlp.experts.3.up_proj", "model.layers.64.mlp.experts.4.up_proj", "model.layers.64.mlp.experts.5.up_proj", "model.layers.64.mlp.experts.6.up_proj", "model.layers.64.mlp.experts.7.up_proj", "model.layers.64.mlp.experts.8.up_proj", "model.layers.64.mlp.experts.9.up_proj", "model.layers.64.mlp.experts.10.up_proj", "model.layers.64.mlp.experts.11.up_proj", "model.layers.64.mlp.experts.12.up_proj", "model.layers.64.mlp.experts.13.up_proj", "model.layers.64.mlp.experts.14.up_proj", "model.layers.64.mlp.experts.15.up_proj", "model.layers.64.mlp.experts.16.up_proj", "model.layers.64.mlp.experts.17.up_proj", "model.layers.64.mlp.experts.18.up_proj", "model.layers.64.mlp.experts.19.up_proj", "model.layers.64.mlp.experts.20.up_proj", "model.layers.64.mlp.experts.21.up_proj", "model.layers.64.mlp.experts.22.up_proj", "model.layers.64.mlp.experts.23.up_proj", "model.layers.64.mlp.experts.24.up_proj", "model.layers.64.mlp.experts.25.up_proj", "model.layers.64.mlp.experts.26.up_proj", "model.layers.64.mlp.experts.27.up_proj", "model.layers.64.mlp.experts.28.up_proj", "model.layers.64.mlp.experts.29.up_proj", "model.layers.64.mlp.experts.30.up_proj", "model.layers.64.mlp.experts.31.up_proj", "model.layers.64.mlp.experts.32.up_proj", "model.layers.64.mlp.experts.33.up_proj", "model.layers.64.mlp.experts.34.up_proj", "model.layers.64.mlp.experts.35.up_proj", "model.layers.64.mlp.experts.36.up_proj", "model.layers.64.mlp.experts.37.up_proj", "model.layers.64.mlp.experts.38.up_proj", "model.layers.64.mlp.experts.39.up_proj", "model.layers.64.mlp.experts.40.up_proj", "model.layers.64.mlp.experts.41.up_proj", "model.layers.64.mlp.experts.42.up_proj", "model.layers.64.mlp.experts.43.up_proj", "model.layers.64.mlp.experts.44.up_proj", "model.layers.64.mlp.experts.45.up_proj", "model.layers.64.mlp.experts.46.up_proj", "model.layers.64.mlp.experts.47.up_proj", "model.layers.64.mlp.experts.48.up_proj", "model.layers.64.mlp.experts.49.up_proj", "model.layers.64.mlp.experts.50.up_proj", "model.layers.64.mlp.experts.51.up_proj", "model.layers.64.mlp.experts.52.up_proj", "model.layers.64.mlp.experts.53.up_proj", "model.layers.64.mlp.experts.54.up_proj", "model.layers.64.mlp.experts.55.up_proj", "model.layers.64.mlp.experts.56.up_proj", "model.layers.64.mlp.experts.57.up_proj", "model.layers.64.mlp.experts.58.up_proj", "model.layers.64.mlp.experts.59.up_proj", "model.layers.64.mlp.experts.60.up_proj", "model.layers.64.mlp.experts.61.up_proj", "model.layers.64.mlp.experts.62.up_proj", "model.layers.64.mlp.experts.63.up_proj", "model.layers.64.mlp.experts.64.up_proj", "model.layers.64.mlp.experts.65.up_proj", "model.layers.64.mlp.experts.66.up_proj", "model.layers.64.mlp.experts.67.up_proj", "model.layers.64.mlp.experts.68.up_proj", "model.layers.64.mlp.experts.69.up_proj", "model.layers.64.mlp.experts.70.up_proj", "model.layers.64.mlp.experts.71.up_proj", "model.layers.64.mlp.experts.72.up_proj", "model.layers.64.mlp.experts.73.up_proj", "model.layers.64.mlp.experts.74.up_proj", "model.layers.64.mlp.experts.75.up_proj", "model.layers.64.mlp.experts.76.up_proj", "model.layers.64.mlp.experts.77.up_proj", "model.layers.64.mlp.experts.78.up_proj", "model.layers.64.mlp.experts.79.up_proj", "model.layers.64.mlp.experts.80.up_proj", "model.layers.64.mlp.experts.81.up_proj", "model.layers.64.mlp.experts.82.up_proj", "model.layers.64.mlp.experts.83.up_proj", "model.layers.64.mlp.experts.84.up_proj", "model.layers.64.mlp.experts.85.up_proj", "model.layers.64.mlp.experts.86.up_proj", "model.layers.64.mlp.experts.87.up_proj", "model.layers.64.mlp.experts.88.up_proj", "model.layers.64.mlp.experts.89.up_proj", "model.layers.64.mlp.experts.90.up_proj", "model.layers.64.mlp.experts.91.up_proj", "model.layers.64.mlp.experts.92.up_proj", "model.layers.64.mlp.experts.93.up_proj", "model.layers.64.mlp.experts.94.up_proj", "model.layers.64.mlp.experts.95.up_proj", "model.layers.64.mlp.experts.96.up_proj", "model.layers.64.mlp.experts.97.up_proj", "model.layers.64.mlp.experts.98.up_proj", "model.layers.64.mlp.experts.99.up_proj", "model.layers.64.mlp.experts.100.up_proj", "model.layers.64.mlp.experts.101.up_proj", "model.layers.64.mlp.experts.102.up_proj", "model.layers.64.mlp.experts.103.up_proj", "model.layers.64.mlp.experts.104.up_proj", "model.layers.64.mlp.experts.105.up_proj", "model.layers.64.mlp.experts.106.up_proj", "model.layers.64.mlp.experts.107.up_proj", "model.layers.64.mlp.experts.108.up_proj", "model.layers.64.mlp.experts.109.up_proj", "model.layers.64.mlp.experts.110.up_proj", "model.layers.64.mlp.experts.111.up_proj", "model.layers.64.mlp.experts.112.up_proj", "model.layers.64.mlp.experts.113.up_proj", "model.layers.64.mlp.experts.114.up_proj", "model.layers.64.mlp.experts.115.up_proj", "model.layers.64.mlp.experts.116.up_proj", "model.layers.64.mlp.experts.117.up_proj", "model.layers.64.mlp.experts.118.up_proj", "model.layers.64.mlp.experts.119.up_proj", "model.layers.64.mlp.experts.120.up_proj", "model.layers.64.mlp.experts.121.up_proj", "model.layers.64.mlp.experts.122.up_proj", "model.layers.64.mlp.experts.123.up_proj", "model.layers.64.mlp.experts.124.up_proj", "model.layers.64.mlp.experts.125.up_proj", "model.layers.64.mlp.experts.126.up_proj", "model.layers.64.mlp.experts.127.up_proj", "model.layers.64.mlp.experts.128.up_proj", "model.layers.64.mlp.experts.129.up_proj", "model.layers.64.mlp.experts.130.up_proj", "model.layers.64.mlp.experts.131.up_proj", "model.layers.64.mlp.experts.132.up_proj", "model.layers.64.mlp.experts.133.up_proj", "model.layers.64.mlp.experts.134.up_proj", "model.layers.64.mlp.experts.135.up_proj", "model.layers.64.mlp.experts.136.up_proj", "model.layers.64.mlp.experts.137.up_proj", "model.layers.64.mlp.experts.138.up_proj", "model.layers.64.mlp.experts.139.up_proj", "model.layers.64.mlp.experts.140.up_proj", "model.layers.64.mlp.experts.141.up_proj", "model.layers.64.mlp.experts.142.up_proj", "model.layers.64.mlp.experts.143.up_proj", "model.layers.64.mlp.experts.144.up_proj", "model.layers.64.mlp.experts.145.up_proj", "model.layers.64.mlp.experts.146.up_proj", "model.layers.64.mlp.experts.147.up_proj", "model.layers.64.mlp.experts.148.up_proj", "model.layers.64.mlp.experts.149.up_proj", "model.layers.64.mlp.experts.150.up_proj", "model.layers.64.mlp.experts.151.up_proj", "model.layers.64.mlp.experts.152.up_proj", "model.layers.64.mlp.experts.153.up_proj", "model.layers.64.mlp.experts.154.up_proj", "model.layers.64.mlp.experts.155.up_proj", "model.layers.64.mlp.experts.156.up_proj", "model.layers.64.mlp.experts.157.up_proj", "model.layers.64.mlp.experts.158.up_proj", "model.layers.64.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0005654476583004109, "dbits": 2516582400 } ] }, { "idx": 386, "layers": [ "model.layers.64.mlp.experts.0.down_proj", "model.layers.64.mlp.experts.1.down_proj", "model.layers.64.mlp.experts.2.down_proj", "model.layers.64.mlp.experts.3.down_proj", "model.layers.64.mlp.experts.4.down_proj", "model.layers.64.mlp.experts.5.down_proj", "model.layers.64.mlp.experts.6.down_proj", "model.layers.64.mlp.experts.7.down_proj", "model.layers.64.mlp.experts.8.down_proj", "model.layers.64.mlp.experts.9.down_proj", "model.layers.64.mlp.experts.10.down_proj", "model.layers.64.mlp.experts.11.down_proj", "model.layers.64.mlp.experts.12.down_proj", "model.layers.64.mlp.experts.13.down_proj", "model.layers.64.mlp.experts.14.down_proj", "model.layers.64.mlp.experts.15.down_proj", "model.layers.64.mlp.experts.16.down_proj", "model.layers.64.mlp.experts.17.down_proj", "model.layers.64.mlp.experts.18.down_proj", "model.layers.64.mlp.experts.19.down_proj", "model.layers.64.mlp.experts.20.down_proj", "model.layers.64.mlp.experts.21.down_proj", "model.layers.64.mlp.experts.22.down_proj", "model.layers.64.mlp.experts.23.down_proj", "model.layers.64.mlp.experts.24.down_proj", "model.layers.64.mlp.experts.25.down_proj", "model.layers.64.mlp.experts.26.down_proj", "model.layers.64.mlp.experts.27.down_proj", "model.layers.64.mlp.experts.28.down_proj", "model.layers.64.mlp.experts.29.down_proj", "model.layers.64.mlp.experts.30.down_proj", "model.layers.64.mlp.experts.31.down_proj", "model.layers.64.mlp.experts.32.down_proj", "model.layers.64.mlp.experts.33.down_proj", "model.layers.64.mlp.experts.34.down_proj", "model.layers.64.mlp.experts.35.down_proj", "model.layers.64.mlp.experts.36.down_proj", "model.layers.64.mlp.experts.37.down_proj", "model.layers.64.mlp.experts.38.down_proj", "model.layers.64.mlp.experts.39.down_proj", "model.layers.64.mlp.experts.40.down_proj", "model.layers.64.mlp.experts.41.down_proj", "model.layers.64.mlp.experts.42.down_proj", "model.layers.64.mlp.experts.43.down_proj", "model.layers.64.mlp.experts.44.down_proj", "model.layers.64.mlp.experts.45.down_proj", "model.layers.64.mlp.experts.46.down_proj", "model.layers.64.mlp.experts.47.down_proj", "model.layers.64.mlp.experts.48.down_proj", "model.layers.64.mlp.experts.49.down_proj", "model.layers.64.mlp.experts.50.down_proj", "model.layers.64.mlp.experts.51.down_proj", "model.layers.64.mlp.experts.52.down_proj", "model.layers.64.mlp.experts.53.down_proj", "model.layers.64.mlp.experts.54.down_proj", "model.layers.64.mlp.experts.55.down_proj", "model.layers.64.mlp.experts.56.down_proj", "model.layers.64.mlp.experts.57.down_proj", "model.layers.64.mlp.experts.58.down_proj", "model.layers.64.mlp.experts.59.down_proj", "model.layers.64.mlp.experts.60.down_proj", "model.layers.64.mlp.experts.61.down_proj", "model.layers.64.mlp.experts.62.down_proj", "model.layers.64.mlp.experts.63.down_proj", "model.layers.64.mlp.experts.64.down_proj", "model.layers.64.mlp.experts.65.down_proj", "model.layers.64.mlp.experts.66.down_proj", "model.layers.64.mlp.experts.67.down_proj", "model.layers.64.mlp.experts.68.down_proj", "model.layers.64.mlp.experts.69.down_proj", "model.layers.64.mlp.experts.70.down_proj", "model.layers.64.mlp.experts.71.down_proj", "model.layers.64.mlp.experts.72.down_proj", "model.layers.64.mlp.experts.73.down_proj", "model.layers.64.mlp.experts.74.down_proj", "model.layers.64.mlp.experts.75.down_proj", "model.layers.64.mlp.experts.76.down_proj", "model.layers.64.mlp.experts.77.down_proj", "model.layers.64.mlp.experts.78.down_proj", "model.layers.64.mlp.experts.79.down_proj", "model.layers.64.mlp.experts.80.down_proj", "model.layers.64.mlp.experts.81.down_proj", "model.layers.64.mlp.experts.82.down_proj", "model.layers.64.mlp.experts.83.down_proj", "model.layers.64.mlp.experts.84.down_proj", "model.layers.64.mlp.experts.85.down_proj", "model.layers.64.mlp.experts.86.down_proj", "model.layers.64.mlp.experts.87.down_proj", "model.layers.64.mlp.experts.88.down_proj", "model.layers.64.mlp.experts.89.down_proj", "model.layers.64.mlp.experts.90.down_proj", "model.layers.64.mlp.experts.91.down_proj", "model.layers.64.mlp.experts.92.down_proj", "model.layers.64.mlp.experts.93.down_proj", "model.layers.64.mlp.experts.94.down_proj", "model.layers.64.mlp.experts.95.down_proj", "model.layers.64.mlp.experts.96.down_proj", "model.layers.64.mlp.experts.97.down_proj", "model.layers.64.mlp.experts.98.down_proj", "model.layers.64.mlp.experts.99.down_proj", "model.layers.64.mlp.experts.100.down_proj", "model.layers.64.mlp.experts.101.down_proj", "model.layers.64.mlp.experts.102.down_proj", "model.layers.64.mlp.experts.103.down_proj", "model.layers.64.mlp.experts.104.down_proj", "model.layers.64.mlp.experts.105.down_proj", "model.layers.64.mlp.experts.106.down_proj", "model.layers.64.mlp.experts.107.down_proj", "model.layers.64.mlp.experts.108.down_proj", "model.layers.64.mlp.experts.109.down_proj", "model.layers.64.mlp.experts.110.down_proj", "model.layers.64.mlp.experts.111.down_proj", "model.layers.64.mlp.experts.112.down_proj", "model.layers.64.mlp.experts.113.down_proj", "model.layers.64.mlp.experts.114.down_proj", "model.layers.64.mlp.experts.115.down_proj", "model.layers.64.mlp.experts.116.down_proj", "model.layers.64.mlp.experts.117.down_proj", "model.layers.64.mlp.experts.118.down_proj", "model.layers.64.mlp.experts.119.down_proj", "model.layers.64.mlp.experts.120.down_proj", "model.layers.64.mlp.experts.121.down_proj", "model.layers.64.mlp.experts.122.down_proj", "model.layers.64.mlp.experts.123.down_proj", "model.layers.64.mlp.experts.124.down_proj", "model.layers.64.mlp.experts.125.down_proj", "model.layers.64.mlp.experts.126.down_proj", "model.layers.64.mlp.experts.127.down_proj", "model.layers.64.mlp.experts.128.down_proj", "model.layers.64.mlp.experts.129.down_proj", "model.layers.64.mlp.experts.130.down_proj", "model.layers.64.mlp.experts.131.down_proj", "model.layers.64.mlp.experts.132.down_proj", "model.layers.64.mlp.experts.133.down_proj", "model.layers.64.mlp.experts.134.down_proj", "model.layers.64.mlp.experts.135.down_proj", "model.layers.64.mlp.experts.136.down_proj", "model.layers.64.mlp.experts.137.down_proj", "model.layers.64.mlp.experts.138.down_proj", "model.layers.64.mlp.experts.139.down_proj", "model.layers.64.mlp.experts.140.down_proj", "model.layers.64.mlp.experts.141.down_proj", "model.layers.64.mlp.experts.142.down_proj", "model.layers.64.mlp.experts.143.down_proj", "model.layers.64.mlp.experts.144.down_proj", "model.layers.64.mlp.experts.145.down_proj", "model.layers.64.mlp.experts.146.down_proj", "model.layers.64.mlp.experts.147.down_proj", "model.layers.64.mlp.experts.148.down_proj", "model.layers.64.mlp.experts.149.down_proj", "model.layers.64.mlp.experts.150.down_proj", "model.layers.64.mlp.experts.151.down_proj", "model.layers.64.mlp.experts.152.down_proj", "model.layers.64.mlp.experts.153.down_proj", "model.layers.64.mlp.experts.154.down_proj", "model.layers.64.mlp.experts.155.down_proj", "model.layers.64.mlp.experts.156.down_proj", "model.layers.64.mlp.experts.157.down_proj", "model.layers.64.mlp.experts.158.down_proj", "model.layers.64.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00046008899807931103, "dbits": 1258291200 } ] }, { "idx": 387, "layers": [ "model.layers.65.self_attn.q_proj" ], "candidates": [ { "dkld": -4.023313521228644e-08, "dbits": 62914560 } ] }, { "idx": 388, "layers": [ "model.layers.65.self_attn.k_proj", "model.layers.65.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00202839672565458, "dbits": 10485760 } ] }, { "idx": 389, "layers": [ "model.layers.65.self_attn.o_proj" ], "candidates": [ { "dkld": 4.277378320705072e-06, "dbits": 62914560 } ] }, { "idx": 390, "layers": [ "model.layers.65.mlp.shared_experts.gate_proj", "model.layers.65.mlp.shared_experts.up_proj", "model.layers.65.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.00027899667620656654, "dbits": 23592960 } ] }, { "idx": 391, "layers": [ "model.layers.65.mlp.experts.0.gate_proj", "model.layers.65.mlp.experts.1.gate_proj", "model.layers.65.mlp.experts.2.gate_proj", "model.layers.65.mlp.experts.3.gate_proj", "model.layers.65.mlp.experts.4.gate_proj", "model.layers.65.mlp.experts.5.gate_proj", "model.layers.65.mlp.experts.6.gate_proj", "model.layers.65.mlp.experts.7.gate_proj", "model.layers.65.mlp.experts.8.gate_proj", "model.layers.65.mlp.experts.9.gate_proj", "model.layers.65.mlp.experts.10.gate_proj", "model.layers.65.mlp.experts.11.gate_proj", "model.layers.65.mlp.experts.12.gate_proj", "model.layers.65.mlp.experts.13.gate_proj", "model.layers.65.mlp.experts.14.gate_proj", "model.layers.65.mlp.experts.15.gate_proj", "model.layers.65.mlp.experts.16.gate_proj", "model.layers.65.mlp.experts.17.gate_proj", "model.layers.65.mlp.experts.18.gate_proj", "model.layers.65.mlp.experts.19.gate_proj", "model.layers.65.mlp.experts.20.gate_proj", "model.layers.65.mlp.experts.21.gate_proj", "model.layers.65.mlp.experts.22.gate_proj", "model.layers.65.mlp.experts.23.gate_proj", "model.layers.65.mlp.experts.24.gate_proj", "model.layers.65.mlp.experts.25.gate_proj", "model.layers.65.mlp.experts.26.gate_proj", "model.layers.65.mlp.experts.27.gate_proj", "model.layers.65.mlp.experts.28.gate_proj", "model.layers.65.mlp.experts.29.gate_proj", "model.layers.65.mlp.experts.30.gate_proj", "model.layers.65.mlp.experts.31.gate_proj", "model.layers.65.mlp.experts.32.gate_proj", "model.layers.65.mlp.experts.33.gate_proj", "model.layers.65.mlp.experts.34.gate_proj", "model.layers.65.mlp.experts.35.gate_proj", "model.layers.65.mlp.experts.36.gate_proj", "model.layers.65.mlp.experts.37.gate_proj", "model.layers.65.mlp.experts.38.gate_proj", "model.layers.65.mlp.experts.39.gate_proj", "model.layers.65.mlp.experts.40.gate_proj", "model.layers.65.mlp.experts.41.gate_proj", "model.layers.65.mlp.experts.42.gate_proj", "model.layers.65.mlp.experts.43.gate_proj", "model.layers.65.mlp.experts.44.gate_proj", "model.layers.65.mlp.experts.45.gate_proj", "model.layers.65.mlp.experts.46.gate_proj", "model.layers.65.mlp.experts.47.gate_proj", "model.layers.65.mlp.experts.48.gate_proj", "model.layers.65.mlp.experts.49.gate_proj", "model.layers.65.mlp.experts.50.gate_proj", "model.layers.65.mlp.experts.51.gate_proj", "model.layers.65.mlp.experts.52.gate_proj", "model.layers.65.mlp.experts.53.gate_proj", "model.layers.65.mlp.experts.54.gate_proj", "model.layers.65.mlp.experts.55.gate_proj", "model.layers.65.mlp.experts.56.gate_proj", "model.layers.65.mlp.experts.57.gate_proj", "model.layers.65.mlp.experts.58.gate_proj", "model.layers.65.mlp.experts.59.gate_proj", "model.layers.65.mlp.experts.60.gate_proj", "model.layers.65.mlp.experts.61.gate_proj", "model.layers.65.mlp.experts.62.gate_proj", "model.layers.65.mlp.experts.63.gate_proj", "model.layers.65.mlp.experts.64.gate_proj", "model.layers.65.mlp.experts.65.gate_proj", "model.layers.65.mlp.experts.66.gate_proj", "model.layers.65.mlp.experts.67.gate_proj", "model.layers.65.mlp.experts.68.gate_proj", "model.layers.65.mlp.experts.69.gate_proj", "model.layers.65.mlp.experts.70.gate_proj", "model.layers.65.mlp.experts.71.gate_proj", "model.layers.65.mlp.experts.72.gate_proj", "model.layers.65.mlp.experts.73.gate_proj", "model.layers.65.mlp.experts.74.gate_proj", "model.layers.65.mlp.experts.75.gate_proj", "model.layers.65.mlp.experts.76.gate_proj", "model.layers.65.mlp.experts.77.gate_proj", "model.layers.65.mlp.experts.78.gate_proj", "model.layers.65.mlp.experts.79.gate_proj", "model.layers.65.mlp.experts.80.gate_proj", "model.layers.65.mlp.experts.81.gate_proj", "model.layers.65.mlp.experts.82.gate_proj", "model.layers.65.mlp.experts.83.gate_proj", "model.layers.65.mlp.experts.84.gate_proj", "model.layers.65.mlp.experts.85.gate_proj", "model.layers.65.mlp.experts.86.gate_proj", "model.layers.65.mlp.experts.87.gate_proj", "model.layers.65.mlp.experts.88.gate_proj", "model.layers.65.mlp.experts.89.gate_proj", "model.layers.65.mlp.experts.90.gate_proj", "model.layers.65.mlp.experts.91.gate_proj", "model.layers.65.mlp.experts.92.gate_proj", "model.layers.65.mlp.experts.93.gate_proj", "model.layers.65.mlp.experts.94.gate_proj", "model.layers.65.mlp.experts.95.gate_proj", "model.layers.65.mlp.experts.96.gate_proj", "model.layers.65.mlp.experts.97.gate_proj", "model.layers.65.mlp.experts.98.gate_proj", "model.layers.65.mlp.experts.99.gate_proj", "model.layers.65.mlp.experts.100.gate_proj", "model.layers.65.mlp.experts.101.gate_proj", "model.layers.65.mlp.experts.102.gate_proj", "model.layers.65.mlp.experts.103.gate_proj", "model.layers.65.mlp.experts.104.gate_proj", "model.layers.65.mlp.experts.105.gate_proj", "model.layers.65.mlp.experts.106.gate_proj", "model.layers.65.mlp.experts.107.gate_proj", "model.layers.65.mlp.experts.108.gate_proj", "model.layers.65.mlp.experts.109.gate_proj", "model.layers.65.mlp.experts.110.gate_proj", "model.layers.65.mlp.experts.111.gate_proj", "model.layers.65.mlp.experts.112.gate_proj", "model.layers.65.mlp.experts.113.gate_proj", "model.layers.65.mlp.experts.114.gate_proj", "model.layers.65.mlp.experts.115.gate_proj", "model.layers.65.mlp.experts.116.gate_proj", "model.layers.65.mlp.experts.117.gate_proj", "model.layers.65.mlp.experts.118.gate_proj", "model.layers.65.mlp.experts.119.gate_proj", "model.layers.65.mlp.experts.120.gate_proj", "model.layers.65.mlp.experts.121.gate_proj", "model.layers.65.mlp.experts.122.gate_proj", "model.layers.65.mlp.experts.123.gate_proj", "model.layers.65.mlp.experts.124.gate_proj", "model.layers.65.mlp.experts.125.gate_proj", "model.layers.65.mlp.experts.126.gate_proj", "model.layers.65.mlp.experts.127.gate_proj", "model.layers.65.mlp.experts.128.gate_proj", "model.layers.65.mlp.experts.129.gate_proj", "model.layers.65.mlp.experts.130.gate_proj", "model.layers.65.mlp.experts.131.gate_proj", "model.layers.65.mlp.experts.132.gate_proj", "model.layers.65.mlp.experts.133.gate_proj", "model.layers.65.mlp.experts.134.gate_proj", "model.layers.65.mlp.experts.135.gate_proj", "model.layers.65.mlp.experts.136.gate_proj", "model.layers.65.mlp.experts.137.gate_proj", "model.layers.65.mlp.experts.138.gate_proj", "model.layers.65.mlp.experts.139.gate_proj", "model.layers.65.mlp.experts.140.gate_proj", "model.layers.65.mlp.experts.141.gate_proj", "model.layers.65.mlp.experts.142.gate_proj", "model.layers.65.mlp.experts.143.gate_proj", "model.layers.65.mlp.experts.144.gate_proj", "model.layers.65.mlp.experts.145.gate_proj", "model.layers.65.mlp.experts.146.gate_proj", "model.layers.65.mlp.experts.147.gate_proj", "model.layers.65.mlp.experts.148.gate_proj", "model.layers.65.mlp.experts.149.gate_proj", "model.layers.65.mlp.experts.150.gate_proj", "model.layers.65.mlp.experts.151.gate_proj", "model.layers.65.mlp.experts.152.gate_proj", "model.layers.65.mlp.experts.153.gate_proj", "model.layers.65.mlp.experts.154.gate_proj", "model.layers.65.mlp.experts.155.gate_proj", "model.layers.65.mlp.experts.156.gate_proj", "model.layers.65.mlp.experts.157.gate_proj", "model.layers.65.mlp.experts.158.gate_proj", "model.layers.65.mlp.experts.159.gate_proj", "model.layers.65.mlp.experts.0.up_proj", "model.layers.65.mlp.experts.1.up_proj", "model.layers.65.mlp.experts.2.up_proj", "model.layers.65.mlp.experts.3.up_proj", "model.layers.65.mlp.experts.4.up_proj", "model.layers.65.mlp.experts.5.up_proj", "model.layers.65.mlp.experts.6.up_proj", "model.layers.65.mlp.experts.7.up_proj", "model.layers.65.mlp.experts.8.up_proj", "model.layers.65.mlp.experts.9.up_proj", "model.layers.65.mlp.experts.10.up_proj", "model.layers.65.mlp.experts.11.up_proj", "model.layers.65.mlp.experts.12.up_proj", "model.layers.65.mlp.experts.13.up_proj", "model.layers.65.mlp.experts.14.up_proj", "model.layers.65.mlp.experts.15.up_proj", "model.layers.65.mlp.experts.16.up_proj", "model.layers.65.mlp.experts.17.up_proj", "model.layers.65.mlp.experts.18.up_proj", "model.layers.65.mlp.experts.19.up_proj", "model.layers.65.mlp.experts.20.up_proj", "model.layers.65.mlp.experts.21.up_proj", "model.layers.65.mlp.experts.22.up_proj", "model.layers.65.mlp.experts.23.up_proj", "model.layers.65.mlp.experts.24.up_proj", "model.layers.65.mlp.experts.25.up_proj", "model.layers.65.mlp.experts.26.up_proj", "model.layers.65.mlp.experts.27.up_proj", "model.layers.65.mlp.experts.28.up_proj", "model.layers.65.mlp.experts.29.up_proj", "model.layers.65.mlp.experts.30.up_proj", "model.layers.65.mlp.experts.31.up_proj", "model.layers.65.mlp.experts.32.up_proj", "model.layers.65.mlp.experts.33.up_proj", "model.layers.65.mlp.experts.34.up_proj", "model.layers.65.mlp.experts.35.up_proj", "model.layers.65.mlp.experts.36.up_proj", "model.layers.65.mlp.experts.37.up_proj", "model.layers.65.mlp.experts.38.up_proj", "model.layers.65.mlp.experts.39.up_proj", "model.layers.65.mlp.experts.40.up_proj", "model.layers.65.mlp.experts.41.up_proj", "model.layers.65.mlp.experts.42.up_proj", "model.layers.65.mlp.experts.43.up_proj", "model.layers.65.mlp.experts.44.up_proj", "model.layers.65.mlp.experts.45.up_proj", "model.layers.65.mlp.experts.46.up_proj", "model.layers.65.mlp.experts.47.up_proj", "model.layers.65.mlp.experts.48.up_proj", "model.layers.65.mlp.experts.49.up_proj", "model.layers.65.mlp.experts.50.up_proj", "model.layers.65.mlp.experts.51.up_proj", "model.layers.65.mlp.experts.52.up_proj", "model.layers.65.mlp.experts.53.up_proj", "model.layers.65.mlp.experts.54.up_proj", "model.layers.65.mlp.experts.55.up_proj", "model.layers.65.mlp.experts.56.up_proj", "model.layers.65.mlp.experts.57.up_proj", "model.layers.65.mlp.experts.58.up_proj", "model.layers.65.mlp.experts.59.up_proj", "model.layers.65.mlp.experts.60.up_proj", "model.layers.65.mlp.experts.61.up_proj", "model.layers.65.mlp.experts.62.up_proj", "model.layers.65.mlp.experts.63.up_proj", "model.layers.65.mlp.experts.64.up_proj", "model.layers.65.mlp.experts.65.up_proj", "model.layers.65.mlp.experts.66.up_proj", "model.layers.65.mlp.experts.67.up_proj", "model.layers.65.mlp.experts.68.up_proj", "model.layers.65.mlp.experts.69.up_proj", "model.layers.65.mlp.experts.70.up_proj", "model.layers.65.mlp.experts.71.up_proj", "model.layers.65.mlp.experts.72.up_proj", "model.layers.65.mlp.experts.73.up_proj", "model.layers.65.mlp.experts.74.up_proj", "model.layers.65.mlp.experts.75.up_proj", "model.layers.65.mlp.experts.76.up_proj", "model.layers.65.mlp.experts.77.up_proj", "model.layers.65.mlp.experts.78.up_proj", "model.layers.65.mlp.experts.79.up_proj", "model.layers.65.mlp.experts.80.up_proj", "model.layers.65.mlp.experts.81.up_proj", "model.layers.65.mlp.experts.82.up_proj", "model.layers.65.mlp.experts.83.up_proj", "model.layers.65.mlp.experts.84.up_proj", "model.layers.65.mlp.experts.85.up_proj", "model.layers.65.mlp.experts.86.up_proj", "model.layers.65.mlp.experts.87.up_proj", "model.layers.65.mlp.experts.88.up_proj", "model.layers.65.mlp.experts.89.up_proj", "model.layers.65.mlp.experts.90.up_proj", "model.layers.65.mlp.experts.91.up_proj", "model.layers.65.mlp.experts.92.up_proj", "model.layers.65.mlp.experts.93.up_proj", "model.layers.65.mlp.experts.94.up_proj", "model.layers.65.mlp.experts.95.up_proj", "model.layers.65.mlp.experts.96.up_proj", "model.layers.65.mlp.experts.97.up_proj", "model.layers.65.mlp.experts.98.up_proj", "model.layers.65.mlp.experts.99.up_proj", "model.layers.65.mlp.experts.100.up_proj", "model.layers.65.mlp.experts.101.up_proj", "model.layers.65.mlp.experts.102.up_proj", "model.layers.65.mlp.experts.103.up_proj", "model.layers.65.mlp.experts.104.up_proj", "model.layers.65.mlp.experts.105.up_proj", "model.layers.65.mlp.experts.106.up_proj", "model.layers.65.mlp.experts.107.up_proj", "model.layers.65.mlp.experts.108.up_proj", "model.layers.65.mlp.experts.109.up_proj", "model.layers.65.mlp.experts.110.up_proj", "model.layers.65.mlp.experts.111.up_proj", "model.layers.65.mlp.experts.112.up_proj", "model.layers.65.mlp.experts.113.up_proj", "model.layers.65.mlp.experts.114.up_proj", "model.layers.65.mlp.experts.115.up_proj", "model.layers.65.mlp.experts.116.up_proj", "model.layers.65.mlp.experts.117.up_proj", "model.layers.65.mlp.experts.118.up_proj", "model.layers.65.mlp.experts.119.up_proj", "model.layers.65.mlp.experts.120.up_proj", "model.layers.65.mlp.experts.121.up_proj", "model.layers.65.mlp.experts.122.up_proj", "model.layers.65.mlp.experts.123.up_proj", "model.layers.65.mlp.experts.124.up_proj", "model.layers.65.mlp.experts.125.up_proj", "model.layers.65.mlp.experts.126.up_proj", "model.layers.65.mlp.experts.127.up_proj", "model.layers.65.mlp.experts.128.up_proj", "model.layers.65.mlp.experts.129.up_proj", "model.layers.65.mlp.experts.130.up_proj", "model.layers.65.mlp.experts.131.up_proj", "model.layers.65.mlp.experts.132.up_proj", "model.layers.65.mlp.experts.133.up_proj", "model.layers.65.mlp.experts.134.up_proj", "model.layers.65.mlp.experts.135.up_proj", "model.layers.65.mlp.experts.136.up_proj", "model.layers.65.mlp.experts.137.up_proj", "model.layers.65.mlp.experts.138.up_proj", "model.layers.65.mlp.experts.139.up_proj", "model.layers.65.mlp.experts.140.up_proj", "model.layers.65.mlp.experts.141.up_proj", "model.layers.65.mlp.experts.142.up_proj", "model.layers.65.mlp.experts.143.up_proj", "model.layers.65.mlp.experts.144.up_proj", "model.layers.65.mlp.experts.145.up_proj", "model.layers.65.mlp.experts.146.up_proj", "model.layers.65.mlp.experts.147.up_proj", "model.layers.65.mlp.experts.148.up_proj", "model.layers.65.mlp.experts.149.up_proj", "model.layers.65.mlp.experts.150.up_proj", "model.layers.65.mlp.experts.151.up_proj", "model.layers.65.mlp.experts.152.up_proj", "model.layers.65.mlp.experts.153.up_proj", "model.layers.65.mlp.experts.154.up_proj", "model.layers.65.mlp.experts.155.up_proj", "model.layers.65.mlp.experts.156.up_proj", "model.layers.65.mlp.experts.157.up_proj", "model.layers.65.mlp.experts.158.up_proj", "model.layers.65.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00044687017798422657, "dbits": 2516582400 } ] }, { "idx": 392, "layers": [ "model.layers.65.mlp.experts.0.down_proj", "model.layers.65.mlp.experts.1.down_proj", "model.layers.65.mlp.experts.2.down_proj", "model.layers.65.mlp.experts.3.down_proj", "model.layers.65.mlp.experts.4.down_proj", "model.layers.65.mlp.experts.5.down_proj", "model.layers.65.mlp.experts.6.down_proj", "model.layers.65.mlp.experts.7.down_proj", "model.layers.65.mlp.experts.8.down_proj", "model.layers.65.mlp.experts.9.down_proj", "model.layers.65.mlp.experts.10.down_proj", "model.layers.65.mlp.experts.11.down_proj", "model.layers.65.mlp.experts.12.down_proj", "model.layers.65.mlp.experts.13.down_proj", "model.layers.65.mlp.experts.14.down_proj", "model.layers.65.mlp.experts.15.down_proj", "model.layers.65.mlp.experts.16.down_proj", "model.layers.65.mlp.experts.17.down_proj", "model.layers.65.mlp.experts.18.down_proj", "model.layers.65.mlp.experts.19.down_proj", "model.layers.65.mlp.experts.20.down_proj", "model.layers.65.mlp.experts.21.down_proj", "model.layers.65.mlp.experts.22.down_proj", "model.layers.65.mlp.experts.23.down_proj", "model.layers.65.mlp.experts.24.down_proj", "model.layers.65.mlp.experts.25.down_proj", "model.layers.65.mlp.experts.26.down_proj", "model.layers.65.mlp.experts.27.down_proj", "model.layers.65.mlp.experts.28.down_proj", "model.layers.65.mlp.experts.29.down_proj", "model.layers.65.mlp.experts.30.down_proj", "model.layers.65.mlp.experts.31.down_proj", "model.layers.65.mlp.experts.32.down_proj", "model.layers.65.mlp.experts.33.down_proj", "model.layers.65.mlp.experts.34.down_proj", "model.layers.65.mlp.experts.35.down_proj", "model.layers.65.mlp.experts.36.down_proj", "model.layers.65.mlp.experts.37.down_proj", "model.layers.65.mlp.experts.38.down_proj", "model.layers.65.mlp.experts.39.down_proj", "model.layers.65.mlp.experts.40.down_proj", "model.layers.65.mlp.experts.41.down_proj", "model.layers.65.mlp.experts.42.down_proj", "model.layers.65.mlp.experts.43.down_proj", "model.layers.65.mlp.experts.44.down_proj", "model.layers.65.mlp.experts.45.down_proj", "model.layers.65.mlp.experts.46.down_proj", "model.layers.65.mlp.experts.47.down_proj", "model.layers.65.mlp.experts.48.down_proj", "model.layers.65.mlp.experts.49.down_proj", "model.layers.65.mlp.experts.50.down_proj", "model.layers.65.mlp.experts.51.down_proj", "model.layers.65.mlp.experts.52.down_proj", "model.layers.65.mlp.experts.53.down_proj", "model.layers.65.mlp.experts.54.down_proj", "model.layers.65.mlp.experts.55.down_proj", "model.layers.65.mlp.experts.56.down_proj", "model.layers.65.mlp.experts.57.down_proj", "model.layers.65.mlp.experts.58.down_proj", "model.layers.65.mlp.experts.59.down_proj", "model.layers.65.mlp.experts.60.down_proj", "model.layers.65.mlp.experts.61.down_proj", "model.layers.65.mlp.experts.62.down_proj", "model.layers.65.mlp.experts.63.down_proj", "model.layers.65.mlp.experts.64.down_proj", "model.layers.65.mlp.experts.65.down_proj", "model.layers.65.mlp.experts.66.down_proj", "model.layers.65.mlp.experts.67.down_proj", "model.layers.65.mlp.experts.68.down_proj", "model.layers.65.mlp.experts.69.down_proj", "model.layers.65.mlp.experts.70.down_proj", "model.layers.65.mlp.experts.71.down_proj", "model.layers.65.mlp.experts.72.down_proj", "model.layers.65.mlp.experts.73.down_proj", "model.layers.65.mlp.experts.74.down_proj", "model.layers.65.mlp.experts.75.down_proj", "model.layers.65.mlp.experts.76.down_proj", "model.layers.65.mlp.experts.77.down_proj", "model.layers.65.mlp.experts.78.down_proj", "model.layers.65.mlp.experts.79.down_proj", "model.layers.65.mlp.experts.80.down_proj", "model.layers.65.mlp.experts.81.down_proj", "model.layers.65.mlp.experts.82.down_proj", "model.layers.65.mlp.experts.83.down_proj", "model.layers.65.mlp.experts.84.down_proj", "model.layers.65.mlp.experts.85.down_proj", "model.layers.65.mlp.experts.86.down_proj", "model.layers.65.mlp.experts.87.down_proj", "model.layers.65.mlp.experts.88.down_proj", "model.layers.65.mlp.experts.89.down_proj", "model.layers.65.mlp.experts.90.down_proj", "model.layers.65.mlp.experts.91.down_proj", "model.layers.65.mlp.experts.92.down_proj", "model.layers.65.mlp.experts.93.down_proj", "model.layers.65.mlp.experts.94.down_proj", "model.layers.65.mlp.experts.95.down_proj", "model.layers.65.mlp.experts.96.down_proj", "model.layers.65.mlp.experts.97.down_proj", "model.layers.65.mlp.experts.98.down_proj", "model.layers.65.mlp.experts.99.down_proj", "model.layers.65.mlp.experts.100.down_proj", "model.layers.65.mlp.experts.101.down_proj", "model.layers.65.mlp.experts.102.down_proj", "model.layers.65.mlp.experts.103.down_proj", "model.layers.65.mlp.experts.104.down_proj", "model.layers.65.mlp.experts.105.down_proj", "model.layers.65.mlp.experts.106.down_proj", "model.layers.65.mlp.experts.107.down_proj", "model.layers.65.mlp.experts.108.down_proj", "model.layers.65.mlp.experts.109.down_proj", "model.layers.65.mlp.experts.110.down_proj", "model.layers.65.mlp.experts.111.down_proj", "model.layers.65.mlp.experts.112.down_proj", "model.layers.65.mlp.experts.113.down_proj", "model.layers.65.mlp.experts.114.down_proj", "model.layers.65.mlp.experts.115.down_proj", "model.layers.65.mlp.experts.116.down_proj", "model.layers.65.mlp.experts.117.down_proj", "model.layers.65.mlp.experts.118.down_proj", "model.layers.65.mlp.experts.119.down_proj", "model.layers.65.mlp.experts.120.down_proj", "model.layers.65.mlp.experts.121.down_proj", "model.layers.65.mlp.experts.122.down_proj", "model.layers.65.mlp.experts.123.down_proj", "model.layers.65.mlp.experts.124.down_proj", "model.layers.65.mlp.experts.125.down_proj", "model.layers.65.mlp.experts.126.down_proj", "model.layers.65.mlp.experts.127.down_proj", "model.layers.65.mlp.experts.128.down_proj", "model.layers.65.mlp.experts.129.down_proj", "model.layers.65.mlp.experts.130.down_proj", "model.layers.65.mlp.experts.131.down_proj", "model.layers.65.mlp.experts.132.down_proj", "model.layers.65.mlp.experts.133.down_proj", "model.layers.65.mlp.experts.134.down_proj", "model.layers.65.mlp.experts.135.down_proj", "model.layers.65.mlp.experts.136.down_proj", "model.layers.65.mlp.experts.137.down_proj", "model.layers.65.mlp.experts.138.down_proj", "model.layers.65.mlp.experts.139.down_proj", "model.layers.65.mlp.experts.140.down_proj", "model.layers.65.mlp.experts.141.down_proj", "model.layers.65.mlp.experts.142.down_proj", "model.layers.65.mlp.experts.143.down_proj", "model.layers.65.mlp.experts.144.down_proj", "model.layers.65.mlp.experts.145.down_proj", "model.layers.65.mlp.experts.146.down_proj", "model.layers.65.mlp.experts.147.down_proj", "model.layers.65.mlp.experts.148.down_proj", "model.layers.65.mlp.experts.149.down_proj", "model.layers.65.mlp.experts.150.down_proj", "model.layers.65.mlp.experts.151.down_proj", "model.layers.65.mlp.experts.152.down_proj", "model.layers.65.mlp.experts.153.down_proj", "model.layers.65.mlp.experts.154.down_proj", "model.layers.65.mlp.experts.155.down_proj", "model.layers.65.mlp.experts.156.down_proj", "model.layers.65.mlp.experts.157.down_proj", "model.layers.65.mlp.experts.158.down_proj", "model.layers.65.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00022432655096055187, "dbits": 1258291200 } ] }, { "idx": 393, "layers": [ "model.layers.66.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0036537908017635345, "dbits": 62914560 } ] }, { "idx": 394, "layers": [ "model.layers.66.self_attn.k_proj", "model.layers.66.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0008898675441741943, "dbits": 10485760 } ] }, { "idx": 395, "layers": [ "model.layers.66.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0006443977355957142, "dbits": 62914560 } ] }, { "idx": 396, "layers": [ "model.layers.66.mlp.shared_experts.gate_proj", "model.layers.66.mlp.shared_experts.up_proj", "model.layers.66.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0001929067075252533, "dbits": 23592960 } ] }, { "idx": 397, "layers": [ "model.layers.66.mlp.experts.0.gate_proj", "model.layers.66.mlp.experts.1.gate_proj", "model.layers.66.mlp.experts.2.gate_proj", "model.layers.66.mlp.experts.3.gate_proj", "model.layers.66.mlp.experts.4.gate_proj", "model.layers.66.mlp.experts.5.gate_proj", "model.layers.66.mlp.experts.6.gate_proj", "model.layers.66.mlp.experts.7.gate_proj", "model.layers.66.mlp.experts.8.gate_proj", "model.layers.66.mlp.experts.9.gate_proj", "model.layers.66.mlp.experts.10.gate_proj", "model.layers.66.mlp.experts.11.gate_proj", "model.layers.66.mlp.experts.12.gate_proj", "model.layers.66.mlp.experts.13.gate_proj", "model.layers.66.mlp.experts.14.gate_proj", "model.layers.66.mlp.experts.15.gate_proj", "model.layers.66.mlp.experts.16.gate_proj", "model.layers.66.mlp.experts.17.gate_proj", "model.layers.66.mlp.experts.18.gate_proj", "model.layers.66.mlp.experts.19.gate_proj", "model.layers.66.mlp.experts.20.gate_proj", "model.layers.66.mlp.experts.21.gate_proj", "model.layers.66.mlp.experts.22.gate_proj", "model.layers.66.mlp.experts.23.gate_proj", "model.layers.66.mlp.experts.24.gate_proj", "model.layers.66.mlp.experts.25.gate_proj", "model.layers.66.mlp.experts.26.gate_proj", "model.layers.66.mlp.experts.27.gate_proj", "model.layers.66.mlp.experts.28.gate_proj", "model.layers.66.mlp.experts.29.gate_proj", "model.layers.66.mlp.experts.30.gate_proj", "model.layers.66.mlp.experts.31.gate_proj", "model.layers.66.mlp.experts.32.gate_proj", "model.layers.66.mlp.experts.33.gate_proj", "model.layers.66.mlp.experts.34.gate_proj", "model.layers.66.mlp.experts.35.gate_proj", "model.layers.66.mlp.experts.36.gate_proj", "model.layers.66.mlp.experts.37.gate_proj", "model.layers.66.mlp.experts.38.gate_proj", "model.layers.66.mlp.experts.39.gate_proj", "model.layers.66.mlp.experts.40.gate_proj", "model.layers.66.mlp.experts.41.gate_proj", "model.layers.66.mlp.experts.42.gate_proj", "model.layers.66.mlp.experts.43.gate_proj", "model.layers.66.mlp.experts.44.gate_proj", "model.layers.66.mlp.experts.45.gate_proj", "model.layers.66.mlp.experts.46.gate_proj", "model.layers.66.mlp.experts.47.gate_proj", "model.layers.66.mlp.experts.48.gate_proj", "model.layers.66.mlp.experts.49.gate_proj", "model.layers.66.mlp.experts.50.gate_proj", "model.layers.66.mlp.experts.51.gate_proj", "model.layers.66.mlp.experts.52.gate_proj", "model.layers.66.mlp.experts.53.gate_proj", "model.layers.66.mlp.experts.54.gate_proj", "model.layers.66.mlp.experts.55.gate_proj", "model.layers.66.mlp.experts.56.gate_proj", "model.layers.66.mlp.experts.57.gate_proj", "model.layers.66.mlp.experts.58.gate_proj", "model.layers.66.mlp.experts.59.gate_proj", "model.layers.66.mlp.experts.60.gate_proj", "model.layers.66.mlp.experts.61.gate_proj", "model.layers.66.mlp.experts.62.gate_proj", "model.layers.66.mlp.experts.63.gate_proj", "model.layers.66.mlp.experts.64.gate_proj", "model.layers.66.mlp.experts.65.gate_proj", "model.layers.66.mlp.experts.66.gate_proj", "model.layers.66.mlp.experts.67.gate_proj", "model.layers.66.mlp.experts.68.gate_proj", "model.layers.66.mlp.experts.69.gate_proj", "model.layers.66.mlp.experts.70.gate_proj", "model.layers.66.mlp.experts.71.gate_proj", "model.layers.66.mlp.experts.72.gate_proj", "model.layers.66.mlp.experts.73.gate_proj", "model.layers.66.mlp.experts.74.gate_proj", "model.layers.66.mlp.experts.75.gate_proj", "model.layers.66.mlp.experts.76.gate_proj", "model.layers.66.mlp.experts.77.gate_proj", "model.layers.66.mlp.experts.78.gate_proj", "model.layers.66.mlp.experts.79.gate_proj", "model.layers.66.mlp.experts.80.gate_proj", "model.layers.66.mlp.experts.81.gate_proj", "model.layers.66.mlp.experts.82.gate_proj", "model.layers.66.mlp.experts.83.gate_proj", "model.layers.66.mlp.experts.84.gate_proj", "model.layers.66.mlp.experts.85.gate_proj", "model.layers.66.mlp.experts.86.gate_proj", "model.layers.66.mlp.experts.87.gate_proj", "model.layers.66.mlp.experts.88.gate_proj", "model.layers.66.mlp.experts.89.gate_proj", "model.layers.66.mlp.experts.90.gate_proj", "model.layers.66.mlp.experts.91.gate_proj", "model.layers.66.mlp.experts.92.gate_proj", "model.layers.66.mlp.experts.93.gate_proj", "model.layers.66.mlp.experts.94.gate_proj", "model.layers.66.mlp.experts.95.gate_proj", "model.layers.66.mlp.experts.96.gate_proj", "model.layers.66.mlp.experts.97.gate_proj", "model.layers.66.mlp.experts.98.gate_proj", "model.layers.66.mlp.experts.99.gate_proj", "model.layers.66.mlp.experts.100.gate_proj", "model.layers.66.mlp.experts.101.gate_proj", "model.layers.66.mlp.experts.102.gate_proj", "model.layers.66.mlp.experts.103.gate_proj", "model.layers.66.mlp.experts.104.gate_proj", "model.layers.66.mlp.experts.105.gate_proj", "model.layers.66.mlp.experts.106.gate_proj", "model.layers.66.mlp.experts.107.gate_proj", "model.layers.66.mlp.experts.108.gate_proj", "model.layers.66.mlp.experts.109.gate_proj", "model.layers.66.mlp.experts.110.gate_proj", "model.layers.66.mlp.experts.111.gate_proj", "model.layers.66.mlp.experts.112.gate_proj", "model.layers.66.mlp.experts.113.gate_proj", "model.layers.66.mlp.experts.114.gate_proj", "model.layers.66.mlp.experts.115.gate_proj", "model.layers.66.mlp.experts.116.gate_proj", "model.layers.66.mlp.experts.117.gate_proj", "model.layers.66.mlp.experts.118.gate_proj", "model.layers.66.mlp.experts.119.gate_proj", "model.layers.66.mlp.experts.120.gate_proj", "model.layers.66.mlp.experts.121.gate_proj", "model.layers.66.mlp.experts.122.gate_proj", "model.layers.66.mlp.experts.123.gate_proj", "model.layers.66.mlp.experts.124.gate_proj", "model.layers.66.mlp.experts.125.gate_proj", "model.layers.66.mlp.experts.126.gate_proj", "model.layers.66.mlp.experts.127.gate_proj", "model.layers.66.mlp.experts.128.gate_proj", "model.layers.66.mlp.experts.129.gate_proj", "model.layers.66.mlp.experts.130.gate_proj", "model.layers.66.mlp.experts.131.gate_proj", "model.layers.66.mlp.experts.132.gate_proj", "model.layers.66.mlp.experts.133.gate_proj", "model.layers.66.mlp.experts.134.gate_proj", "model.layers.66.mlp.experts.135.gate_proj", "model.layers.66.mlp.experts.136.gate_proj", "model.layers.66.mlp.experts.137.gate_proj", "model.layers.66.mlp.experts.138.gate_proj", "model.layers.66.mlp.experts.139.gate_proj", "model.layers.66.mlp.experts.140.gate_proj", "model.layers.66.mlp.experts.141.gate_proj", "model.layers.66.mlp.experts.142.gate_proj", "model.layers.66.mlp.experts.143.gate_proj", "model.layers.66.mlp.experts.144.gate_proj", "model.layers.66.mlp.experts.145.gate_proj", "model.layers.66.mlp.experts.146.gate_proj", "model.layers.66.mlp.experts.147.gate_proj", "model.layers.66.mlp.experts.148.gate_proj", "model.layers.66.mlp.experts.149.gate_proj", "model.layers.66.mlp.experts.150.gate_proj", "model.layers.66.mlp.experts.151.gate_proj", "model.layers.66.mlp.experts.152.gate_proj", "model.layers.66.mlp.experts.153.gate_proj", "model.layers.66.mlp.experts.154.gate_proj", "model.layers.66.mlp.experts.155.gate_proj", "model.layers.66.mlp.experts.156.gate_proj", "model.layers.66.mlp.experts.157.gate_proj", "model.layers.66.mlp.experts.158.gate_proj", "model.layers.66.mlp.experts.159.gate_proj", "model.layers.66.mlp.experts.0.up_proj", "model.layers.66.mlp.experts.1.up_proj", "model.layers.66.mlp.experts.2.up_proj", "model.layers.66.mlp.experts.3.up_proj", "model.layers.66.mlp.experts.4.up_proj", "model.layers.66.mlp.experts.5.up_proj", "model.layers.66.mlp.experts.6.up_proj", "model.layers.66.mlp.experts.7.up_proj", "model.layers.66.mlp.experts.8.up_proj", "model.layers.66.mlp.experts.9.up_proj", "model.layers.66.mlp.experts.10.up_proj", "model.layers.66.mlp.experts.11.up_proj", "model.layers.66.mlp.experts.12.up_proj", "model.layers.66.mlp.experts.13.up_proj", "model.layers.66.mlp.experts.14.up_proj", "model.layers.66.mlp.experts.15.up_proj", "model.layers.66.mlp.experts.16.up_proj", "model.layers.66.mlp.experts.17.up_proj", "model.layers.66.mlp.experts.18.up_proj", "model.layers.66.mlp.experts.19.up_proj", "model.layers.66.mlp.experts.20.up_proj", "model.layers.66.mlp.experts.21.up_proj", "model.layers.66.mlp.experts.22.up_proj", "model.layers.66.mlp.experts.23.up_proj", "model.layers.66.mlp.experts.24.up_proj", "model.layers.66.mlp.experts.25.up_proj", "model.layers.66.mlp.experts.26.up_proj", "model.layers.66.mlp.experts.27.up_proj", "model.layers.66.mlp.experts.28.up_proj", "model.layers.66.mlp.experts.29.up_proj", "model.layers.66.mlp.experts.30.up_proj", "model.layers.66.mlp.experts.31.up_proj", "model.layers.66.mlp.experts.32.up_proj", "model.layers.66.mlp.experts.33.up_proj", "model.layers.66.mlp.experts.34.up_proj", "model.layers.66.mlp.experts.35.up_proj", "model.layers.66.mlp.experts.36.up_proj", "model.layers.66.mlp.experts.37.up_proj", "model.layers.66.mlp.experts.38.up_proj", "model.layers.66.mlp.experts.39.up_proj", "model.layers.66.mlp.experts.40.up_proj", "model.layers.66.mlp.experts.41.up_proj", "model.layers.66.mlp.experts.42.up_proj", "model.layers.66.mlp.experts.43.up_proj", "model.layers.66.mlp.experts.44.up_proj", "model.layers.66.mlp.experts.45.up_proj", "model.layers.66.mlp.experts.46.up_proj", "model.layers.66.mlp.experts.47.up_proj", "model.layers.66.mlp.experts.48.up_proj", "model.layers.66.mlp.experts.49.up_proj", "model.layers.66.mlp.experts.50.up_proj", "model.layers.66.mlp.experts.51.up_proj", "model.layers.66.mlp.experts.52.up_proj", "model.layers.66.mlp.experts.53.up_proj", "model.layers.66.mlp.experts.54.up_proj", "model.layers.66.mlp.experts.55.up_proj", "model.layers.66.mlp.experts.56.up_proj", "model.layers.66.mlp.experts.57.up_proj", "model.layers.66.mlp.experts.58.up_proj", "model.layers.66.mlp.experts.59.up_proj", "model.layers.66.mlp.experts.60.up_proj", "model.layers.66.mlp.experts.61.up_proj", "model.layers.66.mlp.experts.62.up_proj", "model.layers.66.mlp.experts.63.up_proj", "model.layers.66.mlp.experts.64.up_proj", "model.layers.66.mlp.experts.65.up_proj", "model.layers.66.mlp.experts.66.up_proj", "model.layers.66.mlp.experts.67.up_proj", "model.layers.66.mlp.experts.68.up_proj", "model.layers.66.mlp.experts.69.up_proj", "model.layers.66.mlp.experts.70.up_proj", "model.layers.66.mlp.experts.71.up_proj", "model.layers.66.mlp.experts.72.up_proj", "model.layers.66.mlp.experts.73.up_proj", "model.layers.66.mlp.experts.74.up_proj", "model.layers.66.mlp.experts.75.up_proj", "model.layers.66.mlp.experts.76.up_proj", "model.layers.66.mlp.experts.77.up_proj", "model.layers.66.mlp.experts.78.up_proj", "model.layers.66.mlp.experts.79.up_proj", "model.layers.66.mlp.experts.80.up_proj", "model.layers.66.mlp.experts.81.up_proj", "model.layers.66.mlp.experts.82.up_proj", "model.layers.66.mlp.experts.83.up_proj", "model.layers.66.mlp.experts.84.up_proj", "model.layers.66.mlp.experts.85.up_proj", "model.layers.66.mlp.experts.86.up_proj", "model.layers.66.mlp.experts.87.up_proj", "model.layers.66.mlp.experts.88.up_proj", "model.layers.66.mlp.experts.89.up_proj", "model.layers.66.mlp.experts.90.up_proj", "model.layers.66.mlp.experts.91.up_proj", "model.layers.66.mlp.experts.92.up_proj", "model.layers.66.mlp.experts.93.up_proj", "model.layers.66.mlp.experts.94.up_proj", "model.layers.66.mlp.experts.95.up_proj", "model.layers.66.mlp.experts.96.up_proj", "model.layers.66.mlp.experts.97.up_proj", "model.layers.66.mlp.experts.98.up_proj", "model.layers.66.mlp.experts.99.up_proj", "model.layers.66.mlp.experts.100.up_proj", "model.layers.66.mlp.experts.101.up_proj", "model.layers.66.mlp.experts.102.up_proj", "model.layers.66.mlp.experts.103.up_proj", "model.layers.66.mlp.experts.104.up_proj", "model.layers.66.mlp.experts.105.up_proj", "model.layers.66.mlp.experts.106.up_proj", "model.layers.66.mlp.experts.107.up_proj", "model.layers.66.mlp.experts.108.up_proj", "model.layers.66.mlp.experts.109.up_proj", "model.layers.66.mlp.experts.110.up_proj", "model.layers.66.mlp.experts.111.up_proj", "model.layers.66.mlp.experts.112.up_proj", "model.layers.66.mlp.experts.113.up_proj", "model.layers.66.mlp.experts.114.up_proj", "model.layers.66.mlp.experts.115.up_proj", "model.layers.66.mlp.experts.116.up_proj", "model.layers.66.mlp.experts.117.up_proj", "model.layers.66.mlp.experts.118.up_proj", "model.layers.66.mlp.experts.119.up_proj", "model.layers.66.mlp.experts.120.up_proj", "model.layers.66.mlp.experts.121.up_proj", "model.layers.66.mlp.experts.122.up_proj", "model.layers.66.mlp.experts.123.up_proj", "model.layers.66.mlp.experts.124.up_proj", "model.layers.66.mlp.experts.125.up_proj", "model.layers.66.mlp.experts.126.up_proj", "model.layers.66.mlp.experts.127.up_proj", "model.layers.66.mlp.experts.128.up_proj", "model.layers.66.mlp.experts.129.up_proj", "model.layers.66.mlp.experts.130.up_proj", "model.layers.66.mlp.experts.131.up_proj", "model.layers.66.mlp.experts.132.up_proj", "model.layers.66.mlp.experts.133.up_proj", "model.layers.66.mlp.experts.134.up_proj", "model.layers.66.mlp.experts.135.up_proj", "model.layers.66.mlp.experts.136.up_proj", "model.layers.66.mlp.experts.137.up_proj", "model.layers.66.mlp.experts.138.up_proj", "model.layers.66.mlp.experts.139.up_proj", "model.layers.66.mlp.experts.140.up_proj", "model.layers.66.mlp.experts.141.up_proj", "model.layers.66.mlp.experts.142.up_proj", "model.layers.66.mlp.experts.143.up_proj", "model.layers.66.mlp.experts.144.up_proj", "model.layers.66.mlp.experts.145.up_proj", "model.layers.66.mlp.experts.146.up_proj", "model.layers.66.mlp.experts.147.up_proj", "model.layers.66.mlp.experts.148.up_proj", "model.layers.66.mlp.experts.149.up_proj", "model.layers.66.mlp.experts.150.up_proj", "model.layers.66.mlp.experts.151.up_proj", "model.layers.66.mlp.experts.152.up_proj", "model.layers.66.mlp.experts.153.up_proj", "model.layers.66.mlp.experts.154.up_proj", "model.layers.66.mlp.experts.155.up_proj", "model.layers.66.mlp.experts.156.up_proj", "model.layers.66.mlp.experts.157.up_proj", "model.layers.66.mlp.experts.158.up_proj", "model.layers.66.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00036853179335594177, "dbits": 2516582400 } ] }, { "idx": 398, "layers": [ "model.layers.66.mlp.experts.0.down_proj", "model.layers.66.mlp.experts.1.down_proj", "model.layers.66.mlp.experts.2.down_proj", "model.layers.66.mlp.experts.3.down_proj", "model.layers.66.mlp.experts.4.down_proj", "model.layers.66.mlp.experts.5.down_proj", "model.layers.66.mlp.experts.6.down_proj", "model.layers.66.mlp.experts.7.down_proj", "model.layers.66.mlp.experts.8.down_proj", "model.layers.66.mlp.experts.9.down_proj", "model.layers.66.mlp.experts.10.down_proj", "model.layers.66.mlp.experts.11.down_proj", "model.layers.66.mlp.experts.12.down_proj", "model.layers.66.mlp.experts.13.down_proj", "model.layers.66.mlp.experts.14.down_proj", "model.layers.66.mlp.experts.15.down_proj", "model.layers.66.mlp.experts.16.down_proj", "model.layers.66.mlp.experts.17.down_proj", "model.layers.66.mlp.experts.18.down_proj", "model.layers.66.mlp.experts.19.down_proj", "model.layers.66.mlp.experts.20.down_proj", "model.layers.66.mlp.experts.21.down_proj", "model.layers.66.mlp.experts.22.down_proj", "model.layers.66.mlp.experts.23.down_proj", "model.layers.66.mlp.experts.24.down_proj", "model.layers.66.mlp.experts.25.down_proj", "model.layers.66.mlp.experts.26.down_proj", "model.layers.66.mlp.experts.27.down_proj", "model.layers.66.mlp.experts.28.down_proj", "model.layers.66.mlp.experts.29.down_proj", "model.layers.66.mlp.experts.30.down_proj", "model.layers.66.mlp.experts.31.down_proj", "model.layers.66.mlp.experts.32.down_proj", "model.layers.66.mlp.experts.33.down_proj", "model.layers.66.mlp.experts.34.down_proj", "model.layers.66.mlp.experts.35.down_proj", "model.layers.66.mlp.experts.36.down_proj", "model.layers.66.mlp.experts.37.down_proj", "model.layers.66.mlp.experts.38.down_proj", "model.layers.66.mlp.experts.39.down_proj", "model.layers.66.mlp.experts.40.down_proj", "model.layers.66.mlp.experts.41.down_proj", "model.layers.66.mlp.experts.42.down_proj", "model.layers.66.mlp.experts.43.down_proj", "model.layers.66.mlp.experts.44.down_proj", "model.layers.66.mlp.experts.45.down_proj", "model.layers.66.mlp.experts.46.down_proj", "model.layers.66.mlp.experts.47.down_proj", "model.layers.66.mlp.experts.48.down_proj", "model.layers.66.mlp.experts.49.down_proj", "model.layers.66.mlp.experts.50.down_proj", "model.layers.66.mlp.experts.51.down_proj", "model.layers.66.mlp.experts.52.down_proj", "model.layers.66.mlp.experts.53.down_proj", "model.layers.66.mlp.experts.54.down_proj", "model.layers.66.mlp.experts.55.down_proj", "model.layers.66.mlp.experts.56.down_proj", "model.layers.66.mlp.experts.57.down_proj", "model.layers.66.mlp.experts.58.down_proj", "model.layers.66.mlp.experts.59.down_proj", "model.layers.66.mlp.experts.60.down_proj", "model.layers.66.mlp.experts.61.down_proj", "model.layers.66.mlp.experts.62.down_proj", "model.layers.66.mlp.experts.63.down_proj", "model.layers.66.mlp.experts.64.down_proj", "model.layers.66.mlp.experts.65.down_proj", "model.layers.66.mlp.experts.66.down_proj", "model.layers.66.mlp.experts.67.down_proj", "model.layers.66.mlp.experts.68.down_proj", "model.layers.66.mlp.experts.69.down_proj", "model.layers.66.mlp.experts.70.down_proj", "model.layers.66.mlp.experts.71.down_proj", "model.layers.66.mlp.experts.72.down_proj", "model.layers.66.mlp.experts.73.down_proj", "model.layers.66.mlp.experts.74.down_proj", "model.layers.66.mlp.experts.75.down_proj", "model.layers.66.mlp.experts.76.down_proj", "model.layers.66.mlp.experts.77.down_proj", "model.layers.66.mlp.experts.78.down_proj", "model.layers.66.mlp.experts.79.down_proj", "model.layers.66.mlp.experts.80.down_proj", "model.layers.66.mlp.experts.81.down_proj", "model.layers.66.mlp.experts.82.down_proj", "model.layers.66.mlp.experts.83.down_proj", "model.layers.66.mlp.experts.84.down_proj", "model.layers.66.mlp.experts.85.down_proj", "model.layers.66.mlp.experts.86.down_proj", "model.layers.66.mlp.experts.87.down_proj", "model.layers.66.mlp.experts.88.down_proj", "model.layers.66.mlp.experts.89.down_proj", "model.layers.66.mlp.experts.90.down_proj", "model.layers.66.mlp.experts.91.down_proj", "model.layers.66.mlp.experts.92.down_proj", "model.layers.66.mlp.experts.93.down_proj", "model.layers.66.mlp.experts.94.down_proj", "model.layers.66.mlp.experts.95.down_proj", "model.layers.66.mlp.experts.96.down_proj", "model.layers.66.mlp.experts.97.down_proj", "model.layers.66.mlp.experts.98.down_proj", "model.layers.66.mlp.experts.99.down_proj", "model.layers.66.mlp.experts.100.down_proj", "model.layers.66.mlp.experts.101.down_proj", "model.layers.66.mlp.experts.102.down_proj", "model.layers.66.mlp.experts.103.down_proj", "model.layers.66.mlp.experts.104.down_proj", "model.layers.66.mlp.experts.105.down_proj", "model.layers.66.mlp.experts.106.down_proj", "model.layers.66.mlp.experts.107.down_proj", "model.layers.66.mlp.experts.108.down_proj", "model.layers.66.mlp.experts.109.down_proj", "model.layers.66.mlp.experts.110.down_proj", "model.layers.66.mlp.experts.111.down_proj", "model.layers.66.mlp.experts.112.down_proj", "model.layers.66.mlp.experts.113.down_proj", "model.layers.66.mlp.experts.114.down_proj", "model.layers.66.mlp.experts.115.down_proj", "model.layers.66.mlp.experts.116.down_proj", "model.layers.66.mlp.experts.117.down_proj", "model.layers.66.mlp.experts.118.down_proj", "model.layers.66.mlp.experts.119.down_proj", "model.layers.66.mlp.experts.120.down_proj", "model.layers.66.mlp.experts.121.down_proj", "model.layers.66.mlp.experts.122.down_proj", "model.layers.66.mlp.experts.123.down_proj", "model.layers.66.mlp.experts.124.down_proj", "model.layers.66.mlp.experts.125.down_proj", "model.layers.66.mlp.experts.126.down_proj", "model.layers.66.mlp.experts.127.down_proj", "model.layers.66.mlp.experts.128.down_proj", "model.layers.66.mlp.experts.129.down_proj", "model.layers.66.mlp.experts.130.down_proj", "model.layers.66.mlp.experts.131.down_proj", "model.layers.66.mlp.experts.132.down_proj", "model.layers.66.mlp.experts.133.down_proj", "model.layers.66.mlp.experts.134.down_proj", "model.layers.66.mlp.experts.135.down_proj", "model.layers.66.mlp.experts.136.down_proj", "model.layers.66.mlp.experts.137.down_proj", "model.layers.66.mlp.experts.138.down_proj", "model.layers.66.mlp.experts.139.down_proj", "model.layers.66.mlp.experts.140.down_proj", "model.layers.66.mlp.experts.141.down_proj", "model.layers.66.mlp.experts.142.down_proj", "model.layers.66.mlp.experts.143.down_proj", "model.layers.66.mlp.experts.144.down_proj", "model.layers.66.mlp.experts.145.down_proj", "model.layers.66.mlp.experts.146.down_proj", "model.layers.66.mlp.experts.147.down_proj", "model.layers.66.mlp.experts.148.down_proj", "model.layers.66.mlp.experts.149.down_proj", "model.layers.66.mlp.experts.150.down_proj", "model.layers.66.mlp.experts.151.down_proj", "model.layers.66.mlp.experts.152.down_proj", "model.layers.66.mlp.experts.153.down_proj", "model.layers.66.mlp.experts.154.down_proj", "model.layers.66.mlp.experts.155.down_proj", "model.layers.66.mlp.experts.156.down_proj", "model.layers.66.mlp.experts.157.down_proj", "model.layers.66.mlp.experts.158.down_proj", "model.layers.66.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00035042017698289074, "dbits": 1258291200 } ] }, { "idx": 399, "layers": [ "model.layers.67.self_attn.q_proj" ], "candidates": [ { "dkld": -0.000824353098869357, "dbits": 62914560 } ] }, { "idx": 400, "layers": [ "model.layers.67.self_attn.k_proj", "model.layers.67.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0003929913043975719, "dbits": 10485760 } ] }, { "idx": 401, "layers": [ "model.layers.67.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00016532465815544128, "dbits": 62914560 } ] }, { "idx": 402, "layers": [ "model.layers.67.mlp.shared_experts.gate_proj", "model.layers.67.mlp.shared_experts.up_proj", "model.layers.67.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0004263363778591045, "dbits": 23592960 } ] }, { "idx": 403, "layers": [ "model.layers.67.mlp.experts.0.gate_proj", "model.layers.67.mlp.experts.1.gate_proj", "model.layers.67.mlp.experts.2.gate_proj", "model.layers.67.mlp.experts.3.gate_proj", "model.layers.67.mlp.experts.4.gate_proj", "model.layers.67.mlp.experts.5.gate_proj", "model.layers.67.mlp.experts.6.gate_proj", "model.layers.67.mlp.experts.7.gate_proj", "model.layers.67.mlp.experts.8.gate_proj", "model.layers.67.mlp.experts.9.gate_proj", "model.layers.67.mlp.experts.10.gate_proj", "model.layers.67.mlp.experts.11.gate_proj", "model.layers.67.mlp.experts.12.gate_proj", "model.layers.67.mlp.experts.13.gate_proj", "model.layers.67.mlp.experts.14.gate_proj", "model.layers.67.mlp.experts.15.gate_proj", "model.layers.67.mlp.experts.16.gate_proj", "model.layers.67.mlp.experts.17.gate_proj", "model.layers.67.mlp.experts.18.gate_proj", "model.layers.67.mlp.experts.19.gate_proj", "model.layers.67.mlp.experts.20.gate_proj", "model.layers.67.mlp.experts.21.gate_proj", "model.layers.67.mlp.experts.22.gate_proj", "model.layers.67.mlp.experts.23.gate_proj", "model.layers.67.mlp.experts.24.gate_proj", "model.layers.67.mlp.experts.25.gate_proj", "model.layers.67.mlp.experts.26.gate_proj", "model.layers.67.mlp.experts.27.gate_proj", "model.layers.67.mlp.experts.28.gate_proj", "model.layers.67.mlp.experts.29.gate_proj", "model.layers.67.mlp.experts.30.gate_proj", "model.layers.67.mlp.experts.31.gate_proj", "model.layers.67.mlp.experts.32.gate_proj", "model.layers.67.mlp.experts.33.gate_proj", "model.layers.67.mlp.experts.34.gate_proj", "model.layers.67.mlp.experts.35.gate_proj", "model.layers.67.mlp.experts.36.gate_proj", "model.layers.67.mlp.experts.37.gate_proj", "model.layers.67.mlp.experts.38.gate_proj", "model.layers.67.mlp.experts.39.gate_proj", "model.layers.67.mlp.experts.40.gate_proj", "model.layers.67.mlp.experts.41.gate_proj", "model.layers.67.mlp.experts.42.gate_proj", "model.layers.67.mlp.experts.43.gate_proj", "model.layers.67.mlp.experts.44.gate_proj", "model.layers.67.mlp.experts.45.gate_proj", "model.layers.67.mlp.experts.46.gate_proj", "model.layers.67.mlp.experts.47.gate_proj", "model.layers.67.mlp.experts.48.gate_proj", "model.layers.67.mlp.experts.49.gate_proj", "model.layers.67.mlp.experts.50.gate_proj", "model.layers.67.mlp.experts.51.gate_proj", "model.layers.67.mlp.experts.52.gate_proj", "model.layers.67.mlp.experts.53.gate_proj", "model.layers.67.mlp.experts.54.gate_proj", "model.layers.67.mlp.experts.55.gate_proj", "model.layers.67.mlp.experts.56.gate_proj", "model.layers.67.mlp.experts.57.gate_proj", "model.layers.67.mlp.experts.58.gate_proj", "model.layers.67.mlp.experts.59.gate_proj", "model.layers.67.mlp.experts.60.gate_proj", "model.layers.67.mlp.experts.61.gate_proj", "model.layers.67.mlp.experts.62.gate_proj", "model.layers.67.mlp.experts.63.gate_proj", "model.layers.67.mlp.experts.64.gate_proj", "model.layers.67.mlp.experts.65.gate_proj", "model.layers.67.mlp.experts.66.gate_proj", "model.layers.67.mlp.experts.67.gate_proj", "model.layers.67.mlp.experts.68.gate_proj", "model.layers.67.mlp.experts.69.gate_proj", "model.layers.67.mlp.experts.70.gate_proj", "model.layers.67.mlp.experts.71.gate_proj", "model.layers.67.mlp.experts.72.gate_proj", "model.layers.67.mlp.experts.73.gate_proj", "model.layers.67.mlp.experts.74.gate_proj", "model.layers.67.mlp.experts.75.gate_proj", "model.layers.67.mlp.experts.76.gate_proj", "model.layers.67.mlp.experts.77.gate_proj", "model.layers.67.mlp.experts.78.gate_proj", "model.layers.67.mlp.experts.79.gate_proj", "model.layers.67.mlp.experts.80.gate_proj", "model.layers.67.mlp.experts.81.gate_proj", "model.layers.67.mlp.experts.82.gate_proj", "model.layers.67.mlp.experts.83.gate_proj", "model.layers.67.mlp.experts.84.gate_proj", "model.layers.67.mlp.experts.85.gate_proj", "model.layers.67.mlp.experts.86.gate_proj", "model.layers.67.mlp.experts.87.gate_proj", "model.layers.67.mlp.experts.88.gate_proj", "model.layers.67.mlp.experts.89.gate_proj", "model.layers.67.mlp.experts.90.gate_proj", "model.layers.67.mlp.experts.91.gate_proj", "model.layers.67.mlp.experts.92.gate_proj", "model.layers.67.mlp.experts.93.gate_proj", "model.layers.67.mlp.experts.94.gate_proj", "model.layers.67.mlp.experts.95.gate_proj", "model.layers.67.mlp.experts.96.gate_proj", "model.layers.67.mlp.experts.97.gate_proj", "model.layers.67.mlp.experts.98.gate_proj", "model.layers.67.mlp.experts.99.gate_proj", "model.layers.67.mlp.experts.100.gate_proj", "model.layers.67.mlp.experts.101.gate_proj", "model.layers.67.mlp.experts.102.gate_proj", "model.layers.67.mlp.experts.103.gate_proj", "model.layers.67.mlp.experts.104.gate_proj", "model.layers.67.mlp.experts.105.gate_proj", "model.layers.67.mlp.experts.106.gate_proj", "model.layers.67.mlp.experts.107.gate_proj", "model.layers.67.mlp.experts.108.gate_proj", "model.layers.67.mlp.experts.109.gate_proj", "model.layers.67.mlp.experts.110.gate_proj", "model.layers.67.mlp.experts.111.gate_proj", "model.layers.67.mlp.experts.112.gate_proj", "model.layers.67.mlp.experts.113.gate_proj", "model.layers.67.mlp.experts.114.gate_proj", "model.layers.67.mlp.experts.115.gate_proj", "model.layers.67.mlp.experts.116.gate_proj", "model.layers.67.mlp.experts.117.gate_proj", "model.layers.67.mlp.experts.118.gate_proj", "model.layers.67.mlp.experts.119.gate_proj", "model.layers.67.mlp.experts.120.gate_proj", "model.layers.67.mlp.experts.121.gate_proj", "model.layers.67.mlp.experts.122.gate_proj", "model.layers.67.mlp.experts.123.gate_proj", "model.layers.67.mlp.experts.124.gate_proj", "model.layers.67.mlp.experts.125.gate_proj", "model.layers.67.mlp.experts.126.gate_proj", "model.layers.67.mlp.experts.127.gate_proj", "model.layers.67.mlp.experts.128.gate_proj", "model.layers.67.mlp.experts.129.gate_proj", "model.layers.67.mlp.experts.130.gate_proj", "model.layers.67.mlp.experts.131.gate_proj", "model.layers.67.mlp.experts.132.gate_proj", "model.layers.67.mlp.experts.133.gate_proj", "model.layers.67.mlp.experts.134.gate_proj", "model.layers.67.mlp.experts.135.gate_proj", "model.layers.67.mlp.experts.136.gate_proj", "model.layers.67.mlp.experts.137.gate_proj", "model.layers.67.mlp.experts.138.gate_proj", "model.layers.67.mlp.experts.139.gate_proj", "model.layers.67.mlp.experts.140.gate_proj", "model.layers.67.mlp.experts.141.gate_proj", "model.layers.67.mlp.experts.142.gate_proj", "model.layers.67.mlp.experts.143.gate_proj", "model.layers.67.mlp.experts.144.gate_proj", "model.layers.67.mlp.experts.145.gate_proj", "model.layers.67.mlp.experts.146.gate_proj", "model.layers.67.mlp.experts.147.gate_proj", "model.layers.67.mlp.experts.148.gate_proj", "model.layers.67.mlp.experts.149.gate_proj", "model.layers.67.mlp.experts.150.gate_proj", "model.layers.67.mlp.experts.151.gate_proj", "model.layers.67.mlp.experts.152.gate_proj", "model.layers.67.mlp.experts.153.gate_proj", "model.layers.67.mlp.experts.154.gate_proj", "model.layers.67.mlp.experts.155.gate_proj", "model.layers.67.mlp.experts.156.gate_proj", "model.layers.67.mlp.experts.157.gate_proj", "model.layers.67.mlp.experts.158.gate_proj", "model.layers.67.mlp.experts.159.gate_proj", "model.layers.67.mlp.experts.0.up_proj", "model.layers.67.mlp.experts.1.up_proj", "model.layers.67.mlp.experts.2.up_proj", "model.layers.67.mlp.experts.3.up_proj", "model.layers.67.mlp.experts.4.up_proj", "model.layers.67.mlp.experts.5.up_proj", "model.layers.67.mlp.experts.6.up_proj", "model.layers.67.mlp.experts.7.up_proj", "model.layers.67.mlp.experts.8.up_proj", "model.layers.67.mlp.experts.9.up_proj", "model.layers.67.mlp.experts.10.up_proj", "model.layers.67.mlp.experts.11.up_proj", "model.layers.67.mlp.experts.12.up_proj", "model.layers.67.mlp.experts.13.up_proj", "model.layers.67.mlp.experts.14.up_proj", "model.layers.67.mlp.experts.15.up_proj", "model.layers.67.mlp.experts.16.up_proj", "model.layers.67.mlp.experts.17.up_proj", "model.layers.67.mlp.experts.18.up_proj", "model.layers.67.mlp.experts.19.up_proj", "model.layers.67.mlp.experts.20.up_proj", "model.layers.67.mlp.experts.21.up_proj", "model.layers.67.mlp.experts.22.up_proj", "model.layers.67.mlp.experts.23.up_proj", "model.layers.67.mlp.experts.24.up_proj", "model.layers.67.mlp.experts.25.up_proj", "model.layers.67.mlp.experts.26.up_proj", "model.layers.67.mlp.experts.27.up_proj", "model.layers.67.mlp.experts.28.up_proj", "model.layers.67.mlp.experts.29.up_proj", "model.layers.67.mlp.experts.30.up_proj", "model.layers.67.mlp.experts.31.up_proj", "model.layers.67.mlp.experts.32.up_proj", "model.layers.67.mlp.experts.33.up_proj", "model.layers.67.mlp.experts.34.up_proj", "model.layers.67.mlp.experts.35.up_proj", "model.layers.67.mlp.experts.36.up_proj", "model.layers.67.mlp.experts.37.up_proj", "model.layers.67.mlp.experts.38.up_proj", "model.layers.67.mlp.experts.39.up_proj", "model.layers.67.mlp.experts.40.up_proj", "model.layers.67.mlp.experts.41.up_proj", "model.layers.67.mlp.experts.42.up_proj", "model.layers.67.mlp.experts.43.up_proj", "model.layers.67.mlp.experts.44.up_proj", "model.layers.67.mlp.experts.45.up_proj", "model.layers.67.mlp.experts.46.up_proj", "model.layers.67.mlp.experts.47.up_proj", "model.layers.67.mlp.experts.48.up_proj", "model.layers.67.mlp.experts.49.up_proj", "model.layers.67.mlp.experts.50.up_proj", "model.layers.67.mlp.experts.51.up_proj", "model.layers.67.mlp.experts.52.up_proj", "model.layers.67.mlp.experts.53.up_proj", "model.layers.67.mlp.experts.54.up_proj", "model.layers.67.mlp.experts.55.up_proj", "model.layers.67.mlp.experts.56.up_proj", "model.layers.67.mlp.experts.57.up_proj", "model.layers.67.mlp.experts.58.up_proj", "model.layers.67.mlp.experts.59.up_proj", "model.layers.67.mlp.experts.60.up_proj", "model.layers.67.mlp.experts.61.up_proj", "model.layers.67.mlp.experts.62.up_proj", "model.layers.67.mlp.experts.63.up_proj", "model.layers.67.mlp.experts.64.up_proj", "model.layers.67.mlp.experts.65.up_proj", "model.layers.67.mlp.experts.66.up_proj", "model.layers.67.mlp.experts.67.up_proj", "model.layers.67.mlp.experts.68.up_proj", "model.layers.67.mlp.experts.69.up_proj", "model.layers.67.mlp.experts.70.up_proj", "model.layers.67.mlp.experts.71.up_proj", "model.layers.67.mlp.experts.72.up_proj", "model.layers.67.mlp.experts.73.up_proj", "model.layers.67.mlp.experts.74.up_proj", "model.layers.67.mlp.experts.75.up_proj", "model.layers.67.mlp.experts.76.up_proj", "model.layers.67.mlp.experts.77.up_proj", "model.layers.67.mlp.experts.78.up_proj", "model.layers.67.mlp.experts.79.up_proj", "model.layers.67.mlp.experts.80.up_proj", "model.layers.67.mlp.experts.81.up_proj", "model.layers.67.mlp.experts.82.up_proj", "model.layers.67.mlp.experts.83.up_proj", "model.layers.67.mlp.experts.84.up_proj", "model.layers.67.mlp.experts.85.up_proj", "model.layers.67.mlp.experts.86.up_proj", "model.layers.67.mlp.experts.87.up_proj", "model.layers.67.mlp.experts.88.up_proj", "model.layers.67.mlp.experts.89.up_proj", "model.layers.67.mlp.experts.90.up_proj", "model.layers.67.mlp.experts.91.up_proj", "model.layers.67.mlp.experts.92.up_proj", "model.layers.67.mlp.experts.93.up_proj", "model.layers.67.mlp.experts.94.up_proj", "model.layers.67.mlp.experts.95.up_proj", "model.layers.67.mlp.experts.96.up_proj", "model.layers.67.mlp.experts.97.up_proj", "model.layers.67.mlp.experts.98.up_proj", "model.layers.67.mlp.experts.99.up_proj", "model.layers.67.mlp.experts.100.up_proj", "model.layers.67.mlp.experts.101.up_proj", "model.layers.67.mlp.experts.102.up_proj", "model.layers.67.mlp.experts.103.up_proj", "model.layers.67.mlp.experts.104.up_proj", "model.layers.67.mlp.experts.105.up_proj", "model.layers.67.mlp.experts.106.up_proj", "model.layers.67.mlp.experts.107.up_proj", "model.layers.67.mlp.experts.108.up_proj", "model.layers.67.mlp.experts.109.up_proj", "model.layers.67.mlp.experts.110.up_proj", "model.layers.67.mlp.experts.111.up_proj", "model.layers.67.mlp.experts.112.up_proj", "model.layers.67.mlp.experts.113.up_proj", "model.layers.67.mlp.experts.114.up_proj", "model.layers.67.mlp.experts.115.up_proj", "model.layers.67.mlp.experts.116.up_proj", "model.layers.67.mlp.experts.117.up_proj", "model.layers.67.mlp.experts.118.up_proj", "model.layers.67.mlp.experts.119.up_proj", "model.layers.67.mlp.experts.120.up_proj", "model.layers.67.mlp.experts.121.up_proj", "model.layers.67.mlp.experts.122.up_proj", "model.layers.67.mlp.experts.123.up_proj", "model.layers.67.mlp.experts.124.up_proj", "model.layers.67.mlp.experts.125.up_proj", "model.layers.67.mlp.experts.126.up_proj", "model.layers.67.mlp.experts.127.up_proj", "model.layers.67.mlp.experts.128.up_proj", "model.layers.67.mlp.experts.129.up_proj", "model.layers.67.mlp.experts.130.up_proj", "model.layers.67.mlp.experts.131.up_proj", "model.layers.67.mlp.experts.132.up_proj", "model.layers.67.mlp.experts.133.up_proj", "model.layers.67.mlp.experts.134.up_proj", "model.layers.67.mlp.experts.135.up_proj", "model.layers.67.mlp.experts.136.up_proj", "model.layers.67.mlp.experts.137.up_proj", "model.layers.67.mlp.experts.138.up_proj", "model.layers.67.mlp.experts.139.up_proj", "model.layers.67.mlp.experts.140.up_proj", "model.layers.67.mlp.experts.141.up_proj", "model.layers.67.mlp.experts.142.up_proj", "model.layers.67.mlp.experts.143.up_proj", "model.layers.67.mlp.experts.144.up_proj", "model.layers.67.mlp.experts.145.up_proj", "model.layers.67.mlp.experts.146.up_proj", "model.layers.67.mlp.experts.147.up_proj", "model.layers.67.mlp.experts.148.up_proj", "model.layers.67.mlp.experts.149.up_proj", "model.layers.67.mlp.experts.150.up_proj", "model.layers.67.mlp.experts.151.up_proj", "model.layers.67.mlp.experts.152.up_proj", "model.layers.67.mlp.experts.153.up_proj", "model.layers.67.mlp.experts.154.up_proj", "model.layers.67.mlp.experts.155.up_proj", "model.layers.67.mlp.experts.156.up_proj", "model.layers.67.mlp.experts.157.up_proj", "model.layers.67.mlp.experts.158.up_proj", "model.layers.67.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 5.954727530477211e-05, "dbits": 2516582400 } ] }, { "idx": 404, "layers": [ "model.layers.67.mlp.experts.0.down_proj", "model.layers.67.mlp.experts.1.down_proj", "model.layers.67.mlp.experts.2.down_proj", "model.layers.67.mlp.experts.3.down_proj", "model.layers.67.mlp.experts.4.down_proj", "model.layers.67.mlp.experts.5.down_proj", "model.layers.67.mlp.experts.6.down_proj", "model.layers.67.mlp.experts.7.down_proj", "model.layers.67.mlp.experts.8.down_proj", "model.layers.67.mlp.experts.9.down_proj", "model.layers.67.mlp.experts.10.down_proj", "model.layers.67.mlp.experts.11.down_proj", "model.layers.67.mlp.experts.12.down_proj", "model.layers.67.mlp.experts.13.down_proj", "model.layers.67.mlp.experts.14.down_proj", "model.layers.67.mlp.experts.15.down_proj", "model.layers.67.mlp.experts.16.down_proj", "model.layers.67.mlp.experts.17.down_proj", "model.layers.67.mlp.experts.18.down_proj", "model.layers.67.mlp.experts.19.down_proj", "model.layers.67.mlp.experts.20.down_proj", "model.layers.67.mlp.experts.21.down_proj", "model.layers.67.mlp.experts.22.down_proj", "model.layers.67.mlp.experts.23.down_proj", "model.layers.67.mlp.experts.24.down_proj", "model.layers.67.mlp.experts.25.down_proj", "model.layers.67.mlp.experts.26.down_proj", "model.layers.67.mlp.experts.27.down_proj", "model.layers.67.mlp.experts.28.down_proj", "model.layers.67.mlp.experts.29.down_proj", "model.layers.67.mlp.experts.30.down_proj", "model.layers.67.mlp.experts.31.down_proj", "model.layers.67.mlp.experts.32.down_proj", "model.layers.67.mlp.experts.33.down_proj", "model.layers.67.mlp.experts.34.down_proj", "model.layers.67.mlp.experts.35.down_proj", "model.layers.67.mlp.experts.36.down_proj", "model.layers.67.mlp.experts.37.down_proj", "model.layers.67.mlp.experts.38.down_proj", "model.layers.67.mlp.experts.39.down_proj", "model.layers.67.mlp.experts.40.down_proj", "model.layers.67.mlp.experts.41.down_proj", "model.layers.67.mlp.experts.42.down_proj", "model.layers.67.mlp.experts.43.down_proj", "model.layers.67.mlp.experts.44.down_proj", "model.layers.67.mlp.experts.45.down_proj", "model.layers.67.mlp.experts.46.down_proj", "model.layers.67.mlp.experts.47.down_proj", "model.layers.67.mlp.experts.48.down_proj", "model.layers.67.mlp.experts.49.down_proj", "model.layers.67.mlp.experts.50.down_proj", "model.layers.67.mlp.experts.51.down_proj", "model.layers.67.mlp.experts.52.down_proj", "model.layers.67.mlp.experts.53.down_proj", "model.layers.67.mlp.experts.54.down_proj", "model.layers.67.mlp.experts.55.down_proj", "model.layers.67.mlp.experts.56.down_proj", "model.layers.67.mlp.experts.57.down_proj", "model.layers.67.mlp.experts.58.down_proj", "model.layers.67.mlp.experts.59.down_proj", "model.layers.67.mlp.experts.60.down_proj", "model.layers.67.mlp.experts.61.down_proj", "model.layers.67.mlp.experts.62.down_proj", "model.layers.67.mlp.experts.63.down_proj", "model.layers.67.mlp.experts.64.down_proj", "model.layers.67.mlp.experts.65.down_proj", "model.layers.67.mlp.experts.66.down_proj", "model.layers.67.mlp.experts.67.down_proj", "model.layers.67.mlp.experts.68.down_proj", "model.layers.67.mlp.experts.69.down_proj", "model.layers.67.mlp.experts.70.down_proj", "model.layers.67.mlp.experts.71.down_proj", "model.layers.67.mlp.experts.72.down_proj", "model.layers.67.mlp.experts.73.down_proj", "model.layers.67.mlp.experts.74.down_proj", "model.layers.67.mlp.experts.75.down_proj", "model.layers.67.mlp.experts.76.down_proj", "model.layers.67.mlp.experts.77.down_proj", "model.layers.67.mlp.experts.78.down_proj", "model.layers.67.mlp.experts.79.down_proj", "model.layers.67.mlp.experts.80.down_proj", "model.layers.67.mlp.experts.81.down_proj", "model.layers.67.mlp.experts.82.down_proj", "model.layers.67.mlp.experts.83.down_proj", "model.layers.67.mlp.experts.84.down_proj", "model.layers.67.mlp.experts.85.down_proj", "model.layers.67.mlp.experts.86.down_proj", "model.layers.67.mlp.experts.87.down_proj", "model.layers.67.mlp.experts.88.down_proj", "model.layers.67.mlp.experts.89.down_proj", "model.layers.67.mlp.experts.90.down_proj", "model.layers.67.mlp.experts.91.down_proj", "model.layers.67.mlp.experts.92.down_proj", "model.layers.67.mlp.experts.93.down_proj", "model.layers.67.mlp.experts.94.down_proj", "model.layers.67.mlp.experts.95.down_proj", "model.layers.67.mlp.experts.96.down_proj", "model.layers.67.mlp.experts.97.down_proj", "model.layers.67.mlp.experts.98.down_proj", "model.layers.67.mlp.experts.99.down_proj", "model.layers.67.mlp.experts.100.down_proj", "model.layers.67.mlp.experts.101.down_proj", "model.layers.67.mlp.experts.102.down_proj", "model.layers.67.mlp.experts.103.down_proj", "model.layers.67.mlp.experts.104.down_proj", "model.layers.67.mlp.experts.105.down_proj", "model.layers.67.mlp.experts.106.down_proj", "model.layers.67.mlp.experts.107.down_proj", "model.layers.67.mlp.experts.108.down_proj", "model.layers.67.mlp.experts.109.down_proj", "model.layers.67.mlp.experts.110.down_proj", "model.layers.67.mlp.experts.111.down_proj", "model.layers.67.mlp.experts.112.down_proj", "model.layers.67.mlp.experts.113.down_proj", "model.layers.67.mlp.experts.114.down_proj", "model.layers.67.mlp.experts.115.down_proj", "model.layers.67.mlp.experts.116.down_proj", "model.layers.67.mlp.experts.117.down_proj", "model.layers.67.mlp.experts.118.down_proj", "model.layers.67.mlp.experts.119.down_proj", "model.layers.67.mlp.experts.120.down_proj", "model.layers.67.mlp.experts.121.down_proj", "model.layers.67.mlp.experts.122.down_proj", "model.layers.67.mlp.experts.123.down_proj", "model.layers.67.mlp.experts.124.down_proj", "model.layers.67.mlp.experts.125.down_proj", "model.layers.67.mlp.experts.126.down_proj", "model.layers.67.mlp.experts.127.down_proj", "model.layers.67.mlp.experts.128.down_proj", "model.layers.67.mlp.experts.129.down_proj", "model.layers.67.mlp.experts.130.down_proj", "model.layers.67.mlp.experts.131.down_proj", "model.layers.67.mlp.experts.132.down_proj", "model.layers.67.mlp.experts.133.down_proj", "model.layers.67.mlp.experts.134.down_proj", "model.layers.67.mlp.experts.135.down_proj", "model.layers.67.mlp.experts.136.down_proj", "model.layers.67.mlp.experts.137.down_proj", "model.layers.67.mlp.experts.138.down_proj", "model.layers.67.mlp.experts.139.down_proj", "model.layers.67.mlp.experts.140.down_proj", "model.layers.67.mlp.experts.141.down_proj", "model.layers.67.mlp.experts.142.down_proj", "model.layers.67.mlp.experts.143.down_proj", "model.layers.67.mlp.experts.144.down_proj", "model.layers.67.mlp.experts.145.down_proj", "model.layers.67.mlp.experts.146.down_proj", "model.layers.67.mlp.experts.147.down_proj", "model.layers.67.mlp.experts.148.down_proj", "model.layers.67.mlp.experts.149.down_proj", "model.layers.67.mlp.experts.150.down_proj", "model.layers.67.mlp.experts.151.down_proj", "model.layers.67.mlp.experts.152.down_proj", "model.layers.67.mlp.experts.153.down_proj", "model.layers.67.mlp.experts.154.down_proj", "model.layers.67.mlp.experts.155.down_proj", "model.layers.67.mlp.experts.156.down_proj", "model.layers.67.mlp.experts.157.down_proj", "model.layers.67.mlp.experts.158.down_proj", "model.layers.67.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -7.831007242203869e-05, "dbits": 1258291200 } ] }, { "idx": 405, "layers": [ "model.layers.68.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0004702240228652843, "dbits": 62914560 } ] }, { "idx": 406, "layers": [ "model.layers.68.self_attn.k_proj", "model.layers.68.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0004800453782081715, "dbits": 10485760 } ] }, { "idx": 407, "layers": [ "model.layers.68.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00047646909952164807, "dbits": 62914560 } ] }, { "idx": 408, "layers": [ "model.layers.68.mlp.shared_experts.gate_proj", "model.layers.68.mlp.shared_experts.up_proj", "model.layers.68.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0004512093961239194, "dbits": 23592960 } ] }, { "idx": 409, "layers": [ "model.layers.68.mlp.experts.0.gate_proj", "model.layers.68.mlp.experts.1.gate_proj", "model.layers.68.mlp.experts.2.gate_proj", "model.layers.68.mlp.experts.3.gate_proj", "model.layers.68.mlp.experts.4.gate_proj", "model.layers.68.mlp.experts.5.gate_proj", "model.layers.68.mlp.experts.6.gate_proj", "model.layers.68.mlp.experts.7.gate_proj", "model.layers.68.mlp.experts.8.gate_proj", "model.layers.68.mlp.experts.9.gate_proj", "model.layers.68.mlp.experts.10.gate_proj", "model.layers.68.mlp.experts.11.gate_proj", "model.layers.68.mlp.experts.12.gate_proj", "model.layers.68.mlp.experts.13.gate_proj", "model.layers.68.mlp.experts.14.gate_proj", "model.layers.68.mlp.experts.15.gate_proj", "model.layers.68.mlp.experts.16.gate_proj", "model.layers.68.mlp.experts.17.gate_proj", "model.layers.68.mlp.experts.18.gate_proj", "model.layers.68.mlp.experts.19.gate_proj", "model.layers.68.mlp.experts.20.gate_proj", "model.layers.68.mlp.experts.21.gate_proj", "model.layers.68.mlp.experts.22.gate_proj", "model.layers.68.mlp.experts.23.gate_proj", "model.layers.68.mlp.experts.24.gate_proj", "model.layers.68.mlp.experts.25.gate_proj", "model.layers.68.mlp.experts.26.gate_proj", "model.layers.68.mlp.experts.27.gate_proj", "model.layers.68.mlp.experts.28.gate_proj", "model.layers.68.mlp.experts.29.gate_proj", "model.layers.68.mlp.experts.30.gate_proj", "model.layers.68.mlp.experts.31.gate_proj", "model.layers.68.mlp.experts.32.gate_proj", "model.layers.68.mlp.experts.33.gate_proj", "model.layers.68.mlp.experts.34.gate_proj", "model.layers.68.mlp.experts.35.gate_proj", "model.layers.68.mlp.experts.36.gate_proj", "model.layers.68.mlp.experts.37.gate_proj", "model.layers.68.mlp.experts.38.gate_proj", "model.layers.68.mlp.experts.39.gate_proj", "model.layers.68.mlp.experts.40.gate_proj", "model.layers.68.mlp.experts.41.gate_proj", "model.layers.68.mlp.experts.42.gate_proj", "model.layers.68.mlp.experts.43.gate_proj", "model.layers.68.mlp.experts.44.gate_proj", "model.layers.68.mlp.experts.45.gate_proj", "model.layers.68.mlp.experts.46.gate_proj", "model.layers.68.mlp.experts.47.gate_proj", "model.layers.68.mlp.experts.48.gate_proj", "model.layers.68.mlp.experts.49.gate_proj", "model.layers.68.mlp.experts.50.gate_proj", "model.layers.68.mlp.experts.51.gate_proj", "model.layers.68.mlp.experts.52.gate_proj", "model.layers.68.mlp.experts.53.gate_proj", "model.layers.68.mlp.experts.54.gate_proj", "model.layers.68.mlp.experts.55.gate_proj", "model.layers.68.mlp.experts.56.gate_proj", "model.layers.68.mlp.experts.57.gate_proj", "model.layers.68.mlp.experts.58.gate_proj", "model.layers.68.mlp.experts.59.gate_proj", "model.layers.68.mlp.experts.60.gate_proj", "model.layers.68.mlp.experts.61.gate_proj", "model.layers.68.mlp.experts.62.gate_proj", "model.layers.68.mlp.experts.63.gate_proj", "model.layers.68.mlp.experts.64.gate_proj", "model.layers.68.mlp.experts.65.gate_proj", "model.layers.68.mlp.experts.66.gate_proj", "model.layers.68.mlp.experts.67.gate_proj", "model.layers.68.mlp.experts.68.gate_proj", "model.layers.68.mlp.experts.69.gate_proj", "model.layers.68.mlp.experts.70.gate_proj", "model.layers.68.mlp.experts.71.gate_proj", "model.layers.68.mlp.experts.72.gate_proj", "model.layers.68.mlp.experts.73.gate_proj", "model.layers.68.mlp.experts.74.gate_proj", "model.layers.68.mlp.experts.75.gate_proj", "model.layers.68.mlp.experts.76.gate_proj", "model.layers.68.mlp.experts.77.gate_proj", "model.layers.68.mlp.experts.78.gate_proj", "model.layers.68.mlp.experts.79.gate_proj", "model.layers.68.mlp.experts.80.gate_proj", "model.layers.68.mlp.experts.81.gate_proj", "model.layers.68.mlp.experts.82.gate_proj", "model.layers.68.mlp.experts.83.gate_proj", "model.layers.68.mlp.experts.84.gate_proj", "model.layers.68.mlp.experts.85.gate_proj", "model.layers.68.mlp.experts.86.gate_proj", "model.layers.68.mlp.experts.87.gate_proj", "model.layers.68.mlp.experts.88.gate_proj", "model.layers.68.mlp.experts.89.gate_proj", "model.layers.68.mlp.experts.90.gate_proj", "model.layers.68.mlp.experts.91.gate_proj", "model.layers.68.mlp.experts.92.gate_proj", "model.layers.68.mlp.experts.93.gate_proj", "model.layers.68.mlp.experts.94.gate_proj", "model.layers.68.mlp.experts.95.gate_proj", "model.layers.68.mlp.experts.96.gate_proj", "model.layers.68.mlp.experts.97.gate_proj", "model.layers.68.mlp.experts.98.gate_proj", "model.layers.68.mlp.experts.99.gate_proj", "model.layers.68.mlp.experts.100.gate_proj", "model.layers.68.mlp.experts.101.gate_proj", "model.layers.68.mlp.experts.102.gate_proj", "model.layers.68.mlp.experts.103.gate_proj", "model.layers.68.mlp.experts.104.gate_proj", "model.layers.68.mlp.experts.105.gate_proj", "model.layers.68.mlp.experts.106.gate_proj", "model.layers.68.mlp.experts.107.gate_proj", "model.layers.68.mlp.experts.108.gate_proj", "model.layers.68.mlp.experts.109.gate_proj", "model.layers.68.mlp.experts.110.gate_proj", "model.layers.68.mlp.experts.111.gate_proj", "model.layers.68.mlp.experts.112.gate_proj", "model.layers.68.mlp.experts.113.gate_proj", "model.layers.68.mlp.experts.114.gate_proj", "model.layers.68.mlp.experts.115.gate_proj", "model.layers.68.mlp.experts.116.gate_proj", "model.layers.68.mlp.experts.117.gate_proj", "model.layers.68.mlp.experts.118.gate_proj", "model.layers.68.mlp.experts.119.gate_proj", "model.layers.68.mlp.experts.120.gate_proj", "model.layers.68.mlp.experts.121.gate_proj", "model.layers.68.mlp.experts.122.gate_proj", "model.layers.68.mlp.experts.123.gate_proj", "model.layers.68.mlp.experts.124.gate_proj", "model.layers.68.mlp.experts.125.gate_proj", "model.layers.68.mlp.experts.126.gate_proj", "model.layers.68.mlp.experts.127.gate_proj", "model.layers.68.mlp.experts.128.gate_proj", "model.layers.68.mlp.experts.129.gate_proj", "model.layers.68.mlp.experts.130.gate_proj", "model.layers.68.mlp.experts.131.gate_proj", "model.layers.68.mlp.experts.132.gate_proj", "model.layers.68.mlp.experts.133.gate_proj", "model.layers.68.mlp.experts.134.gate_proj", "model.layers.68.mlp.experts.135.gate_proj", "model.layers.68.mlp.experts.136.gate_proj", "model.layers.68.mlp.experts.137.gate_proj", "model.layers.68.mlp.experts.138.gate_proj", "model.layers.68.mlp.experts.139.gate_proj", "model.layers.68.mlp.experts.140.gate_proj", "model.layers.68.mlp.experts.141.gate_proj", "model.layers.68.mlp.experts.142.gate_proj", "model.layers.68.mlp.experts.143.gate_proj", "model.layers.68.mlp.experts.144.gate_proj", "model.layers.68.mlp.experts.145.gate_proj", "model.layers.68.mlp.experts.146.gate_proj", "model.layers.68.mlp.experts.147.gate_proj", "model.layers.68.mlp.experts.148.gate_proj", "model.layers.68.mlp.experts.149.gate_proj", "model.layers.68.mlp.experts.150.gate_proj", "model.layers.68.mlp.experts.151.gate_proj", "model.layers.68.mlp.experts.152.gate_proj", "model.layers.68.mlp.experts.153.gate_proj", "model.layers.68.mlp.experts.154.gate_proj", "model.layers.68.mlp.experts.155.gate_proj", "model.layers.68.mlp.experts.156.gate_proj", "model.layers.68.mlp.experts.157.gate_proj", "model.layers.68.mlp.experts.158.gate_proj", "model.layers.68.mlp.experts.159.gate_proj", "model.layers.68.mlp.experts.0.up_proj", "model.layers.68.mlp.experts.1.up_proj", "model.layers.68.mlp.experts.2.up_proj", "model.layers.68.mlp.experts.3.up_proj", "model.layers.68.mlp.experts.4.up_proj", "model.layers.68.mlp.experts.5.up_proj", "model.layers.68.mlp.experts.6.up_proj", "model.layers.68.mlp.experts.7.up_proj", "model.layers.68.mlp.experts.8.up_proj", "model.layers.68.mlp.experts.9.up_proj", "model.layers.68.mlp.experts.10.up_proj", "model.layers.68.mlp.experts.11.up_proj", "model.layers.68.mlp.experts.12.up_proj", "model.layers.68.mlp.experts.13.up_proj", "model.layers.68.mlp.experts.14.up_proj", "model.layers.68.mlp.experts.15.up_proj", "model.layers.68.mlp.experts.16.up_proj", "model.layers.68.mlp.experts.17.up_proj", "model.layers.68.mlp.experts.18.up_proj", "model.layers.68.mlp.experts.19.up_proj", "model.layers.68.mlp.experts.20.up_proj", "model.layers.68.mlp.experts.21.up_proj", "model.layers.68.mlp.experts.22.up_proj", "model.layers.68.mlp.experts.23.up_proj", "model.layers.68.mlp.experts.24.up_proj", "model.layers.68.mlp.experts.25.up_proj", "model.layers.68.mlp.experts.26.up_proj", "model.layers.68.mlp.experts.27.up_proj", "model.layers.68.mlp.experts.28.up_proj", "model.layers.68.mlp.experts.29.up_proj", "model.layers.68.mlp.experts.30.up_proj", "model.layers.68.mlp.experts.31.up_proj", "model.layers.68.mlp.experts.32.up_proj", "model.layers.68.mlp.experts.33.up_proj", "model.layers.68.mlp.experts.34.up_proj", "model.layers.68.mlp.experts.35.up_proj", "model.layers.68.mlp.experts.36.up_proj", "model.layers.68.mlp.experts.37.up_proj", "model.layers.68.mlp.experts.38.up_proj", "model.layers.68.mlp.experts.39.up_proj", "model.layers.68.mlp.experts.40.up_proj", "model.layers.68.mlp.experts.41.up_proj", "model.layers.68.mlp.experts.42.up_proj", "model.layers.68.mlp.experts.43.up_proj", "model.layers.68.mlp.experts.44.up_proj", "model.layers.68.mlp.experts.45.up_proj", "model.layers.68.mlp.experts.46.up_proj", "model.layers.68.mlp.experts.47.up_proj", "model.layers.68.mlp.experts.48.up_proj", "model.layers.68.mlp.experts.49.up_proj", "model.layers.68.mlp.experts.50.up_proj", "model.layers.68.mlp.experts.51.up_proj", "model.layers.68.mlp.experts.52.up_proj", "model.layers.68.mlp.experts.53.up_proj", "model.layers.68.mlp.experts.54.up_proj", "model.layers.68.mlp.experts.55.up_proj", "model.layers.68.mlp.experts.56.up_proj", "model.layers.68.mlp.experts.57.up_proj", "model.layers.68.mlp.experts.58.up_proj", "model.layers.68.mlp.experts.59.up_proj", "model.layers.68.mlp.experts.60.up_proj", "model.layers.68.mlp.experts.61.up_proj", "model.layers.68.mlp.experts.62.up_proj", "model.layers.68.mlp.experts.63.up_proj", "model.layers.68.mlp.experts.64.up_proj", "model.layers.68.mlp.experts.65.up_proj", "model.layers.68.mlp.experts.66.up_proj", "model.layers.68.mlp.experts.67.up_proj", "model.layers.68.mlp.experts.68.up_proj", "model.layers.68.mlp.experts.69.up_proj", "model.layers.68.mlp.experts.70.up_proj", "model.layers.68.mlp.experts.71.up_proj", "model.layers.68.mlp.experts.72.up_proj", "model.layers.68.mlp.experts.73.up_proj", "model.layers.68.mlp.experts.74.up_proj", "model.layers.68.mlp.experts.75.up_proj", "model.layers.68.mlp.experts.76.up_proj", "model.layers.68.mlp.experts.77.up_proj", "model.layers.68.mlp.experts.78.up_proj", "model.layers.68.mlp.experts.79.up_proj", "model.layers.68.mlp.experts.80.up_proj", "model.layers.68.mlp.experts.81.up_proj", "model.layers.68.mlp.experts.82.up_proj", "model.layers.68.mlp.experts.83.up_proj", "model.layers.68.mlp.experts.84.up_proj", "model.layers.68.mlp.experts.85.up_proj", "model.layers.68.mlp.experts.86.up_proj", "model.layers.68.mlp.experts.87.up_proj", "model.layers.68.mlp.experts.88.up_proj", "model.layers.68.mlp.experts.89.up_proj", "model.layers.68.mlp.experts.90.up_proj", "model.layers.68.mlp.experts.91.up_proj", "model.layers.68.mlp.experts.92.up_proj", "model.layers.68.mlp.experts.93.up_proj", "model.layers.68.mlp.experts.94.up_proj", "model.layers.68.mlp.experts.95.up_proj", "model.layers.68.mlp.experts.96.up_proj", "model.layers.68.mlp.experts.97.up_proj", "model.layers.68.mlp.experts.98.up_proj", "model.layers.68.mlp.experts.99.up_proj", "model.layers.68.mlp.experts.100.up_proj", "model.layers.68.mlp.experts.101.up_proj", "model.layers.68.mlp.experts.102.up_proj", "model.layers.68.mlp.experts.103.up_proj", "model.layers.68.mlp.experts.104.up_proj", "model.layers.68.mlp.experts.105.up_proj", "model.layers.68.mlp.experts.106.up_proj", "model.layers.68.mlp.experts.107.up_proj", "model.layers.68.mlp.experts.108.up_proj", "model.layers.68.mlp.experts.109.up_proj", "model.layers.68.mlp.experts.110.up_proj", "model.layers.68.mlp.experts.111.up_proj", "model.layers.68.mlp.experts.112.up_proj", "model.layers.68.mlp.experts.113.up_proj", "model.layers.68.mlp.experts.114.up_proj", "model.layers.68.mlp.experts.115.up_proj", "model.layers.68.mlp.experts.116.up_proj", "model.layers.68.mlp.experts.117.up_proj", "model.layers.68.mlp.experts.118.up_proj", "model.layers.68.mlp.experts.119.up_proj", "model.layers.68.mlp.experts.120.up_proj", "model.layers.68.mlp.experts.121.up_proj", "model.layers.68.mlp.experts.122.up_proj", "model.layers.68.mlp.experts.123.up_proj", "model.layers.68.mlp.experts.124.up_proj", "model.layers.68.mlp.experts.125.up_proj", "model.layers.68.mlp.experts.126.up_proj", "model.layers.68.mlp.experts.127.up_proj", "model.layers.68.mlp.experts.128.up_proj", "model.layers.68.mlp.experts.129.up_proj", "model.layers.68.mlp.experts.130.up_proj", "model.layers.68.mlp.experts.131.up_proj", "model.layers.68.mlp.experts.132.up_proj", "model.layers.68.mlp.experts.133.up_proj", "model.layers.68.mlp.experts.134.up_proj", "model.layers.68.mlp.experts.135.up_proj", "model.layers.68.mlp.experts.136.up_proj", "model.layers.68.mlp.experts.137.up_proj", "model.layers.68.mlp.experts.138.up_proj", "model.layers.68.mlp.experts.139.up_proj", "model.layers.68.mlp.experts.140.up_proj", "model.layers.68.mlp.experts.141.up_proj", "model.layers.68.mlp.experts.142.up_proj", "model.layers.68.mlp.experts.143.up_proj", "model.layers.68.mlp.experts.144.up_proj", "model.layers.68.mlp.experts.145.up_proj", "model.layers.68.mlp.experts.146.up_proj", "model.layers.68.mlp.experts.147.up_proj", "model.layers.68.mlp.experts.148.up_proj", "model.layers.68.mlp.experts.149.up_proj", "model.layers.68.mlp.experts.150.up_proj", "model.layers.68.mlp.experts.151.up_proj", "model.layers.68.mlp.experts.152.up_proj", "model.layers.68.mlp.experts.153.up_proj", "model.layers.68.mlp.experts.154.up_proj", "model.layers.68.mlp.experts.155.up_proj", "model.layers.68.mlp.experts.156.up_proj", "model.layers.68.mlp.experts.157.up_proj", "model.layers.68.mlp.experts.158.up_proj", "model.layers.68.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 1.7294287681546283e-05, "dbits": 2516582400 } ] }, { "idx": 410, "layers": [ "model.layers.68.mlp.experts.0.down_proj", "model.layers.68.mlp.experts.1.down_proj", "model.layers.68.mlp.experts.2.down_proj", "model.layers.68.mlp.experts.3.down_proj", "model.layers.68.mlp.experts.4.down_proj", "model.layers.68.mlp.experts.5.down_proj", "model.layers.68.mlp.experts.6.down_proj", "model.layers.68.mlp.experts.7.down_proj", "model.layers.68.mlp.experts.8.down_proj", "model.layers.68.mlp.experts.9.down_proj", "model.layers.68.mlp.experts.10.down_proj", "model.layers.68.mlp.experts.11.down_proj", "model.layers.68.mlp.experts.12.down_proj", "model.layers.68.mlp.experts.13.down_proj", "model.layers.68.mlp.experts.14.down_proj", "model.layers.68.mlp.experts.15.down_proj", "model.layers.68.mlp.experts.16.down_proj", "model.layers.68.mlp.experts.17.down_proj", "model.layers.68.mlp.experts.18.down_proj", "model.layers.68.mlp.experts.19.down_proj", "model.layers.68.mlp.experts.20.down_proj", "model.layers.68.mlp.experts.21.down_proj", "model.layers.68.mlp.experts.22.down_proj", "model.layers.68.mlp.experts.23.down_proj", "model.layers.68.mlp.experts.24.down_proj", "model.layers.68.mlp.experts.25.down_proj", "model.layers.68.mlp.experts.26.down_proj", "model.layers.68.mlp.experts.27.down_proj", "model.layers.68.mlp.experts.28.down_proj", "model.layers.68.mlp.experts.29.down_proj", "model.layers.68.mlp.experts.30.down_proj", "model.layers.68.mlp.experts.31.down_proj", "model.layers.68.mlp.experts.32.down_proj", "model.layers.68.mlp.experts.33.down_proj", "model.layers.68.mlp.experts.34.down_proj", "model.layers.68.mlp.experts.35.down_proj", "model.layers.68.mlp.experts.36.down_proj", "model.layers.68.mlp.experts.37.down_proj", "model.layers.68.mlp.experts.38.down_proj", "model.layers.68.mlp.experts.39.down_proj", "model.layers.68.mlp.experts.40.down_proj", "model.layers.68.mlp.experts.41.down_proj", "model.layers.68.mlp.experts.42.down_proj", "model.layers.68.mlp.experts.43.down_proj", "model.layers.68.mlp.experts.44.down_proj", "model.layers.68.mlp.experts.45.down_proj", "model.layers.68.mlp.experts.46.down_proj", "model.layers.68.mlp.experts.47.down_proj", "model.layers.68.mlp.experts.48.down_proj", "model.layers.68.mlp.experts.49.down_proj", "model.layers.68.mlp.experts.50.down_proj", "model.layers.68.mlp.experts.51.down_proj", "model.layers.68.mlp.experts.52.down_proj", "model.layers.68.mlp.experts.53.down_proj", "model.layers.68.mlp.experts.54.down_proj", "model.layers.68.mlp.experts.55.down_proj", "model.layers.68.mlp.experts.56.down_proj", "model.layers.68.mlp.experts.57.down_proj", "model.layers.68.mlp.experts.58.down_proj", "model.layers.68.mlp.experts.59.down_proj", "model.layers.68.mlp.experts.60.down_proj", "model.layers.68.mlp.experts.61.down_proj", "model.layers.68.mlp.experts.62.down_proj", "model.layers.68.mlp.experts.63.down_proj", "model.layers.68.mlp.experts.64.down_proj", "model.layers.68.mlp.experts.65.down_proj", "model.layers.68.mlp.experts.66.down_proj", "model.layers.68.mlp.experts.67.down_proj", "model.layers.68.mlp.experts.68.down_proj", "model.layers.68.mlp.experts.69.down_proj", "model.layers.68.mlp.experts.70.down_proj", "model.layers.68.mlp.experts.71.down_proj", "model.layers.68.mlp.experts.72.down_proj", "model.layers.68.mlp.experts.73.down_proj", "model.layers.68.mlp.experts.74.down_proj", "model.layers.68.mlp.experts.75.down_proj", "model.layers.68.mlp.experts.76.down_proj", "model.layers.68.mlp.experts.77.down_proj", "model.layers.68.mlp.experts.78.down_proj", "model.layers.68.mlp.experts.79.down_proj", "model.layers.68.mlp.experts.80.down_proj", "model.layers.68.mlp.experts.81.down_proj", "model.layers.68.mlp.experts.82.down_proj", "model.layers.68.mlp.experts.83.down_proj", "model.layers.68.mlp.experts.84.down_proj", "model.layers.68.mlp.experts.85.down_proj", "model.layers.68.mlp.experts.86.down_proj", "model.layers.68.mlp.experts.87.down_proj", "model.layers.68.mlp.experts.88.down_proj", "model.layers.68.mlp.experts.89.down_proj", "model.layers.68.mlp.experts.90.down_proj", "model.layers.68.mlp.experts.91.down_proj", "model.layers.68.mlp.experts.92.down_proj", "model.layers.68.mlp.experts.93.down_proj", "model.layers.68.mlp.experts.94.down_proj", "model.layers.68.mlp.experts.95.down_proj", "model.layers.68.mlp.experts.96.down_proj", "model.layers.68.mlp.experts.97.down_proj", "model.layers.68.mlp.experts.98.down_proj", "model.layers.68.mlp.experts.99.down_proj", "model.layers.68.mlp.experts.100.down_proj", "model.layers.68.mlp.experts.101.down_proj", "model.layers.68.mlp.experts.102.down_proj", "model.layers.68.mlp.experts.103.down_proj", "model.layers.68.mlp.experts.104.down_proj", "model.layers.68.mlp.experts.105.down_proj", "model.layers.68.mlp.experts.106.down_proj", "model.layers.68.mlp.experts.107.down_proj", "model.layers.68.mlp.experts.108.down_proj", "model.layers.68.mlp.experts.109.down_proj", "model.layers.68.mlp.experts.110.down_proj", "model.layers.68.mlp.experts.111.down_proj", "model.layers.68.mlp.experts.112.down_proj", "model.layers.68.mlp.experts.113.down_proj", "model.layers.68.mlp.experts.114.down_proj", "model.layers.68.mlp.experts.115.down_proj", "model.layers.68.mlp.experts.116.down_proj", "model.layers.68.mlp.experts.117.down_proj", "model.layers.68.mlp.experts.118.down_proj", "model.layers.68.mlp.experts.119.down_proj", "model.layers.68.mlp.experts.120.down_proj", "model.layers.68.mlp.experts.121.down_proj", "model.layers.68.mlp.experts.122.down_proj", "model.layers.68.mlp.experts.123.down_proj", "model.layers.68.mlp.experts.124.down_proj", "model.layers.68.mlp.experts.125.down_proj", "model.layers.68.mlp.experts.126.down_proj", "model.layers.68.mlp.experts.127.down_proj", "model.layers.68.mlp.experts.128.down_proj", "model.layers.68.mlp.experts.129.down_proj", "model.layers.68.mlp.experts.130.down_proj", "model.layers.68.mlp.experts.131.down_proj", "model.layers.68.mlp.experts.132.down_proj", "model.layers.68.mlp.experts.133.down_proj", "model.layers.68.mlp.experts.134.down_proj", "model.layers.68.mlp.experts.135.down_proj", "model.layers.68.mlp.experts.136.down_proj", "model.layers.68.mlp.experts.137.down_proj", "model.layers.68.mlp.experts.138.down_proj", "model.layers.68.mlp.experts.139.down_proj", "model.layers.68.mlp.experts.140.down_proj", "model.layers.68.mlp.experts.141.down_proj", "model.layers.68.mlp.experts.142.down_proj", "model.layers.68.mlp.experts.143.down_proj", "model.layers.68.mlp.experts.144.down_proj", "model.layers.68.mlp.experts.145.down_proj", "model.layers.68.mlp.experts.146.down_proj", "model.layers.68.mlp.experts.147.down_proj", "model.layers.68.mlp.experts.148.down_proj", "model.layers.68.mlp.experts.149.down_proj", "model.layers.68.mlp.experts.150.down_proj", "model.layers.68.mlp.experts.151.down_proj", "model.layers.68.mlp.experts.152.down_proj", "model.layers.68.mlp.experts.153.down_proj", "model.layers.68.mlp.experts.154.down_proj", "model.layers.68.mlp.experts.155.down_proj", "model.layers.68.mlp.experts.156.down_proj", "model.layers.68.mlp.experts.157.down_proj", "model.layers.68.mlp.experts.158.down_proj", "model.layers.68.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00039631351828578465, "dbits": 1258291200 } ] }, { "idx": 411, "layers": [ "model.layers.69.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00022650361061099522, "dbits": 62914560 } ] }, { "idx": 412, "layers": [ "model.layers.69.self_attn.k_proj", "model.layers.69.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0007200241088867188, "dbits": 10485760 } ] }, { "idx": 413, "layers": [ "model.layers.69.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00032303184270859875, "dbits": 62914560 } ] }, { "idx": 414, "layers": [ "model.layers.69.mlp.shared_experts.gate_proj", "model.layers.69.mlp.shared_experts.up_proj", "model.layers.69.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.00018563419580458485, "dbits": 23592960 } ] }, { "idx": 415, "layers": [ "model.layers.69.mlp.experts.0.gate_proj", "model.layers.69.mlp.experts.1.gate_proj", "model.layers.69.mlp.experts.2.gate_proj", "model.layers.69.mlp.experts.3.gate_proj", "model.layers.69.mlp.experts.4.gate_proj", "model.layers.69.mlp.experts.5.gate_proj", "model.layers.69.mlp.experts.6.gate_proj", "model.layers.69.mlp.experts.7.gate_proj", "model.layers.69.mlp.experts.8.gate_proj", "model.layers.69.mlp.experts.9.gate_proj", "model.layers.69.mlp.experts.10.gate_proj", "model.layers.69.mlp.experts.11.gate_proj", "model.layers.69.mlp.experts.12.gate_proj", "model.layers.69.mlp.experts.13.gate_proj", "model.layers.69.mlp.experts.14.gate_proj", "model.layers.69.mlp.experts.15.gate_proj", "model.layers.69.mlp.experts.16.gate_proj", "model.layers.69.mlp.experts.17.gate_proj", "model.layers.69.mlp.experts.18.gate_proj", "model.layers.69.mlp.experts.19.gate_proj", "model.layers.69.mlp.experts.20.gate_proj", "model.layers.69.mlp.experts.21.gate_proj", "model.layers.69.mlp.experts.22.gate_proj", "model.layers.69.mlp.experts.23.gate_proj", "model.layers.69.mlp.experts.24.gate_proj", "model.layers.69.mlp.experts.25.gate_proj", "model.layers.69.mlp.experts.26.gate_proj", "model.layers.69.mlp.experts.27.gate_proj", "model.layers.69.mlp.experts.28.gate_proj", "model.layers.69.mlp.experts.29.gate_proj", "model.layers.69.mlp.experts.30.gate_proj", "model.layers.69.mlp.experts.31.gate_proj", "model.layers.69.mlp.experts.32.gate_proj", "model.layers.69.mlp.experts.33.gate_proj", "model.layers.69.mlp.experts.34.gate_proj", "model.layers.69.mlp.experts.35.gate_proj", "model.layers.69.mlp.experts.36.gate_proj", "model.layers.69.mlp.experts.37.gate_proj", "model.layers.69.mlp.experts.38.gate_proj", "model.layers.69.mlp.experts.39.gate_proj", "model.layers.69.mlp.experts.40.gate_proj", "model.layers.69.mlp.experts.41.gate_proj", "model.layers.69.mlp.experts.42.gate_proj", "model.layers.69.mlp.experts.43.gate_proj", "model.layers.69.mlp.experts.44.gate_proj", "model.layers.69.mlp.experts.45.gate_proj", "model.layers.69.mlp.experts.46.gate_proj", "model.layers.69.mlp.experts.47.gate_proj", "model.layers.69.mlp.experts.48.gate_proj", "model.layers.69.mlp.experts.49.gate_proj", "model.layers.69.mlp.experts.50.gate_proj", "model.layers.69.mlp.experts.51.gate_proj", "model.layers.69.mlp.experts.52.gate_proj", "model.layers.69.mlp.experts.53.gate_proj", "model.layers.69.mlp.experts.54.gate_proj", "model.layers.69.mlp.experts.55.gate_proj", "model.layers.69.mlp.experts.56.gate_proj", "model.layers.69.mlp.experts.57.gate_proj", "model.layers.69.mlp.experts.58.gate_proj", "model.layers.69.mlp.experts.59.gate_proj", "model.layers.69.mlp.experts.60.gate_proj", "model.layers.69.mlp.experts.61.gate_proj", "model.layers.69.mlp.experts.62.gate_proj", "model.layers.69.mlp.experts.63.gate_proj", "model.layers.69.mlp.experts.64.gate_proj", "model.layers.69.mlp.experts.65.gate_proj", "model.layers.69.mlp.experts.66.gate_proj", "model.layers.69.mlp.experts.67.gate_proj", "model.layers.69.mlp.experts.68.gate_proj", "model.layers.69.mlp.experts.69.gate_proj", "model.layers.69.mlp.experts.70.gate_proj", "model.layers.69.mlp.experts.71.gate_proj", "model.layers.69.mlp.experts.72.gate_proj", "model.layers.69.mlp.experts.73.gate_proj", "model.layers.69.mlp.experts.74.gate_proj", "model.layers.69.mlp.experts.75.gate_proj", "model.layers.69.mlp.experts.76.gate_proj", "model.layers.69.mlp.experts.77.gate_proj", "model.layers.69.mlp.experts.78.gate_proj", "model.layers.69.mlp.experts.79.gate_proj", "model.layers.69.mlp.experts.80.gate_proj", "model.layers.69.mlp.experts.81.gate_proj", "model.layers.69.mlp.experts.82.gate_proj", "model.layers.69.mlp.experts.83.gate_proj", "model.layers.69.mlp.experts.84.gate_proj", "model.layers.69.mlp.experts.85.gate_proj", "model.layers.69.mlp.experts.86.gate_proj", "model.layers.69.mlp.experts.87.gate_proj", "model.layers.69.mlp.experts.88.gate_proj", "model.layers.69.mlp.experts.89.gate_proj", "model.layers.69.mlp.experts.90.gate_proj", "model.layers.69.mlp.experts.91.gate_proj", "model.layers.69.mlp.experts.92.gate_proj", "model.layers.69.mlp.experts.93.gate_proj", "model.layers.69.mlp.experts.94.gate_proj", "model.layers.69.mlp.experts.95.gate_proj", "model.layers.69.mlp.experts.96.gate_proj", "model.layers.69.mlp.experts.97.gate_proj", "model.layers.69.mlp.experts.98.gate_proj", "model.layers.69.mlp.experts.99.gate_proj", "model.layers.69.mlp.experts.100.gate_proj", "model.layers.69.mlp.experts.101.gate_proj", "model.layers.69.mlp.experts.102.gate_proj", "model.layers.69.mlp.experts.103.gate_proj", "model.layers.69.mlp.experts.104.gate_proj", "model.layers.69.mlp.experts.105.gate_proj", "model.layers.69.mlp.experts.106.gate_proj", "model.layers.69.mlp.experts.107.gate_proj", "model.layers.69.mlp.experts.108.gate_proj", "model.layers.69.mlp.experts.109.gate_proj", "model.layers.69.mlp.experts.110.gate_proj", "model.layers.69.mlp.experts.111.gate_proj", "model.layers.69.mlp.experts.112.gate_proj", "model.layers.69.mlp.experts.113.gate_proj", "model.layers.69.mlp.experts.114.gate_proj", "model.layers.69.mlp.experts.115.gate_proj", "model.layers.69.mlp.experts.116.gate_proj", "model.layers.69.mlp.experts.117.gate_proj", "model.layers.69.mlp.experts.118.gate_proj", "model.layers.69.mlp.experts.119.gate_proj", "model.layers.69.mlp.experts.120.gate_proj", "model.layers.69.mlp.experts.121.gate_proj", "model.layers.69.mlp.experts.122.gate_proj", "model.layers.69.mlp.experts.123.gate_proj", "model.layers.69.mlp.experts.124.gate_proj", "model.layers.69.mlp.experts.125.gate_proj", "model.layers.69.mlp.experts.126.gate_proj", "model.layers.69.mlp.experts.127.gate_proj", "model.layers.69.mlp.experts.128.gate_proj", "model.layers.69.mlp.experts.129.gate_proj", "model.layers.69.mlp.experts.130.gate_proj", "model.layers.69.mlp.experts.131.gate_proj", "model.layers.69.mlp.experts.132.gate_proj", "model.layers.69.mlp.experts.133.gate_proj", "model.layers.69.mlp.experts.134.gate_proj", "model.layers.69.mlp.experts.135.gate_proj", "model.layers.69.mlp.experts.136.gate_proj", "model.layers.69.mlp.experts.137.gate_proj", "model.layers.69.mlp.experts.138.gate_proj", "model.layers.69.mlp.experts.139.gate_proj", "model.layers.69.mlp.experts.140.gate_proj", "model.layers.69.mlp.experts.141.gate_proj", "model.layers.69.mlp.experts.142.gate_proj", "model.layers.69.mlp.experts.143.gate_proj", "model.layers.69.mlp.experts.144.gate_proj", "model.layers.69.mlp.experts.145.gate_proj", "model.layers.69.mlp.experts.146.gate_proj", "model.layers.69.mlp.experts.147.gate_proj", "model.layers.69.mlp.experts.148.gate_proj", "model.layers.69.mlp.experts.149.gate_proj", "model.layers.69.mlp.experts.150.gate_proj", "model.layers.69.mlp.experts.151.gate_proj", "model.layers.69.mlp.experts.152.gate_proj", "model.layers.69.mlp.experts.153.gate_proj", "model.layers.69.mlp.experts.154.gate_proj", "model.layers.69.mlp.experts.155.gate_proj", "model.layers.69.mlp.experts.156.gate_proj", "model.layers.69.mlp.experts.157.gate_proj", "model.layers.69.mlp.experts.158.gate_proj", "model.layers.69.mlp.experts.159.gate_proj", "model.layers.69.mlp.experts.0.up_proj", "model.layers.69.mlp.experts.1.up_proj", "model.layers.69.mlp.experts.2.up_proj", "model.layers.69.mlp.experts.3.up_proj", "model.layers.69.mlp.experts.4.up_proj", "model.layers.69.mlp.experts.5.up_proj", "model.layers.69.mlp.experts.6.up_proj", "model.layers.69.mlp.experts.7.up_proj", "model.layers.69.mlp.experts.8.up_proj", "model.layers.69.mlp.experts.9.up_proj", "model.layers.69.mlp.experts.10.up_proj", "model.layers.69.mlp.experts.11.up_proj", "model.layers.69.mlp.experts.12.up_proj", "model.layers.69.mlp.experts.13.up_proj", "model.layers.69.mlp.experts.14.up_proj", "model.layers.69.mlp.experts.15.up_proj", "model.layers.69.mlp.experts.16.up_proj", "model.layers.69.mlp.experts.17.up_proj", "model.layers.69.mlp.experts.18.up_proj", "model.layers.69.mlp.experts.19.up_proj", "model.layers.69.mlp.experts.20.up_proj", "model.layers.69.mlp.experts.21.up_proj", "model.layers.69.mlp.experts.22.up_proj", "model.layers.69.mlp.experts.23.up_proj", "model.layers.69.mlp.experts.24.up_proj", "model.layers.69.mlp.experts.25.up_proj", "model.layers.69.mlp.experts.26.up_proj", "model.layers.69.mlp.experts.27.up_proj", "model.layers.69.mlp.experts.28.up_proj", "model.layers.69.mlp.experts.29.up_proj", "model.layers.69.mlp.experts.30.up_proj", "model.layers.69.mlp.experts.31.up_proj", "model.layers.69.mlp.experts.32.up_proj", "model.layers.69.mlp.experts.33.up_proj", "model.layers.69.mlp.experts.34.up_proj", "model.layers.69.mlp.experts.35.up_proj", "model.layers.69.mlp.experts.36.up_proj", "model.layers.69.mlp.experts.37.up_proj", "model.layers.69.mlp.experts.38.up_proj", "model.layers.69.mlp.experts.39.up_proj", "model.layers.69.mlp.experts.40.up_proj", "model.layers.69.mlp.experts.41.up_proj", "model.layers.69.mlp.experts.42.up_proj", "model.layers.69.mlp.experts.43.up_proj", "model.layers.69.mlp.experts.44.up_proj", "model.layers.69.mlp.experts.45.up_proj", "model.layers.69.mlp.experts.46.up_proj", "model.layers.69.mlp.experts.47.up_proj", "model.layers.69.mlp.experts.48.up_proj", "model.layers.69.mlp.experts.49.up_proj", "model.layers.69.mlp.experts.50.up_proj", "model.layers.69.mlp.experts.51.up_proj", "model.layers.69.mlp.experts.52.up_proj", "model.layers.69.mlp.experts.53.up_proj", "model.layers.69.mlp.experts.54.up_proj", "model.layers.69.mlp.experts.55.up_proj", "model.layers.69.mlp.experts.56.up_proj", "model.layers.69.mlp.experts.57.up_proj", "model.layers.69.mlp.experts.58.up_proj", "model.layers.69.mlp.experts.59.up_proj", "model.layers.69.mlp.experts.60.up_proj", "model.layers.69.mlp.experts.61.up_proj", "model.layers.69.mlp.experts.62.up_proj", "model.layers.69.mlp.experts.63.up_proj", "model.layers.69.mlp.experts.64.up_proj", "model.layers.69.mlp.experts.65.up_proj", "model.layers.69.mlp.experts.66.up_proj", "model.layers.69.mlp.experts.67.up_proj", "model.layers.69.mlp.experts.68.up_proj", "model.layers.69.mlp.experts.69.up_proj", "model.layers.69.mlp.experts.70.up_proj", "model.layers.69.mlp.experts.71.up_proj", "model.layers.69.mlp.experts.72.up_proj", "model.layers.69.mlp.experts.73.up_proj", "model.layers.69.mlp.experts.74.up_proj", "model.layers.69.mlp.experts.75.up_proj", "model.layers.69.mlp.experts.76.up_proj", "model.layers.69.mlp.experts.77.up_proj", "model.layers.69.mlp.experts.78.up_proj", "model.layers.69.mlp.experts.79.up_proj", "model.layers.69.mlp.experts.80.up_proj", "model.layers.69.mlp.experts.81.up_proj", "model.layers.69.mlp.experts.82.up_proj", "model.layers.69.mlp.experts.83.up_proj", "model.layers.69.mlp.experts.84.up_proj", "model.layers.69.mlp.experts.85.up_proj", "model.layers.69.mlp.experts.86.up_proj", "model.layers.69.mlp.experts.87.up_proj", "model.layers.69.mlp.experts.88.up_proj", "model.layers.69.mlp.experts.89.up_proj", "model.layers.69.mlp.experts.90.up_proj", "model.layers.69.mlp.experts.91.up_proj", "model.layers.69.mlp.experts.92.up_proj", "model.layers.69.mlp.experts.93.up_proj", "model.layers.69.mlp.experts.94.up_proj", "model.layers.69.mlp.experts.95.up_proj", "model.layers.69.mlp.experts.96.up_proj", "model.layers.69.mlp.experts.97.up_proj", "model.layers.69.mlp.experts.98.up_proj", "model.layers.69.mlp.experts.99.up_proj", "model.layers.69.mlp.experts.100.up_proj", "model.layers.69.mlp.experts.101.up_proj", "model.layers.69.mlp.experts.102.up_proj", "model.layers.69.mlp.experts.103.up_proj", "model.layers.69.mlp.experts.104.up_proj", "model.layers.69.mlp.experts.105.up_proj", "model.layers.69.mlp.experts.106.up_proj", "model.layers.69.mlp.experts.107.up_proj", "model.layers.69.mlp.experts.108.up_proj", "model.layers.69.mlp.experts.109.up_proj", "model.layers.69.mlp.experts.110.up_proj", "model.layers.69.mlp.experts.111.up_proj", "model.layers.69.mlp.experts.112.up_proj", "model.layers.69.mlp.experts.113.up_proj", "model.layers.69.mlp.experts.114.up_proj", "model.layers.69.mlp.experts.115.up_proj", "model.layers.69.mlp.experts.116.up_proj", "model.layers.69.mlp.experts.117.up_proj", "model.layers.69.mlp.experts.118.up_proj", "model.layers.69.mlp.experts.119.up_proj", "model.layers.69.mlp.experts.120.up_proj", "model.layers.69.mlp.experts.121.up_proj", "model.layers.69.mlp.experts.122.up_proj", "model.layers.69.mlp.experts.123.up_proj", "model.layers.69.mlp.experts.124.up_proj", "model.layers.69.mlp.experts.125.up_proj", "model.layers.69.mlp.experts.126.up_proj", "model.layers.69.mlp.experts.127.up_proj", "model.layers.69.mlp.experts.128.up_proj", "model.layers.69.mlp.experts.129.up_proj", "model.layers.69.mlp.experts.130.up_proj", "model.layers.69.mlp.experts.131.up_proj", "model.layers.69.mlp.experts.132.up_proj", "model.layers.69.mlp.experts.133.up_proj", "model.layers.69.mlp.experts.134.up_proj", "model.layers.69.mlp.experts.135.up_proj", "model.layers.69.mlp.experts.136.up_proj", "model.layers.69.mlp.experts.137.up_proj", "model.layers.69.mlp.experts.138.up_proj", "model.layers.69.mlp.experts.139.up_proj", "model.layers.69.mlp.experts.140.up_proj", "model.layers.69.mlp.experts.141.up_proj", "model.layers.69.mlp.experts.142.up_proj", "model.layers.69.mlp.experts.143.up_proj", "model.layers.69.mlp.experts.144.up_proj", "model.layers.69.mlp.experts.145.up_proj", "model.layers.69.mlp.experts.146.up_proj", "model.layers.69.mlp.experts.147.up_proj", "model.layers.69.mlp.experts.148.up_proj", "model.layers.69.mlp.experts.149.up_proj", "model.layers.69.mlp.experts.150.up_proj", "model.layers.69.mlp.experts.151.up_proj", "model.layers.69.mlp.experts.152.up_proj", "model.layers.69.mlp.experts.153.up_proj", "model.layers.69.mlp.experts.154.up_proj", "model.layers.69.mlp.experts.155.up_proj", "model.layers.69.mlp.experts.156.up_proj", "model.layers.69.mlp.experts.157.up_proj", "model.layers.69.mlp.experts.158.up_proj", "model.layers.69.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00037528276443482556, "dbits": 2516582400 } ] }, { "idx": 416, "layers": [ "model.layers.69.mlp.experts.0.down_proj", "model.layers.69.mlp.experts.1.down_proj", "model.layers.69.mlp.experts.2.down_proj", "model.layers.69.mlp.experts.3.down_proj", "model.layers.69.mlp.experts.4.down_proj", "model.layers.69.mlp.experts.5.down_proj", "model.layers.69.mlp.experts.6.down_proj", "model.layers.69.mlp.experts.7.down_proj", "model.layers.69.mlp.experts.8.down_proj", "model.layers.69.mlp.experts.9.down_proj", "model.layers.69.mlp.experts.10.down_proj", "model.layers.69.mlp.experts.11.down_proj", "model.layers.69.mlp.experts.12.down_proj", "model.layers.69.mlp.experts.13.down_proj", "model.layers.69.mlp.experts.14.down_proj", "model.layers.69.mlp.experts.15.down_proj", "model.layers.69.mlp.experts.16.down_proj", "model.layers.69.mlp.experts.17.down_proj", "model.layers.69.mlp.experts.18.down_proj", "model.layers.69.mlp.experts.19.down_proj", "model.layers.69.mlp.experts.20.down_proj", "model.layers.69.mlp.experts.21.down_proj", "model.layers.69.mlp.experts.22.down_proj", "model.layers.69.mlp.experts.23.down_proj", "model.layers.69.mlp.experts.24.down_proj", "model.layers.69.mlp.experts.25.down_proj", "model.layers.69.mlp.experts.26.down_proj", "model.layers.69.mlp.experts.27.down_proj", "model.layers.69.mlp.experts.28.down_proj", "model.layers.69.mlp.experts.29.down_proj", "model.layers.69.mlp.experts.30.down_proj", "model.layers.69.mlp.experts.31.down_proj", "model.layers.69.mlp.experts.32.down_proj", "model.layers.69.mlp.experts.33.down_proj", "model.layers.69.mlp.experts.34.down_proj", "model.layers.69.mlp.experts.35.down_proj", "model.layers.69.mlp.experts.36.down_proj", "model.layers.69.mlp.experts.37.down_proj", "model.layers.69.mlp.experts.38.down_proj", "model.layers.69.mlp.experts.39.down_proj", "model.layers.69.mlp.experts.40.down_proj", "model.layers.69.mlp.experts.41.down_proj", "model.layers.69.mlp.experts.42.down_proj", "model.layers.69.mlp.experts.43.down_proj", "model.layers.69.mlp.experts.44.down_proj", "model.layers.69.mlp.experts.45.down_proj", "model.layers.69.mlp.experts.46.down_proj", "model.layers.69.mlp.experts.47.down_proj", "model.layers.69.mlp.experts.48.down_proj", "model.layers.69.mlp.experts.49.down_proj", "model.layers.69.mlp.experts.50.down_proj", "model.layers.69.mlp.experts.51.down_proj", "model.layers.69.mlp.experts.52.down_proj", "model.layers.69.mlp.experts.53.down_proj", "model.layers.69.mlp.experts.54.down_proj", "model.layers.69.mlp.experts.55.down_proj", "model.layers.69.mlp.experts.56.down_proj", "model.layers.69.mlp.experts.57.down_proj", "model.layers.69.mlp.experts.58.down_proj", "model.layers.69.mlp.experts.59.down_proj", "model.layers.69.mlp.experts.60.down_proj", "model.layers.69.mlp.experts.61.down_proj", "model.layers.69.mlp.experts.62.down_proj", "model.layers.69.mlp.experts.63.down_proj", "model.layers.69.mlp.experts.64.down_proj", "model.layers.69.mlp.experts.65.down_proj", "model.layers.69.mlp.experts.66.down_proj", "model.layers.69.mlp.experts.67.down_proj", "model.layers.69.mlp.experts.68.down_proj", "model.layers.69.mlp.experts.69.down_proj", "model.layers.69.mlp.experts.70.down_proj", "model.layers.69.mlp.experts.71.down_proj", "model.layers.69.mlp.experts.72.down_proj", "model.layers.69.mlp.experts.73.down_proj", "model.layers.69.mlp.experts.74.down_proj", "model.layers.69.mlp.experts.75.down_proj", "model.layers.69.mlp.experts.76.down_proj", "model.layers.69.mlp.experts.77.down_proj", "model.layers.69.mlp.experts.78.down_proj", "model.layers.69.mlp.experts.79.down_proj", "model.layers.69.mlp.experts.80.down_proj", "model.layers.69.mlp.experts.81.down_proj", "model.layers.69.mlp.experts.82.down_proj", "model.layers.69.mlp.experts.83.down_proj", "model.layers.69.mlp.experts.84.down_proj", "model.layers.69.mlp.experts.85.down_proj", "model.layers.69.mlp.experts.86.down_proj", "model.layers.69.mlp.experts.87.down_proj", "model.layers.69.mlp.experts.88.down_proj", "model.layers.69.mlp.experts.89.down_proj", "model.layers.69.mlp.experts.90.down_proj", "model.layers.69.mlp.experts.91.down_proj", "model.layers.69.mlp.experts.92.down_proj", "model.layers.69.mlp.experts.93.down_proj", "model.layers.69.mlp.experts.94.down_proj", "model.layers.69.mlp.experts.95.down_proj", "model.layers.69.mlp.experts.96.down_proj", "model.layers.69.mlp.experts.97.down_proj", "model.layers.69.mlp.experts.98.down_proj", "model.layers.69.mlp.experts.99.down_proj", "model.layers.69.mlp.experts.100.down_proj", "model.layers.69.mlp.experts.101.down_proj", "model.layers.69.mlp.experts.102.down_proj", "model.layers.69.mlp.experts.103.down_proj", "model.layers.69.mlp.experts.104.down_proj", "model.layers.69.mlp.experts.105.down_proj", "model.layers.69.mlp.experts.106.down_proj", "model.layers.69.mlp.experts.107.down_proj", "model.layers.69.mlp.experts.108.down_proj", "model.layers.69.mlp.experts.109.down_proj", "model.layers.69.mlp.experts.110.down_proj", "model.layers.69.mlp.experts.111.down_proj", "model.layers.69.mlp.experts.112.down_proj", "model.layers.69.mlp.experts.113.down_proj", "model.layers.69.mlp.experts.114.down_proj", "model.layers.69.mlp.experts.115.down_proj", "model.layers.69.mlp.experts.116.down_proj", "model.layers.69.mlp.experts.117.down_proj", "model.layers.69.mlp.experts.118.down_proj", "model.layers.69.mlp.experts.119.down_proj", "model.layers.69.mlp.experts.120.down_proj", "model.layers.69.mlp.experts.121.down_proj", "model.layers.69.mlp.experts.122.down_proj", "model.layers.69.mlp.experts.123.down_proj", "model.layers.69.mlp.experts.124.down_proj", "model.layers.69.mlp.experts.125.down_proj", "model.layers.69.mlp.experts.126.down_proj", "model.layers.69.mlp.experts.127.down_proj", "model.layers.69.mlp.experts.128.down_proj", "model.layers.69.mlp.experts.129.down_proj", "model.layers.69.mlp.experts.130.down_proj", "model.layers.69.mlp.experts.131.down_proj", "model.layers.69.mlp.experts.132.down_proj", "model.layers.69.mlp.experts.133.down_proj", "model.layers.69.mlp.experts.134.down_proj", "model.layers.69.mlp.experts.135.down_proj", "model.layers.69.mlp.experts.136.down_proj", "model.layers.69.mlp.experts.137.down_proj", "model.layers.69.mlp.experts.138.down_proj", "model.layers.69.mlp.experts.139.down_proj", "model.layers.69.mlp.experts.140.down_proj", "model.layers.69.mlp.experts.141.down_proj", "model.layers.69.mlp.experts.142.down_proj", "model.layers.69.mlp.experts.143.down_proj", "model.layers.69.mlp.experts.144.down_proj", "model.layers.69.mlp.experts.145.down_proj", "model.layers.69.mlp.experts.146.down_proj", "model.layers.69.mlp.experts.147.down_proj", "model.layers.69.mlp.experts.148.down_proj", "model.layers.69.mlp.experts.149.down_proj", "model.layers.69.mlp.experts.150.down_proj", "model.layers.69.mlp.experts.151.down_proj", "model.layers.69.mlp.experts.152.down_proj", "model.layers.69.mlp.experts.153.down_proj", "model.layers.69.mlp.experts.154.down_proj", "model.layers.69.mlp.experts.155.down_proj", "model.layers.69.mlp.experts.156.down_proj", "model.layers.69.mlp.experts.157.down_proj", "model.layers.69.mlp.experts.158.down_proj", "model.layers.69.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00011793076992033802, "dbits": 1258291200 } ] }, { "idx": 417, "layers": [ "model.layers.70.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00047333166003227234, "dbits": 62914560 } ] }, { "idx": 418, "layers": [ "model.layers.70.self_attn.k_proj", "model.layers.70.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0010799102485180012, "dbits": 10485760 } ] }, { "idx": 419, "layers": [ "model.layers.70.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0002946518361568451, "dbits": 62914560 } ] }, { "idx": 420, "layers": [ "model.layers.70.mlp.shared_experts.gate_proj", "model.layers.70.mlp.shared_experts.up_proj", "model.layers.70.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -8.897557854653515e-05, "dbits": 23592960 } ] }, { "idx": 421, "layers": [ "model.layers.70.mlp.experts.0.gate_proj", "model.layers.70.mlp.experts.1.gate_proj", "model.layers.70.mlp.experts.2.gate_proj", "model.layers.70.mlp.experts.3.gate_proj", "model.layers.70.mlp.experts.4.gate_proj", "model.layers.70.mlp.experts.5.gate_proj", "model.layers.70.mlp.experts.6.gate_proj", "model.layers.70.mlp.experts.7.gate_proj", "model.layers.70.mlp.experts.8.gate_proj", "model.layers.70.mlp.experts.9.gate_proj", "model.layers.70.mlp.experts.10.gate_proj", "model.layers.70.mlp.experts.11.gate_proj", "model.layers.70.mlp.experts.12.gate_proj", "model.layers.70.mlp.experts.13.gate_proj", "model.layers.70.mlp.experts.14.gate_proj", "model.layers.70.mlp.experts.15.gate_proj", "model.layers.70.mlp.experts.16.gate_proj", "model.layers.70.mlp.experts.17.gate_proj", "model.layers.70.mlp.experts.18.gate_proj", "model.layers.70.mlp.experts.19.gate_proj", "model.layers.70.mlp.experts.20.gate_proj", "model.layers.70.mlp.experts.21.gate_proj", "model.layers.70.mlp.experts.22.gate_proj", "model.layers.70.mlp.experts.23.gate_proj", "model.layers.70.mlp.experts.24.gate_proj", "model.layers.70.mlp.experts.25.gate_proj", "model.layers.70.mlp.experts.26.gate_proj", "model.layers.70.mlp.experts.27.gate_proj", "model.layers.70.mlp.experts.28.gate_proj", "model.layers.70.mlp.experts.29.gate_proj", "model.layers.70.mlp.experts.30.gate_proj", "model.layers.70.mlp.experts.31.gate_proj", "model.layers.70.mlp.experts.32.gate_proj", "model.layers.70.mlp.experts.33.gate_proj", "model.layers.70.mlp.experts.34.gate_proj", "model.layers.70.mlp.experts.35.gate_proj", "model.layers.70.mlp.experts.36.gate_proj", "model.layers.70.mlp.experts.37.gate_proj", "model.layers.70.mlp.experts.38.gate_proj", "model.layers.70.mlp.experts.39.gate_proj", "model.layers.70.mlp.experts.40.gate_proj", "model.layers.70.mlp.experts.41.gate_proj", "model.layers.70.mlp.experts.42.gate_proj", "model.layers.70.mlp.experts.43.gate_proj", "model.layers.70.mlp.experts.44.gate_proj", "model.layers.70.mlp.experts.45.gate_proj", "model.layers.70.mlp.experts.46.gate_proj", "model.layers.70.mlp.experts.47.gate_proj", "model.layers.70.mlp.experts.48.gate_proj", "model.layers.70.mlp.experts.49.gate_proj", "model.layers.70.mlp.experts.50.gate_proj", "model.layers.70.mlp.experts.51.gate_proj", "model.layers.70.mlp.experts.52.gate_proj", "model.layers.70.mlp.experts.53.gate_proj", "model.layers.70.mlp.experts.54.gate_proj", "model.layers.70.mlp.experts.55.gate_proj", "model.layers.70.mlp.experts.56.gate_proj", "model.layers.70.mlp.experts.57.gate_proj", "model.layers.70.mlp.experts.58.gate_proj", "model.layers.70.mlp.experts.59.gate_proj", "model.layers.70.mlp.experts.60.gate_proj", "model.layers.70.mlp.experts.61.gate_proj", "model.layers.70.mlp.experts.62.gate_proj", "model.layers.70.mlp.experts.63.gate_proj", "model.layers.70.mlp.experts.64.gate_proj", "model.layers.70.mlp.experts.65.gate_proj", "model.layers.70.mlp.experts.66.gate_proj", "model.layers.70.mlp.experts.67.gate_proj", "model.layers.70.mlp.experts.68.gate_proj", "model.layers.70.mlp.experts.69.gate_proj", "model.layers.70.mlp.experts.70.gate_proj", "model.layers.70.mlp.experts.71.gate_proj", "model.layers.70.mlp.experts.72.gate_proj", "model.layers.70.mlp.experts.73.gate_proj", "model.layers.70.mlp.experts.74.gate_proj", "model.layers.70.mlp.experts.75.gate_proj", "model.layers.70.mlp.experts.76.gate_proj", "model.layers.70.mlp.experts.77.gate_proj", "model.layers.70.mlp.experts.78.gate_proj", "model.layers.70.mlp.experts.79.gate_proj", "model.layers.70.mlp.experts.80.gate_proj", "model.layers.70.mlp.experts.81.gate_proj", "model.layers.70.mlp.experts.82.gate_proj", "model.layers.70.mlp.experts.83.gate_proj", "model.layers.70.mlp.experts.84.gate_proj", "model.layers.70.mlp.experts.85.gate_proj", "model.layers.70.mlp.experts.86.gate_proj", "model.layers.70.mlp.experts.87.gate_proj", "model.layers.70.mlp.experts.88.gate_proj", "model.layers.70.mlp.experts.89.gate_proj", "model.layers.70.mlp.experts.90.gate_proj", "model.layers.70.mlp.experts.91.gate_proj", "model.layers.70.mlp.experts.92.gate_proj", "model.layers.70.mlp.experts.93.gate_proj", "model.layers.70.mlp.experts.94.gate_proj", "model.layers.70.mlp.experts.95.gate_proj", "model.layers.70.mlp.experts.96.gate_proj", "model.layers.70.mlp.experts.97.gate_proj", "model.layers.70.mlp.experts.98.gate_proj", "model.layers.70.mlp.experts.99.gate_proj", "model.layers.70.mlp.experts.100.gate_proj", "model.layers.70.mlp.experts.101.gate_proj", "model.layers.70.mlp.experts.102.gate_proj", "model.layers.70.mlp.experts.103.gate_proj", "model.layers.70.mlp.experts.104.gate_proj", "model.layers.70.mlp.experts.105.gate_proj", "model.layers.70.mlp.experts.106.gate_proj", "model.layers.70.mlp.experts.107.gate_proj", "model.layers.70.mlp.experts.108.gate_proj", "model.layers.70.mlp.experts.109.gate_proj", "model.layers.70.mlp.experts.110.gate_proj", "model.layers.70.mlp.experts.111.gate_proj", "model.layers.70.mlp.experts.112.gate_proj", "model.layers.70.mlp.experts.113.gate_proj", "model.layers.70.mlp.experts.114.gate_proj", "model.layers.70.mlp.experts.115.gate_proj", "model.layers.70.mlp.experts.116.gate_proj", "model.layers.70.mlp.experts.117.gate_proj", "model.layers.70.mlp.experts.118.gate_proj", "model.layers.70.mlp.experts.119.gate_proj", "model.layers.70.mlp.experts.120.gate_proj", "model.layers.70.mlp.experts.121.gate_proj", "model.layers.70.mlp.experts.122.gate_proj", "model.layers.70.mlp.experts.123.gate_proj", "model.layers.70.mlp.experts.124.gate_proj", "model.layers.70.mlp.experts.125.gate_proj", "model.layers.70.mlp.experts.126.gate_proj", "model.layers.70.mlp.experts.127.gate_proj", "model.layers.70.mlp.experts.128.gate_proj", "model.layers.70.mlp.experts.129.gate_proj", "model.layers.70.mlp.experts.130.gate_proj", "model.layers.70.mlp.experts.131.gate_proj", "model.layers.70.mlp.experts.132.gate_proj", "model.layers.70.mlp.experts.133.gate_proj", "model.layers.70.mlp.experts.134.gate_proj", "model.layers.70.mlp.experts.135.gate_proj", "model.layers.70.mlp.experts.136.gate_proj", "model.layers.70.mlp.experts.137.gate_proj", "model.layers.70.mlp.experts.138.gate_proj", "model.layers.70.mlp.experts.139.gate_proj", "model.layers.70.mlp.experts.140.gate_proj", "model.layers.70.mlp.experts.141.gate_proj", "model.layers.70.mlp.experts.142.gate_proj", "model.layers.70.mlp.experts.143.gate_proj", "model.layers.70.mlp.experts.144.gate_proj", "model.layers.70.mlp.experts.145.gate_proj", "model.layers.70.mlp.experts.146.gate_proj", "model.layers.70.mlp.experts.147.gate_proj", "model.layers.70.mlp.experts.148.gate_proj", "model.layers.70.mlp.experts.149.gate_proj", "model.layers.70.mlp.experts.150.gate_proj", "model.layers.70.mlp.experts.151.gate_proj", "model.layers.70.mlp.experts.152.gate_proj", "model.layers.70.mlp.experts.153.gate_proj", "model.layers.70.mlp.experts.154.gate_proj", "model.layers.70.mlp.experts.155.gate_proj", "model.layers.70.mlp.experts.156.gate_proj", "model.layers.70.mlp.experts.157.gate_proj", "model.layers.70.mlp.experts.158.gate_proj", "model.layers.70.mlp.experts.159.gate_proj", "model.layers.70.mlp.experts.0.up_proj", "model.layers.70.mlp.experts.1.up_proj", "model.layers.70.mlp.experts.2.up_proj", "model.layers.70.mlp.experts.3.up_proj", "model.layers.70.mlp.experts.4.up_proj", "model.layers.70.mlp.experts.5.up_proj", "model.layers.70.mlp.experts.6.up_proj", "model.layers.70.mlp.experts.7.up_proj", "model.layers.70.mlp.experts.8.up_proj", "model.layers.70.mlp.experts.9.up_proj", "model.layers.70.mlp.experts.10.up_proj", "model.layers.70.mlp.experts.11.up_proj", "model.layers.70.mlp.experts.12.up_proj", "model.layers.70.mlp.experts.13.up_proj", "model.layers.70.mlp.experts.14.up_proj", "model.layers.70.mlp.experts.15.up_proj", "model.layers.70.mlp.experts.16.up_proj", "model.layers.70.mlp.experts.17.up_proj", "model.layers.70.mlp.experts.18.up_proj", "model.layers.70.mlp.experts.19.up_proj", "model.layers.70.mlp.experts.20.up_proj", "model.layers.70.mlp.experts.21.up_proj", "model.layers.70.mlp.experts.22.up_proj", "model.layers.70.mlp.experts.23.up_proj", "model.layers.70.mlp.experts.24.up_proj", "model.layers.70.mlp.experts.25.up_proj", "model.layers.70.mlp.experts.26.up_proj", "model.layers.70.mlp.experts.27.up_proj", "model.layers.70.mlp.experts.28.up_proj", "model.layers.70.mlp.experts.29.up_proj", "model.layers.70.mlp.experts.30.up_proj", "model.layers.70.mlp.experts.31.up_proj", "model.layers.70.mlp.experts.32.up_proj", "model.layers.70.mlp.experts.33.up_proj", "model.layers.70.mlp.experts.34.up_proj", "model.layers.70.mlp.experts.35.up_proj", "model.layers.70.mlp.experts.36.up_proj", "model.layers.70.mlp.experts.37.up_proj", "model.layers.70.mlp.experts.38.up_proj", "model.layers.70.mlp.experts.39.up_proj", "model.layers.70.mlp.experts.40.up_proj", "model.layers.70.mlp.experts.41.up_proj", "model.layers.70.mlp.experts.42.up_proj", "model.layers.70.mlp.experts.43.up_proj", "model.layers.70.mlp.experts.44.up_proj", "model.layers.70.mlp.experts.45.up_proj", "model.layers.70.mlp.experts.46.up_proj", "model.layers.70.mlp.experts.47.up_proj", "model.layers.70.mlp.experts.48.up_proj", "model.layers.70.mlp.experts.49.up_proj", "model.layers.70.mlp.experts.50.up_proj", "model.layers.70.mlp.experts.51.up_proj", "model.layers.70.mlp.experts.52.up_proj", "model.layers.70.mlp.experts.53.up_proj", "model.layers.70.mlp.experts.54.up_proj", "model.layers.70.mlp.experts.55.up_proj", "model.layers.70.mlp.experts.56.up_proj", "model.layers.70.mlp.experts.57.up_proj", "model.layers.70.mlp.experts.58.up_proj", "model.layers.70.mlp.experts.59.up_proj", "model.layers.70.mlp.experts.60.up_proj", "model.layers.70.mlp.experts.61.up_proj", "model.layers.70.mlp.experts.62.up_proj", "model.layers.70.mlp.experts.63.up_proj", "model.layers.70.mlp.experts.64.up_proj", "model.layers.70.mlp.experts.65.up_proj", "model.layers.70.mlp.experts.66.up_proj", "model.layers.70.mlp.experts.67.up_proj", "model.layers.70.mlp.experts.68.up_proj", "model.layers.70.mlp.experts.69.up_proj", "model.layers.70.mlp.experts.70.up_proj", "model.layers.70.mlp.experts.71.up_proj", "model.layers.70.mlp.experts.72.up_proj", "model.layers.70.mlp.experts.73.up_proj", "model.layers.70.mlp.experts.74.up_proj", "model.layers.70.mlp.experts.75.up_proj", "model.layers.70.mlp.experts.76.up_proj", "model.layers.70.mlp.experts.77.up_proj", "model.layers.70.mlp.experts.78.up_proj", "model.layers.70.mlp.experts.79.up_proj", "model.layers.70.mlp.experts.80.up_proj", "model.layers.70.mlp.experts.81.up_proj", "model.layers.70.mlp.experts.82.up_proj", "model.layers.70.mlp.experts.83.up_proj", "model.layers.70.mlp.experts.84.up_proj", "model.layers.70.mlp.experts.85.up_proj", "model.layers.70.mlp.experts.86.up_proj", "model.layers.70.mlp.experts.87.up_proj", "model.layers.70.mlp.experts.88.up_proj", "model.layers.70.mlp.experts.89.up_proj", "model.layers.70.mlp.experts.90.up_proj", "model.layers.70.mlp.experts.91.up_proj", "model.layers.70.mlp.experts.92.up_proj", "model.layers.70.mlp.experts.93.up_proj", "model.layers.70.mlp.experts.94.up_proj", "model.layers.70.mlp.experts.95.up_proj", "model.layers.70.mlp.experts.96.up_proj", "model.layers.70.mlp.experts.97.up_proj", "model.layers.70.mlp.experts.98.up_proj", "model.layers.70.mlp.experts.99.up_proj", "model.layers.70.mlp.experts.100.up_proj", "model.layers.70.mlp.experts.101.up_proj", "model.layers.70.mlp.experts.102.up_proj", "model.layers.70.mlp.experts.103.up_proj", "model.layers.70.mlp.experts.104.up_proj", "model.layers.70.mlp.experts.105.up_proj", "model.layers.70.mlp.experts.106.up_proj", "model.layers.70.mlp.experts.107.up_proj", "model.layers.70.mlp.experts.108.up_proj", "model.layers.70.mlp.experts.109.up_proj", "model.layers.70.mlp.experts.110.up_proj", "model.layers.70.mlp.experts.111.up_proj", "model.layers.70.mlp.experts.112.up_proj", "model.layers.70.mlp.experts.113.up_proj", "model.layers.70.mlp.experts.114.up_proj", "model.layers.70.mlp.experts.115.up_proj", "model.layers.70.mlp.experts.116.up_proj", "model.layers.70.mlp.experts.117.up_proj", "model.layers.70.mlp.experts.118.up_proj", "model.layers.70.mlp.experts.119.up_proj", "model.layers.70.mlp.experts.120.up_proj", "model.layers.70.mlp.experts.121.up_proj", "model.layers.70.mlp.experts.122.up_proj", "model.layers.70.mlp.experts.123.up_proj", "model.layers.70.mlp.experts.124.up_proj", "model.layers.70.mlp.experts.125.up_proj", "model.layers.70.mlp.experts.126.up_proj", "model.layers.70.mlp.experts.127.up_proj", "model.layers.70.mlp.experts.128.up_proj", "model.layers.70.mlp.experts.129.up_proj", "model.layers.70.mlp.experts.130.up_proj", "model.layers.70.mlp.experts.131.up_proj", "model.layers.70.mlp.experts.132.up_proj", "model.layers.70.mlp.experts.133.up_proj", "model.layers.70.mlp.experts.134.up_proj", "model.layers.70.mlp.experts.135.up_proj", "model.layers.70.mlp.experts.136.up_proj", "model.layers.70.mlp.experts.137.up_proj", "model.layers.70.mlp.experts.138.up_proj", "model.layers.70.mlp.experts.139.up_proj", "model.layers.70.mlp.experts.140.up_proj", "model.layers.70.mlp.experts.141.up_proj", "model.layers.70.mlp.experts.142.up_proj", "model.layers.70.mlp.experts.143.up_proj", "model.layers.70.mlp.experts.144.up_proj", "model.layers.70.mlp.experts.145.up_proj", "model.layers.70.mlp.experts.146.up_proj", "model.layers.70.mlp.experts.147.up_proj", "model.layers.70.mlp.experts.148.up_proj", "model.layers.70.mlp.experts.149.up_proj", "model.layers.70.mlp.experts.150.up_proj", "model.layers.70.mlp.experts.151.up_proj", "model.layers.70.mlp.experts.152.up_proj", "model.layers.70.mlp.experts.153.up_proj", "model.layers.70.mlp.experts.154.up_proj", "model.layers.70.mlp.experts.155.up_proj", "model.layers.70.mlp.experts.156.up_proj", "model.layers.70.mlp.experts.157.up_proj", "model.layers.70.mlp.experts.158.up_proj", "model.layers.70.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.000521109998226188, "dbits": 2516582400 } ] }, { "idx": 422, "layers": [ "model.layers.70.mlp.experts.0.down_proj", "model.layers.70.mlp.experts.1.down_proj", "model.layers.70.mlp.experts.2.down_proj", "model.layers.70.mlp.experts.3.down_proj", "model.layers.70.mlp.experts.4.down_proj", "model.layers.70.mlp.experts.5.down_proj", "model.layers.70.mlp.experts.6.down_proj", "model.layers.70.mlp.experts.7.down_proj", "model.layers.70.mlp.experts.8.down_proj", "model.layers.70.mlp.experts.9.down_proj", "model.layers.70.mlp.experts.10.down_proj", "model.layers.70.mlp.experts.11.down_proj", "model.layers.70.mlp.experts.12.down_proj", "model.layers.70.mlp.experts.13.down_proj", "model.layers.70.mlp.experts.14.down_proj", "model.layers.70.mlp.experts.15.down_proj", "model.layers.70.mlp.experts.16.down_proj", "model.layers.70.mlp.experts.17.down_proj", "model.layers.70.mlp.experts.18.down_proj", "model.layers.70.mlp.experts.19.down_proj", "model.layers.70.mlp.experts.20.down_proj", "model.layers.70.mlp.experts.21.down_proj", "model.layers.70.mlp.experts.22.down_proj", "model.layers.70.mlp.experts.23.down_proj", "model.layers.70.mlp.experts.24.down_proj", "model.layers.70.mlp.experts.25.down_proj", "model.layers.70.mlp.experts.26.down_proj", "model.layers.70.mlp.experts.27.down_proj", "model.layers.70.mlp.experts.28.down_proj", "model.layers.70.mlp.experts.29.down_proj", "model.layers.70.mlp.experts.30.down_proj", "model.layers.70.mlp.experts.31.down_proj", "model.layers.70.mlp.experts.32.down_proj", "model.layers.70.mlp.experts.33.down_proj", "model.layers.70.mlp.experts.34.down_proj", "model.layers.70.mlp.experts.35.down_proj", "model.layers.70.mlp.experts.36.down_proj", "model.layers.70.mlp.experts.37.down_proj", "model.layers.70.mlp.experts.38.down_proj", "model.layers.70.mlp.experts.39.down_proj", "model.layers.70.mlp.experts.40.down_proj", "model.layers.70.mlp.experts.41.down_proj", "model.layers.70.mlp.experts.42.down_proj", "model.layers.70.mlp.experts.43.down_proj", "model.layers.70.mlp.experts.44.down_proj", "model.layers.70.mlp.experts.45.down_proj", "model.layers.70.mlp.experts.46.down_proj", "model.layers.70.mlp.experts.47.down_proj", "model.layers.70.mlp.experts.48.down_proj", "model.layers.70.mlp.experts.49.down_proj", "model.layers.70.mlp.experts.50.down_proj", "model.layers.70.mlp.experts.51.down_proj", "model.layers.70.mlp.experts.52.down_proj", "model.layers.70.mlp.experts.53.down_proj", "model.layers.70.mlp.experts.54.down_proj", "model.layers.70.mlp.experts.55.down_proj", "model.layers.70.mlp.experts.56.down_proj", "model.layers.70.mlp.experts.57.down_proj", "model.layers.70.mlp.experts.58.down_proj", "model.layers.70.mlp.experts.59.down_proj", "model.layers.70.mlp.experts.60.down_proj", "model.layers.70.mlp.experts.61.down_proj", "model.layers.70.mlp.experts.62.down_proj", "model.layers.70.mlp.experts.63.down_proj", "model.layers.70.mlp.experts.64.down_proj", "model.layers.70.mlp.experts.65.down_proj", "model.layers.70.mlp.experts.66.down_proj", "model.layers.70.mlp.experts.67.down_proj", "model.layers.70.mlp.experts.68.down_proj", "model.layers.70.mlp.experts.69.down_proj", "model.layers.70.mlp.experts.70.down_proj", "model.layers.70.mlp.experts.71.down_proj", "model.layers.70.mlp.experts.72.down_proj", "model.layers.70.mlp.experts.73.down_proj", "model.layers.70.mlp.experts.74.down_proj", "model.layers.70.mlp.experts.75.down_proj", "model.layers.70.mlp.experts.76.down_proj", "model.layers.70.mlp.experts.77.down_proj", "model.layers.70.mlp.experts.78.down_proj", "model.layers.70.mlp.experts.79.down_proj", "model.layers.70.mlp.experts.80.down_proj", "model.layers.70.mlp.experts.81.down_proj", "model.layers.70.mlp.experts.82.down_proj", "model.layers.70.mlp.experts.83.down_proj", "model.layers.70.mlp.experts.84.down_proj", "model.layers.70.mlp.experts.85.down_proj", "model.layers.70.mlp.experts.86.down_proj", "model.layers.70.mlp.experts.87.down_proj", "model.layers.70.mlp.experts.88.down_proj", "model.layers.70.mlp.experts.89.down_proj", "model.layers.70.mlp.experts.90.down_proj", "model.layers.70.mlp.experts.91.down_proj", "model.layers.70.mlp.experts.92.down_proj", "model.layers.70.mlp.experts.93.down_proj", "model.layers.70.mlp.experts.94.down_proj", "model.layers.70.mlp.experts.95.down_proj", "model.layers.70.mlp.experts.96.down_proj", "model.layers.70.mlp.experts.97.down_proj", "model.layers.70.mlp.experts.98.down_proj", "model.layers.70.mlp.experts.99.down_proj", "model.layers.70.mlp.experts.100.down_proj", "model.layers.70.mlp.experts.101.down_proj", "model.layers.70.mlp.experts.102.down_proj", "model.layers.70.mlp.experts.103.down_proj", "model.layers.70.mlp.experts.104.down_proj", "model.layers.70.mlp.experts.105.down_proj", "model.layers.70.mlp.experts.106.down_proj", "model.layers.70.mlp.experts.107.down_proj", "model.layers.70.mlp.experts.108.down_proj", "model.layers.70.mlp.experts.109.down_proj", "model.layers.70.mlp.experts.110.down_proj", "model.layers.70.mlp.experts.111.down_proj", "model.layers.70.mlp.experts.112.down_proj", "model.layers.70.mlp.experts.113.down_proj", "model.layers.70.mlp.experts.114.down_proj", "model.layers.70.mlp.experts.115.down_proj", "model.layers.70.mlp.experts.116.down_proj", "model.layers.70.mlp.experts.117.down_proj", "model.layers.70.mlp.experts.118.down_proj", "model.layers.70.mlp.experts.119.down_proj", "model.layers.70.mlp.experts.120.down_proj", "model.layers.70.mlp.experts.121.down_proj", "model.layers.70.mlp.experts.122.down_proj", "model.layers.70.mlp.experts.123.down_proj", "model.layers.70.mlp.experts.124.down_proj", "model.layers.70.mlp.experts.125.down_proj", "model.layers.70.mlp.experts.126.down_proj", "model.layers.70.mlp.experts.127.down_proj", "model.layers.70.mlp.experts.128.down_proj", "model.layers.70.mlp.experts.129.down_proj", "model.layers.70.mlp.experts.130.down_proj", "model.layers.70.mlp.experts.131.down_proj", "model.layers.70.mlp.experts.132.down_proj", "model.layers.70.mlp.experts.133.down_proj", "model.layers.70.mlp.experts.134.down_proj", "model.layers.70.mlp.experts.135.down_proj", "model.layers.70.mlp.experts.136.down_proj", "model.layers.70.mlp.experts.137.down_proj", "model.layers.70.mlp.experts.138.down_proj", "model.layers.70.mlp.experts.139.down_proj", "model.layers.70.mlp.experts.140.down_proj", "model.layers.70.mlp.experts.141.down_proj", "model.layers.70.mlp.experts.142.down_proj", "model.layers.70.mlp.experts.143.down_proj", "model.layers.70.mlp.experts.144.down_proj", "model.layers.70.mlp.experts.145.down_proj", "model.layers.70.mlp.experts.146.down_proj", "model.layers.70.mlp.experts.147.down_proj", "model.layers.70.mlp.experts.148.down_proj", "model.layers.70.mlp.experts.149.down_proj", "model.layers.70.mlp.experts.150.down_proj", "model.layers.70.mlp.experts.151.down_proj", "model.layers.70.mlp.experts.152.down_proj", "model.layers.70.mlp.experts.153.down_proj", "model.layers.70.mlp.experts.154.down_proj", "model.layers.70.mlp.experts.155.down_proj", "model.layers.70.mlp.experts.156.down_proj", "model.layers.70.mlp.experts.157.down_proj", "model.layers.70.mlp.experts.158.down_proj", "model.layers.70.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0001791529357433319, "dbits": 1258291200 } ] }, { "idx": 423, "layers": [ "model.layers.71.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0002534605562686809, "dbits": 62914560 } ] }, { "idx": 424, "layers": [ "model.layers.71.self_attn.k_proj", "model.layers.71.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0010726615786552651, "dbits": 10485760 } ] }, { "idx": 425, "layers": [ "model.layers.71.self_attn.o_proj" ], "candidates": [ { "dkld": -5.205720663070679e-05, "dbits": 62914560 } ] }, { "idx": 426, "layers": [ "model.layers.71.mlp.shared_experts.gate_proj", "model.layers.71.mlp.shared_experts.up_proj", "model.layers.71.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0010715313255786896, "dbits": 23592960 } ] }, { "idx": 427, "layers": [ "model.layers.71.mlp.experts.0.gate_proj", "model.layers.71.mlp.experts.1.gate_proj", "model.layers.71.mlp.experts.2.gate_proj", "model.layers.71.mlp.experts.3.gate_proj", "model.layers.71.mlp.experts.4.gate_proj", "model.layers.71.mlp.experts.5.gate_proj", "model.layers.71.mlp.experts.6.gate_proj", "model.layers.71.mlp.experts.7.gate_proj", "model.layers.71.mlp.experts.8.gate_proj", "model.layers.71.mlp.experts.9.gate_proj", "model.layers.71.mlp.experts.10.gate_proj", "model.layers.71.mlp.experts.11.gate_proj", "model.layers.71.mlp.experts.12.gate_proj", "model.layers.71.mlp.experts.13.gate_proj", "model.layers.71.mlp.experts.14.gate_proj", "model.layers.71.mlp.experts.15.gate_proj", "model.layers.71.mlp.experts.16.gate_proj", "model.layers.71.mlp.experts.17.gate_proj", "model.layers.71.mlp.experts.18.gate_proj", "model.layers.71.mlp.experts.19.gate_proj", "model.layers.71.mlp.experts.20.gate_proj", "model.layers.71.mlp.experts.21.gate_proj", "model.layers.71.mlp.experts.22.gate_proj", "model.layers.71.mlp.experts.23.gate_proj", "model.layers.71.mlp.experts.24.gate_proj", "model.layers.71.mlp.experts.25.gate_proj", "model.layers.71.mlp.experts.26.gate_proj", "model.layers.71.mlp.experts.27.gate_proj", "model.layers.71.mlp.experts.28.gate_proj", "model.layers.71.mlp.experts.29.gate_proj", "model.layers.71.mlp.experts.30.gate_proj", "model.layers.71.mlp.experts.31.gate_proj", "model.layers.71.mlp.experts.32.gate_proj", "model.layers.71.mlp.experts.33.gate_proj", "model.layers.71.mlp.experts.34.gate_proj", "model.layers.71.mlp.experts.35.gate_proj", "model.layers.71.mlp.experts.36.gate_proj", "model.layers.71.mlp.experts.37.gate_proj", "model.layers.71.mlp.experts.38.gate_proj", "model.layers.71.mlp.experts.39.gate_proj", "model.layers.71.mlp.experts.40.gate_proj", "model.layers.71.mlp.experts.41.gate_proj", "model.layers.71.mlp.experts.42.gate_proj", "model.layers.71.mlp.experts.43.gate_proj", "model.layers.71.mlp.experts.44.gate_proj", "model.layers.71.mlp.experts.45.gate_proj", "model.layers.71.mlp.experts.46.gate_proj", "model.layers.71.mlp.experts.47.gate_proj", "model.layers.71.mlp.experts.48.gate_proj", "model.layers.71.mlp.experts.49.gate_proj", "model.layers.71.mlp.experts.50.gate_proj", "model.layers.71.mlp.experts.51.gate_proj", "model.layers.71.mlp.experts.52.gate_proj", "model.layers.71.mlp.experts.53.gate_proj", "model.layers.71.mlp.experts.54.gate_proj", "model.layers.71.mlp.experts.55.gate_proj", "model.layers.71.mlp.experts.56.gate_proj", "model.layers.71.mlp.experts.57.gate_proj", "model.layers.71.mlp.experts.58.gate_proj", "model.layers.71.mlp.experts.59.gate_proj", "model.layers.71.mlp.experts.60.gate_proj", "model.layers.71.mlp.experts.61.gate_proj", "model.layers.71.mlp.experts.62.gate_proj", "model.layers.71.mlp.experts.63.gate_proj", "model.layers.71.mlp.experts.64.gate_proj", "model.layers.71.mlp.experts.65.gate_proj", "model.layers.71.mlp.experts.66.gate_proj", "model.layers.71.mlp.experts.67.gate_proj", "model.layers.71.mlp.experts.68.gate_proj", "model.layers.71.mlp.experts.69.gate_proj", "model.layers.71.mlp.experts.70.gate_proj", "model.layers.71.mlp.experts.71.gate_proj", "model.layers.71.mlp.experts.72.gate_proj", "model.layers.71.mlp.experts.73.gate_proj", "model.layers.71.mlp.experts.74.gate_proj", "model.layers.71.mlp.experts.75.gate_proj", "model.layers.71.mlp.experts.76.gate_proj", "model.layers.71.mlp.experts.77.gate_proj", "model.layers.71.mlp.experts.78.gate_proj", "model.layers.71.mlp.experts.79.gate_proj", "model.layers.71.mlp.experts.80.gate_proj", "model.layers.71.mlp.experts.81.gate_proj", "model.layers.71.mlp.experts.82.gate_proj", "model.layers.71.mlp.experts.83.gate_proj", "model.layers.71.mlp.experts.84.gate_proj", "model.layers.71.mlp.experts.85.gate_proj", "model.layers.71.mlp.experts.86.gate_proj", "model.layers.71.mlp.experts.87.gate_proj", "model.layers.71.mlp.experts.88.gate_proj", "model.layers.71.mlp.experts.89.gate_proj", "model.layers.71.mlp.experts.90.gate_proj", "model.layers.71.mlp.experts.91.gate_proj", "model.layers.71.mlp.experts.92.gate_proj", "model.layers.71.mlp.experts.93.gate_proj", "model.layers.71.mlp.experts.94.gate_proj", "model.layers.71.mlp.experts.95.gate_proj", "model.layers.71.mlp.experts.96.gate_proj", "model.layers.71.mlp.experts.97.gate_proj", "model.layers.71.mlp.experts.98.gate_proj", "model.layers.71.mlp.experts.99.gate_proj", "model.layers.71.mlp.experts.100.gate_proj", "model.layers.71.mlp.experts.101.gate_proj", "model.layers.71.mlp.experts.102.gate_proj", "model.layers.71.mlp.experts.103.gate_proj", "model.layers.71.mlp.experts.104.gate_proj", "model.layers.71.mlp.experts.105.gate_proj", "model.layers.71.mlp.experts.106.gate_proj", "model.layers.71.mlp.experts.107.gate_proj", "model.layers.71.mlp.experts.108.gate_proj", "model.layers.71.mlp.experts.109.gate_proj", "model.layers.71.mlp.experts.110.gate_proj", "model.layers.71.mlp.experts.111.gate_proj", "model.layers.71.mlp.experts.112.gate_proj", "model.layers.71.mlp.experts.113.gate_proj", "model.layers.71.mlp.experts.114.gate_proj", "model.layers.71.mlp.experts.115.gate_proj", "model.layers.71.mlp.experts.116.gate_proj", "model.layers.71.mlp.experts.117.gate_proj", "model.layers.71.mlp.experts.118.gate_proj", "model.layers.71.mlp.experts.119.gate_proj", "model.layers.71.mlp.experts.120.gate_proj", "model.layers.71.mlp.experts.121.gate_proj", "model.layers.71.mlp.experts.122.gate_proj", "model.layers.71.mlp.experts.123.gate_proj", "model.layers.71.mlp.experts.124.gate_proj", "model.layers.71.mlp.experts.125.gate_proj", "model.layers.71.mlp.experts.126.gate_proj", "model.layers.71.mlp.experts.127.gate_proj", "model.layers.71.mlp.experts.128.gate_proj", "model.layers.71.mlp.experts.129.gate_proj", "model.layers.71.mlp.experts.130.gate_proj", "model.layers.71.mlp.experts.131.gate_proj", "model.layers.71.mlp.experts.132.gate_proj", "model.layers.71.mlp.experts.133.gate_proj", "model.layers.71.mlp.experts.134.gate_proj", "model.layers.71.mlp.experts.135.gate_proj", "model.layers.71.mlp.experts.136.gate_proj", "model.layers.71.mlp.experts.137.gate_proj", "model.layers.71.mlp.experts.138.gate_proj", "model.layers.71.mlp.experts.139.gate_proj", "model.layers.71.mlp.experts.140.gate_proj", "model.layers.71.mlp.experts.141.gate_proj", "model.layers.71.mlp.experts.142.gate_proj", "model.layers.71.mlp.experts.143.gate_proj", "model.layers.71.mlp.experts.144.gate_proj", "model.layers.71.mlp.experts.145.gate_proj", "model.layers.71.mlp.experts.146.gate_proj", "model.layers.71.mlp.experts.147.gate_proj", "model.layers.71.mlp.experts.148.gate_proj", "model.layers.71.mlp.experts.149.gate_proj", "model.layers.71.mlp.experts.150.gate_proj", "model.layers.71.mlp.experts.151.gate_proj", "model.layers.71.mlp.experts.152.gate_proj", "model.layers.71.mlp.experts.153.gate_proj", "model.layers.71.mlp.experts.154.gate_proj", "model.layers.71.mlp.experts.155.gate_proj", "model.layers.71.mlp.experts.156.gate_proj", "model.layers.71.mlp.experts.157.gate_proj", "model.layers.71.mlp.experts.158.gate_proj", "model.layers.71.mlp.experts.159.gate_proj", "model.layers.71.mlp.experts.0.up_proj", "model.layers.71.mlp.experts.1.up_proj", "model.layers.71.mlp.experts.2.up_proj", "model.layers.71.mlp.experts.3.up_proj", "model.layers.71.mlp.experts.4.up_proj", "model.layers.71.mlp.experts.5.up_proj", "model.layers.71.mlp.experts.6.up_proj", "model.layers.71.mlp.experts.7.up_proj", "model.layers.71.mlp.experts.8.up_proj", "model.layers.71.mlp.experts.9.up_proj", "model.layers.71.mlp.experts.10.up_proj", "model.layers.71.mlp.experts.11.up_proj", "model.layers.71.mlp.experts.12.up_proj", "model.layers.71.mlp.experts.13.up_proj", "model.layers.71.mlp.experts.14.up_proj", "model.layers.71.mlp.experts.15.up_proj", "model.layers.71.mlp.experts.16.up_proj", "model.layers.71.mlp.experts.17.up_proj", "model.layers.71.mlp.experts.18.up_proj", "model.layers.71.mlp.experts.19.up_proj", "model.layers.71.mlp.experts.20.up_proj", "model.layers.71.mlp.experts.21.up_proj", "model.layers.71.mlp.experts.22.up_proj", "model.layers.71.mlp.experts.23.up_proj", "model.layers.71.mlp.experts.24.up_proj", "model.layers.71.mlp.experts.25.up_proj", "model.layers.71.mlp.experts.26.up_proj", "model.layers.71.mlp.experts.27.up_proj", "model.layers.71.mlp.experts.28.up_proj", "model.layers.71.mlp.experts.29.up_proj", "model.layers.71.mlp.experts.30.up_proj", "model.layers.71.mlp.experts.31.up_proj", "model.layers.71.mlp.experts.32.up_proj", "model.layers.71.mlp.experts.33.up_proj", "model.layers.71.mlp.experts.34.up_proj", "model.layers.71.mlp.experts.35.up_proj", "model.layers.71.mlp.experts.36.up_proj", "model.layers.71.mlp.experts.37.up_proj", "model.layers.71.mlp.experts.38.up_proj", "model.layers.71.mlp.experts.39.up_proj", "model.layers.71.mlp.experts.40.up_proj", "model.layers.71.mlp.experts.41.up_proj", "model.layers.71.mlp.experts.42.up_proj", "model.layers.71.mlp.experts.43.up_proj", "model.layers.71.mlp.experts.44.up_proj", "model.layers.71.mlp.experts.45.up_proj", "model.layers.71.mlp.experts.46.up_proj", "model.layers.71.mlp.experts.47.up_proj", "model.layers.71.mlp.experts.48.up_proj", "model.layers.71.mlp.experts.49.up_proj", "model.layers.71.mlp.experts.50.up_proj", "model.layers.71.mlp.experts.51.up_proj", "model.layers.71.mlp.experts.52.up_proj", "model.layers.71.mlp.experts.53.up_proj", "model.layers.71.mlp.experts.54.up_proj", "model.layers.71.mlp.experts.55.up_proj", "model.layers.71.mlp.experts.56.up_proj", "model.layers.71.mlp.experts.57.up_proj", "model.layers.71.mlp.experts.58.up_proj", "model.layers.71.mlp.experts.59.up_proj", "model.layers.71.mlp.experts.60.up_proj", "model.layers.71.mlp.experts.61.up_proj", "model.layers.71.mlp.experts.62.up_proj", "model.layers.71.mlp.experts.63.up_proj", "model.layers.71.mlp.experts.64.up_proj", "model.layers.71.mlp.experts.65.up_proj", "model.layers.71.mlp.experts.66.up_proj", "model.layers.71.mlp.experts.67.up_proj", "model.layers.71.mlp.experts.68.up_proj", "model.layers.71.mlp.experts.69.up_proj", "model.layers.71.mlp.experts.70.up_proj", "model.layers.71.mlp.experts.71.up_proj", "model.layers.71.mlp.experts.72.up_proj", "model.layers.71.mlp.experts.73.up_proj", "model.layers.71.mlp.experts.74.up_proj", "model.layers.71.mlp.experts.75.up_proj", "model.layers.71.mlp.experts.76.up_proj", "model.layers.71.mlp.experts.77.up_proj", "model.layers.71.mlp.experts.78.up_proj", "model.layers.71.mlp.experts.79.up_proj", "model.layers.71.mlp.experts.80.up_proj", "model.layers.71.mlp.experts.81.up_proj", "model.layers.71.mlp.experts.82.up_proj", "model.layers.71.mlp.experts.83.up_proj", "model.layers.71.mlp.experts.84.up_proj", "model.layers.71.mlp.experts.85.up_proj", "model.layers.71.mlp.experts.86.up_proj", "model.layers.71.mlp.experts.87.up_proj", "model.layers.71.mlp.experts.88.up_proj", "model.layers.71.mlp.experts.89.up_proj", "model.layers.71.mlp.experts.90.up_proj", "model.layers.71.mlp.experts.91.up_proj", "model.layers.71.mlp.experts.92.up_proj", "model.layers.71.mlp.experts.93.up_proj", "model.layers.71.mlp.experts.94.up_proj", "model.layers.71.mlp.experts.95.up_proj", "model.layers.71.mlp.experts.96.up_proj", "model.layers.71.mlp.experts.97.up_proj", "model.layers.71.mlp.experts.98.up_proj", "model.layers.71.mlp.experts.99.up_proj", "model.layers.71.mlp.experts.100.up_proj", "model.layers.71.mlp.experts.101.up_proj", "model.layers.71.mlp.experts.102.up_proj", "model.layers.71.mlp.experts.103.up_proj", "model.layers.71.mlp.experts.104.up_proj", "model.layers.71.mlp.experts.105.up_proj", "model.layers.71.mlp.experts.106.up_proj", "model.layers.71.mlp.experts.107.up_proj", "model.layers.71.mlp.experts.108.up_proj", "model.layers.71.mlp.experts.109.up_proj", "model.layers.71.mlp.experts.110.up_proj", "model.layers.71.mlp.experts.111.up_proj", "model.layers.71.mlp.experts.112.up_proj", "model.layers.71.mlp.experts.113.up_proj", "model.layers.71.mlp.experts.114.up_proj", "model.layers.71.mlp.experts.115.up_proj", "model.layers.71.mlp.experts.116.up_proj", "model.layers.71.mlp.experts.117.up_proj", "model.layers.71.mlp.experts.118.up_proj", "model.layers.71.mlp.experts.119.up_proj", "model.layers.71.mlp.experts.120.up_proj", "model.layers.71.mlp.experts.121.up_proj", "model.layers.71.mlp.experts.122.up_proj", "model.layers.71.mlp.experts.123.up_proj", "model.layers.71.mlp.experts.124.up_proj", "model.layers.71.mlp.experts.125.up_proj", "model.layers.71.mlp.experts.126.up_proj", "model.layers.71.mlp.experts.127.up_proj", "model.layers.71.mlp.experts.128.up_proj", "model.layers.71.mlp.experts.129.up_proj", "model.layers.71.mlp.experts.130.up_proj", "model.layers.71.mlp.experts.131.up_proj", "model.layers.71.mlp.experts.132.up_proj", "model.layers.71.mlp.experts.133.up_proj", "model.layers.71.mlp.experts.134.up_proj", "model.layers.71.mlp.experts.135.up_proj", "model.layers.71.mlp.experts.136.up_proj", "model.layers.71.mlp.experts.137.up_proj", "model.layers.71.mlp.experts.138.up_proj", "model.layers.71.mlp.experts.139.up_proj", "model.layers.71.mlp.experts.140.up_proj", "model.layers.71.mlp.experts.141.up_proj", "model.layers.71.mlp.experts.142.up_proj", "model.layers.71.mlp.experts.143.up_proj", "model.layers.71.mlp.experts.144.up_proj", "model.layers.71.mlp.experts.145.up_proj", "model.layers.71.mlp.experts.146.up_proj", "model.layers.71.mlp.experts.147.up_proj", "model.layers.71.mlp.experts.148.up_proj", "model.layers.71.mlp.experts.149.up_proj", "model.layers.71.mlp.experts.150.up_proj", "model.layers.71.mlp.experts.151.up_proj", "model.layers.71.mlp.experts.152.up_proj", "model.layers.71.mlp.experts.153.up_proj", "model.layers.71.mlp.experts.154.up_proj", "model.layers.71.mlp.experts.155.up_proj", "model.layers.71.mlp.experts.156.up_proj", "model.layers.71.mlp.experts.157.up_proj", "model.layers.71.mlp.experts.158.up_proj", "model.layers.71.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0005435943603515625, "dbits": 2516582400 } ] }, { "idx": 428, "layers": [ "model.layers.71.mlp.experts.0.down_proj", "model.layers.71.mlp.experts.1.down_proj", "model.layers.71.mlp.experts.2.down_proj", "model.layers.71.mlp.experts.3.down_proj", "model.layers.71.mlp.experts.4.down_proj", "model.layers.71.mlp.experts.5.down_proj", "model.layers.71.mlp.experts.6.down_proj", "model.layers.71.mlp.experts.7.down_proj", "model.layers.71.mlp.experts.8.down_proj", "model.layers.71.mlp.experts.9.down_proj", "model.layers.71.mlp.experts.10.down_proj", "model.layers.71.mlp.experts.11.down_proj", "model.layers.71.mlp.experts.12.down_proj", "model.layers.71.mlp.experts.13.down_proj", "model.layers.71.mlp.experts.14.down_proj", "model.layers.71.mlp.experts.15.down_proj", "model.layers.71.mlp.experts.16.down_proj", "model.layers.71.mlp.experts.17.down_proj", "model.layers.71.mlp.experts.18.down_proj", "model.layers.71.mlp.experts.19.down_proj", "model.layers.71.mlp.experts.20.down_proj", "model.layers.71.mlp.experts.21.down_proj", "model.layers.71.mlp.experts.22.down_proj", "model.layers.71.mlp.experts.23.down_proj", "model.layers.71.mlp.experts.24.down_proj", "model.layers.71.mlp.experts.25.down_proj", "model.layers.71.mlp.experts.26.down_proj", "model.layers.71.mlp.experts.27.down_proj", "model.layers.71.mlp.experts.28.down_proj", "model.layers.71.mlp.experts.29.down_proj", "model.layers.71.mlp.experts.30.down_proj", "model.layers.71.mlp.experts.31.down_proj", "model.layers.71.mlp.experts.32.down_proj", "model.layers.71.mlp.experts.33.down_proj", "model.layers.71.mlp.experts.34.down_proj", "model.layers.71.mlp.experts.35.down_proj", "model.layers.71.mlp.experts.36.down_proj", "model.layers.71.mlp.experts.37.down_proj", "model.layers.71.mlp.experts.38.down_proj", "model.layers.71.mlp.experts.39.down_proj", "model.layers.71.mlp.experts.40.down_proj", "model.layers.71.mlp.experts.41.down_proj", "model.layers.71.mlp.experts.42.down_proj", "model.layers.71.mlp.experts.43.down_proj", "model.layers.71.mlp.experts.44.down_proj", "model.layers.71.mlp.experts.45.down_proj", "model.layers.71.mlp.experts.46.down_proj", "model.layers.71.mlp.experts.47.down_proj", "model.layers.71.mlp.experts.48.down_proj", "model.layers.71.mlp.experts.49.down_proj", "model.layers.71.mlp.experts.50.down_proj", "model.layers.71.mlp.experts.51.down_proj", "model.layers.71.mlp.experts.52.down_proj", "model.layers.71.mlp.experts.53.down_proj", "model.layers.71.mlp.experts.54.down_proj", "model.layers.71.mlp.experts.55.down_proj", "model.layers.71.mlp.experts.56.down_proj", "model.layers.71.mlp.experts.57.down_proj", "model.layers.71.mlp.experts.58.down_proj", "model.layers.71.mlp.experts.59.down_proj", "model.layers.71.mlp.experts.60.down_proj", "model.layers.71.mlp.experts.61.down_proj", "model.layers.71.mlp.experts.62.down_proj", "model.layers.71.mlp.experts.63.down_proj", "model.layers.71.mlp.experts.64.down_proj", "model.layers.71.mlp.experts.65.down_proj", "model.layers.71.mlp.experts.66.down_proj", "model.layers.71.mlp.experts.67.down_proj", "model.layers.71.mlp.experts.68.down_proj", "model.layers.71.mlp.experts.69.down_proj", "model.layers.71.mlp.experts.70.down_proj", "model.layers.71.mlp.experts.71.down_proj", "model.layers.71.mlp.experts.72.down_proj", "model.layers.71.mlp.experts.73.down_proj", "model.layers.71.mlp.experts.74.down_proj", "model.layers.71.mlp.experts.75.down_proj", "model.layers.71.mlp.experts.76.down_proj", "model.layers.71.mlp.experts.77.down_proj", "model.layers.71.mlp.experts.78.down_proj", "model.layers.71.mlp.experts.79.down_proj", "model.layers.71.mlp.experts.80.down_proj", "model.layers.71.mlp.experts.81.down_proj", "model.layers.71.mlp.experts.82.down_proj", "model.layers.71.mlp.experts.83.down_proj", "model.layers.71.mlp.experts.84.down_proj", "model.layers.71.mlp.experts.85.down_proj", "model.layers.71.mlp.experts.86.down_proj", "model.layers.71.mlp.experts.87.down_proj", "model.layers.71.mlp.experts.88.down_proj", "model.layers.71.mlp.experts.89.down_proj", "model.layers.71.mlp.experts.90.down_proj", "model.layers.71.mlp.experts.91.down_proj", "model.layers.71.mlp.experts.92.down_proj", "model.layers.71.mlp.experts.93.down_proj", "model.layers.71.mlp.experts.94.down_proj", "model.layers.71.mlp.experts.95.down_proj", "model.layers.71.mlp.experts.96.down_proj", "model.layers.71.mlp.experts.97.down_proj", "model.layers.71.mlp.experts.98.down_proj", "model.layers.71.mlp.experts.99.down_proj", "model.layers.71.mlp.experts.100.down_proj", "model.layers.71.mlp.experts.101.down_proj", "model.layers.71.mlp.experts.102.down_proj", "model.layers.71.mlp.experts.103.down_proj", "model.layers.71.mlp.experts.104.down_proj", "model.layers.71.mlp.experts.105.down_proj", "model.layers.71.mlp.experts.106.down_proj", "model.layers.71.mlp.experts.107.down_proj", "model.layers.71.mlp.experts.108.down_proj", "model.layers.71.mlp.experts.109.down_proj", "model.layers.71.mlp.experts.110.down_proj", "model.layers.71.mlp.experts.111.down_proj", "model.layers.71.mlp.experts.112.down_proj", "model.layers.71.mlp.experts.113.down_proj", "model.layers.71.mlp.experts.114.down_proj", "model.layers.71.mlp.experts.115.down_proj", "model.layers.71.mlp.experts.116.down_proj", "model.layers.71.mlp.experts.117.down_proj", "model.layers.71.mlp.experts.118.down_proj", "model.layers.71.mlp.experts.119.down_proj", "model.layers.71.mlp.experts.120.down_proj", "model.layers.71.mlp.experts.121.down_proj", "model.layers.71.mlp.experts.122.down_proj", "model.layers.71.mlp.experts.123.down_proj", "model.layers.71.mlp.experts.124.down_proj", "model.layers.71.mlp.experts.125.down_proj", "model.layers.71.mlp.experts.126.down_proj", "model.layers.71.mlp.experts.127.down_proj", "model.layers.71.mlp.experts.128.down_proj", "model.layers.71.mlp.experts.129.down_proj", "model.layers.71.mlp.experts.130.down_proj", "model.layers.71.mlp.experts.131.down_proj", "model.layers.71.mlp.experts.132.down_proj", "model.layers.71.mlp.experts.133.down_proj", "model.layers.71.mlp.experts.134.down_proj", "model.layers.71.mlp.experts.135.down_proj", "model.layers.71.mlp.experts.136.down_proj", "model.layers.71.mlp.experts.137.down_proj", "model.layers.71.mlp.experts.138.down_proj", "model.layers.71.mlp.experts.139.down_proj", "model.layers.71.mlp.experts.140.down_proj", "model.layers.71.mlp.experts.141.down_proj", "model.layers.71.mlp.experts.142.down_proj", "model.layers.71.mlp.experts.143.down_proj", "model.layers.71.mlp.experts.144.down_proj", "model.layers.71.mlp.experts.145.down_proj", "model.layers.71.mlp.experts.146.down_proj", "model.layers.71.mlp.experts.147.down_proj", "model.layers.71.mlp.experts.148.down_proj", "model.layers.71.mlp.experts.149.down_proj", "model.layers.71.mlp.experts.150.down_proj", "model.layers.71.mlp.experts.151.down_proj", "model.layers.71.mlp.experts.152.down_proj", "model.layers.71.mlp.experts.153.down_proj", "model.layers.71.mlp.experts.154.down_proj", "model.layers.71.mlp.experts.155.down_proj", "model.layers.71.mlp.experts.156.down_proj", "model.layers.71.mlp.experts.157.down_proj", "model.layers.71.mlp.experts.158.down_proj", "model.layers.71.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -8.33176076412423e-05, "dbits": 1258291200 } ] }, { "idx": 429, "layers": [ "model.layers.72.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00033966004848479114, "dbits": 62914560 } ] }, { "idx": 430, "layers": [ "model.layers.72.self_attn.k_proj", "model.layers.72.self_attn.v_proj" ], "candidates": [ { "dkld": 3.767386078834534e-05, "dbits": 10485760 } ] }, { "idx": 431, "layers": [ "model.layers.72.self_attn.o_proj" ], "candidates": [ { "dkld": -1.7742812633503302e-05, "dbits": 62914560 } ] }, { "idx": 432, "layers": [ "model.layers.72.mlp.shared_experts.gate_proj", "model.layers.72.mlp.shared_experts.up_proj", "model.layers.72.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.002107380330562625, "dbits": 23592960 } ] }, { "idx": 433, "layers": [ "model.layers.72.mlp.experts.0.gate_proj", "model.layers.72.mlp.experts.1.gate_proj", "model.layers.72.mlp.experts.2.gate_proj", "model.layers.72.mlp.experts.3.gate_proj", "model.layers.72.mlp.experts.4.gate_proj", "model.layers.72.mlp.experts.5.gate_proj", "model.layers.72.mlp.experts.6.gate_proj", "model.layers.72.mlp.experts.7.gate_proj", "model.layers.72.mlp.experts.8.gate_proj", "model.layers.72.mlp.experts.9.gate_proj", "model.layers.72.mlp.experts.10.gate_proj", "model.layers.72.mlp.experts.11.gate_proj", "model.layers.72.mlp.experts.12.gate_proj", "model.layers.72.mlp.experts.13.gate_proj", "model.layers.72.mlp.experts.14.gate_proj", "model.layers.72.mlp.experts.15.gate_proj", "model.layers.72.mlp.experts.16.gate_proj", "model.layers.72.mlp.experts.17.gate_proj", "model.layers.72.mlp.experts.18.gate_proj", "model.layers.72.mlp.experts.19.gate_proj", "model.layers.72.mlp.experts.20.gate_proj", "model.layers.72.mlp.experts.21.gate_proj", "model.layers.72.mlp.experts.22.gate_proj", "model.layers.72.mlp.experts.23.gate_proj", "model.layers.72.mlp.experts.24.gate_proj", "model.layers.72.mlp.experts.25.gate_proj", "model.layers.72.mlp.experts.26.gate_proj", "model.layers.72.mlp.experts.27.gate_proj", "model.layers.72.mlp.experts.28.gate_proj", "model.layers.72.mlp.experts.29.gate_proj", "model.layers.72.mlp.experts.30.gate_proj", "model.layers.72.mlp.experts.31.gate_proj", "model.layers.72.mlp.experts.32.gate_proj", "model.layers.72.mlp.experts.33.gate_proj", "model.layers.72.mlp.experts.34.gate_proj", "model.layers.72.mlp.experts.35.gate_proj", "model.layers.72.mlp.experts.36.gate_proj", "model.layers.72.mlp.experts.37.gate_proj", "model.layers.72.mlp.experts.38.gate_proj", "model.layers.72.mlp.experts.39.gate_proj", "model.layers.72.mlp.experts.40.gate_proj", "model.layers.72.mlp.experts.41.gate_proj", "model.layers.72.mlp.experts.42.gate_proj", "model.layers.72.mlp.experts.43.gate_proj", "model.layers.72.mlp.experts.44.gate_proj", "model.layers.72.mlp.experts.45.gate_proj", "model.layers.72.mlp.experts.46.gate_proj", "model.layers.72.mlp.experts.47.gate_proj", "model.layers.72.mlp.experts.48.gate_proj", "model.layers.72.mlp.experts.49.gate_proj", "model.layers.72.mlp.experts.50.gate_proj", "model.layers.72.mlp.experts.51.gate_proj", "model.layers.72.mlp.experts.52.gate_proj", "model.layers.72.mlp.experts.53.gate_proj", "model.layers.72.mlp.experts.54.gate_proj", "model.layers.72.mlp.experts.55.gate_proj", "model.layers.72.mlp.experts.56.gate_proj", "model.layers.72.mlp.experts.57.gate_proj", "model.layers.72.mlp.experts.58.gate_proj", "model.layers.72.mlp.experts.59.gate_proj", "model.layers.72.mlp.experts.60.gate_proj", "model.layers.72.mlp.experts.61.gate_proj", "model.layers.72.mlp.experts.62.gate_proj", "model.layers.72.mlp.experts.63.gate_proj", "model.layers.72.mlp.experts.64.gate_proj", "model.layers.72.mlp.experts.65.gate_proj", "model.layers.72.mlp.experts.66.gate_proj", "model.layers.72.mlp.experts.67.gate_proj", "model.layers.72.mlp.experts.68.gate_proj", "model.layers.72.mlp.experts.69.gate_proj", "model.layers.72.mlp.experts.70.gate_proj", "model.layers.72.mlp.experts.71.gate_proj", "model.layers.72.mlp.experts.72.gate_proj", "model.layers.72.mlp.experts.73.gate_proj", "model.layers.72.mlp.experts.74.gate_proj", "model.layers.72.mlp.experts.75.gate_proj", "model.layers.72.mlp.experts.76.gate_proj", "model.layers.72.mlp.experts.77.gate_proj", "model.layers.72.mlp.experts.78.gate_proj", "model.layers.72.mlp.experts.79.gate_proj", "model.layers.72.mlp.experts.80.gate_proj", "model.layers.72.mlp.experts.81.gate_proj", "model.layers.72.mlp.experts.82.gate_proj", "model.layers.72.mlp.experts.83.gate_proj", "model.layers.72.mlp.experts.84.gate_proj", "model.layers.72.mlp.experts.85.gate_proj", "model.layers.72.mlp.experts.86.gate_proj", "model.layers.72.mlp.experts.87.gate_proj", "model.layers.72.mlp.experts.88.gate_proj", "model.layers.72.mlp.experts.89.gate_proj", "model.layers.72.mlp.experts.90.gate_proj", "model.layers.72.mlp.experts.91.gate_proj", "model.layers.72.mlp.experts.92.gate_proj", "model.layers.72.mlp.experts.93.gate_proj", "model.layers.72.mlp.experts.94.gate_proj", "model.layers.72.mlp.experts.95.gate_proj", "model.layers.72.mlp.experts.96.gate_proj", "model.layers.72.mlp.experts.97.gate_proj", "model.layers.72.mlp.experts.98.gate_proj", "model.layers.72.mlp.experts.99.gate_proj", "model.layers.72.mlp.experts.100.gate_proj", "model.layers.72.mlp.experts.101.gate_proj", "model.layers.72.mlp.experts.102.gate_proj", "model.layers.72.mlp.experts.103.gate_proj", "model.layers.72.mlp.experts.104.gate_proj", "model.layers.72.mlp.experts.105.gate_proj", "model.layers.72.mlp.experts.106.gate_proj", "model.layers.72.mlp.experts.107.gate_proj", "model.layers.72.mlp.experts.108.gate_proj", "model.layers.72.mlp.experts.109.gate_proj", "model.layers.72.mlp.experts.110.gate_proj", "model.layers.72.mlp.experts.111.gate_proj", "model.layers.72.mlp.experts.112.gate_proj", "model.layers.72.mlp.experts.113.gate_proj", "model.layers.72.mlp.experts.114.gate_proj", "model.layers.72.mlp.experts.115.gate_proj", "model.layers.72.mlp.experts.116.gate_proj", "model.layers.72.mlp.experts.117.gate_proj", "model.layers.72.mlp.experts.118.gate_proj", "model.layers.72.mlp.experts.119.gate_proj", "model.layers.72.mlp.experts.120.gate_proj", "model.layers.72.mlp.experts.121.gate_proj", "model.layers.72.mlp.experts.122.gate_proj", "model.layers.72.mlp.experts.123.gate_proj", "model.layers.72.mlp.experts.124.gate_proj", "model.layers.72.mlp.experts.125.gate_proj", "model.layers.72.mlp.experts.126.gate_proj", "model.layers.72.mlp.experts.127.gate_proj", "model.layers.72.mlp.experts.128.gate_proj", "model.layers.72.mlp.experts.129.gate_proj", "model.layers.72.mlp.experts.130.gate_proj", "model.layers.72.mlp.experts.131.gate_proj", "model.layers.72.mlp.experts.132.gate_proj", "model.layers.72.mlp.experts.133.gate_proj", "model.layers.72.mlp.experts.134.gate_proj", "model.layers.72.mlp.experts.135.gate_proj", "model.layers.72.mlp.experts.136.gate_proj", "model.layers.72.mlp.experts.137.gate_proj", "model.layers.72.mlp.experts.138.gate_proj", "model.layers.72.mlp.experts.139.gate_proj", "model.layers.72.mlp.experts.140.gate_proj", "model.layers.72.mlp.experts.141.gate_proj", "model.layers.72.mlp.experts.142.gate_proj", "model.layers.72.mlp.experts.143.gate_proj", "model.layers.72.mlp.experts.144.gate_proj", "model.layers.72.mlp.experts.145.gate_proj", "model.layers.72.mlp.experts.146.gate_proj", "model.layers.72.mlp.experts.147.gate_proj", "model.layers.72.mlp.experts.148.gate_proj", "model.layers.72.mlp.experts.149.gate_proj", "model.layers.72.mlp.experts.150.gate_proj", "model.layers.72.mlp.experts.151.gate_proj", "model.layers.72.mlp.experts.152.gate_proj", "model.layers.72.mlp.experts.153.gate_proj", "model.layers.72.mlp.experts.154.gate_proj", "model.layers.72.mlp.experts.155.gate_proj", "model.layers.72.mlp.experts.156.gate_proj", "model.layers.72.mlp.experts.157.gate_proj", "model.layers.72.mlp.experts.158.gate_proj", "model.layers.72.mlp.experts.159.gate_proj", "model.layers.72.mlp.experts.0.up_proj", "model.layers.72.mlp.experts.1.up_proj", "model.layers.72.mlp.experts.2.up_proj", "model.layers.72.mlp.experts.3.up_proj", "model.layers.72.mlp.experts.4.up_proj", "model.layers.72.mlp.experts.5.up_proj", "model.layers.72.mlp.experts.6.up_proj", "model.layers.72.mlp.experts.7.up_proj", "model.layers.72.mlp.experts.8.up_proj", "model.layers.72.mlp.experts.9.up_proj", "model.layers.72.mlp.experts.10.up_proj", "model.layers.72.mlp.experts.11.up_proj", "model.layers.72.mlp.experts.12.up_proj", "model.layers.72.mlp.experts.13.up_proj", "model.layers.72.mlp.experts.14.up_proj", "model.layers.72.mlp.experts.15.up_proj", "model.layers.72.mlp.experts.16.up_proj", "model.layers.72.mlp.experts.17.up_proj", "model.layers.72.mlp.experts.18.up_proj", "model.layers.72.mlp.experts.19.up_proj", "model.layers.72.mlp.experts.20.up_proj", "model.layers.72.mlp.experts.21.up_proj", "model.layers.72.mlp.experts.22.up_proj", "model.layers.72.mlp.experts.23.up_proj", "model.layers.72.mlp.experts.24.up_proj", "model.layers.72.mlp.experts.25.up_proj", "model.layers.72.mlp.experts.26.up_proj", "model.layers.72.mlp.experts.27.up_proj", "model.layers.72.mlp.experts.28.up_proj", "model.layers.72.mlp.experts.29.up_proj", "model.layers.72.mlp.experts.30.up_proj", "model.layers.72.mlp.experts.31.up_proj", "model.layers.72.mlp.experts.32.up_proj", "model.layers.72.mlp.experts.33.up_proj", "model.layers.72.mlp.experts.34.up_proj", "model.layers.72.mlp.experts.35.up_proj", "model.layers.72.mlp.experts.36.up_proj", "model.layers.72.mlp.experts.37.up_proj", "model.layers.72.mlp.experts.38.up_proj", "model.layers.72.mlp.experts.39.up_proj", "model.layers.72.mlp.experts.40.up_proj", "model.layers.72.mlp.experts.41.up_proj", "model.layers.72.mlp.experts.42.up_proj", "model.layers.72.mlp.experts.43.up_proj", "model.layers.72.mlp.experts.44.up_proj", "model.layers.72.mlp.experts.45.up_proj", "model.layers.72.mlp.experts.46.up_proj", "model.layers.72.mlp.experts.47.up_proj", "model.layers.72.mlp.experts.48.up_proj", "model.layers.72.mlp.experts.49.up_proj", "model.layers.72.mlp.experts.50.up_proj", "model.layers.72.mlp.experts.51.up_proj", "model.layers.72.mlp.experts.52.up_proj", "model.layers.72.mlp.experts.53.up_proj", "model.layers.72.mlp.experts.54.up_proj", "model.layers.72.mlp.experts.55.up_proj", "model.layers.72.mlp.experts.56.up_proj", "model.layers.72.mlp.experts.57.up_proj", "model.layers.72.mlp.experts.58.up_proj", "model.layers.72.mlp.experts.59.up_proj", "model.layers.72.mlp.experts.60.up_proj", "model.layers.72.mlp.experts.61.up_proj", "model.layers.72.mlp.experts.62.up_proj", "model.layers.72.mlp.experts.63.up_proj", "model.layers.72.mlp.experts.64.up_proj", "model.layers.72.mlp.experts.65.up_proj", "model.layers.72.mlp.experts.66.up_proj", "model.layers.72.mlp.experts.67.up_proj", "model.layers.72.mlp.experts.68.up_proj", "model.layers.72.mlp.experts.69.up_proj", "model.layers.72.mlp.experts.70.up_proj", "model.layers.72.mlp.experts.71.up_proj", "model.layers.72.mlp.experts.72.up_proj", "model.layers.72.mlp.experts.73.up_proj", "model.layers.72.mlp.experts.74.up_proj", "model.layers.72.mlp.experts.75.up_proj", "model.layers.72.mlp.experts.76.up_proj", "model.layers.72.mlp.experts.77.up_proj", "model.layers.72.mlp.experts.78.up_proj", "model.layers.72.mlp.experts.79.up_proj", "model.layers.72.mlp.experts.80.up_proj", "model.layers.72.mlp.experts.81.up_proj", "model.layers.72.mlp.experts.82.up_proj", "model.layers.72.mlp.experts.83.up_proj", "model.layers.72.mlp.experts.84.up_proj", "model.layers.72.mlp.experts.85.up_proj", "model.layers.72.mlp.experts.86.up_proj", "model.layers.72.mlp.experts.87.up_proj", "model.layers.72.mlp.experts.88.up_proj", "model.layers.72.mlp.experts.89.up_proj", "model.layers.72.mlp.experts.90.up_proj", "model.layers.72.mlp.experts.91.up_proj", "model.layers.72.mlp.experts.92.up_proj", "model.layers.72.mlp.experts.93.up_proj", "model.layers.72.mlp.experts.94.up_proj", "model.layers.72.mlp.experts.95.up_proj", "model.layers.72.mlp.experts.96.up_proj", "model.layers.72.mlp.experts.97.up_proj", "model.layers.72.mlp.experts.98.up_proj", "model.layers.72.mlp.experts.99.up_proj", "model.layers.72.mlp.experts.100.up_proj", "model.layers.72.mlp.experts.101.up_proj", "model.layers.72.mlp.experts.102.up_proj", "model.layers.72.mlp.experts.103.up_proj", "model.layers.72.mlp.experts.104.up_proj", "model.layers.72.mlp.experts.105.up_proj", "model.layers.72.mlp.experts.106.up_proj", "model.layers.72.mlp.experts.107.up_proj", "model.layers.72.mlp.experts.108.up_proj", "model.layers.72.mlp.experts.109.up_proj", "model.layers.72.mlp.experts.110.up_proj", "model.layers.72.mlp.experts.111.up_proj", "model.layers.72.mlp.experts.112.up_proj", "model.layers.72.mlp.experts.113.up_proj", "model.layers.72.mlp.experts.114.up_proj", "model.layers.72.mlp.experts.115.up_proj", "model.layers.72.mlp.experts.116.up_proj", "model.layers.72.mlp.experts.117.up_proj", "model.layers.72.mlp.experts.118.up_proj", "model.layers.72.mlp.experts.119.up_proj", "model.layers.72.mlp.experts.120.up_proj", "model.layers.72.mlp.experts.121.up_proj", "model.layers.72.mlp.experts.122.up_proj", "model.layers.72.mlp.experts.123.up_proj", "model.layers.72.mlp.experts.124.up_proj", "model.layers.72.mlp.experts.125.up_proj", "model.layers.72.mlp.experts.126.up_proj", "model.layers.72.mlp.experts.127.up_proj", "model.layers.72.mlp.experts.128.up_proj", "model.layers.72.mlp.experts.129.up_proj", "model.layers.72.mlp.experts.130.up_proj", "model.layers.72.mlp.experts.131.up_proj", "model.layers.72.mlp.experts.132.up_proj", "model.layers.72.mlp.experts.133.up_proj", "model.layers.72.mlp.experts.134.up_proj", "model.layers.72.mlp.experts.135.up_proj", "model.layers.72.mlp.experts.136.up_proj", "model.layers.72.mlp.experts.137.up_proj", "model.layers.72.mlp.experts.138.up_proj", "model.layers.72.mlp.experts.139.up_proj", "model.layers.72.mlp.experts.140.up_proj", "model.layers.72.mlp.experts.141.up_proj", "model.layers.72.mlp.experts.142.up_proj", "model.layers.72.mlp.experts.143.up_proj", "model.layers.72.mlp.experts.144.up_proj", "model.layers.72.mlp.experts.145.up_proj", "model.layers.72.mlp.experts.146.up_proj", "model.layers.72.mlp.experts.147.up_proj", "model.layers.72.mlp.experts.148.up_proj", "model.layers.72.mlp.experts.149.up_proj", "model.layers.72.mlp.experts.150.up_proj", "model.layers.72.mlp.experts.151.up_proj", "model.layers.72.mlp.experts.152.up_proj", "model.layers.72.mlp.experts.153.up_proj", "model.layers.72.mlp.experts.154.up_proj", "model.layers.72.mlp.experts.155.up_proj", "model.layers.72.mlp.experts.156.up_proj", "model.layers.72.mlp.experts.157.up_proj", "model.layers.72.mlp.experts.158.up_proj", "model.layers.72.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00027051493525503956, "dbits": 2516582400 } ] }, { "idx": 434, "layers": [ "model.layers.72.mlp.experts.0.down_proj", "model.layers.72.mlp.experts.1.down_proj", "model.layers.72.mlp.experts.2.down_proj", "model.layers.72.mlp.experts.3.down_proj", "model.layers.72.mlp.experts.4.down_proj", "model.layers.72.mlp.experts.5.down_proj", "model.layers.72.mlp.experts.6.down_proj", "model.layers.72.mlp.experts.7.down_proj", "model.layers.72.mlp.experts.8.down_proj", "model.layers.72.mlp.experts.9.down_proj", "model.layers.72.mlp.experts.10.down_proj", "model.layers.72.mlp.experts.11.down_proj", "model.layers.72.mlp.experts.12.down_proj", "model.layers.72.mlp.experts.13.down_proj", "model.layers.72.mlp.experts.14.down_proj", "model.layers.72.mlp.experts.15.down_proj", "model.layers.72.mlp.experts.16.down_proj", "model.layers.72.mlp.experts.17.down_proj", "model.layers.72.mlp.experts.18.down_proj", "model.layers.72.mlp.experts.19.down_proj", "model.layers.72.mlp.experts.20.down_proj", "model.layers.72.mlp.experts.21.down_proj", "model.layers.72.mlp.experts.22.down_proj", "model.layers.72.mlp.experts.23.down_proj", "model.layers.72.mlp.experts.24.down_proj", "model.layers.72.mlp.experts.25.down_proj", "model.layers.72.mlp.experts.26.down_proj", "model.layers.72.mlp.experts.27.down_proj", "model.layers.72.mlp.experts.28.down_proj", "model.layers.72.mlp.experts.29.down_proj", "model.layers.72.mlp.experts.30.down_proj", "model.layers.72.mlp.experts.31.down_proj", "model.layers.72.mlp.experts.32.down_proj", "model.layers.72.mlp.experts.33.down_proj", "model.layers.72.mlp.experts.34.down_proj", "model.layers.72.mlp.experts.35.down_proj", "model.layers.72.mlp.experts.36.down_proj", "model.layers.72.mlp.experts.37.down_proj", "model.layers.72.mlp.experts.38.down_proj", "model.layers.72.mlp.experts.39.down_proj", "model.layers.72.mlp.experts.40.down_proj", "model.layers.72.mlp.experts.41.down_proj", "model.layers.72.mlp.experts.42.down_proj", "model.layers.72.mlp.experts.43.down_proj", "model.layers.72.mlp.experts.44.down_proj", "model.layers.72.mlp.experts.45.down_proj", "model.layers.72.mlp.experts.46.down_proj", "model.layers.72.mlp.experts.47.down_proj", "model.layers.72.mlp.experts.48.down_proj", "model.layers.72.mlp.experts.49.down_proj", "model.layers.72.mlp.experts.50.down_proj", "model.layers.72.mlp.experts.51.down_proj", "model.layers.72.mlp.experts.52.down_proj", "model.layers.72.mlp.experts.53.down_proj", "model.layers.72.mlp.experts.54.down_proj", "model.layers.72.mlp.experts.55.down_proj", "model.layers.72.mlp.experts.56.down_proj", "model.layers.72.mlp.experts.57.down_proj", "model.layers.72.mlp.experts.58.down_proj", "model.layers.72.mlp.experts.59.down_proj", "model.layers.72.mlp.experts.60.down_proj", "model.layers.72.mlp.experts.61.down_proj", "model.layers.72.mlp.experts.62.down_proj", "model.layers.72.mlp.experts.63.down_proj", "model.layers.72.mlp.experts.64.down_proj", "model.layers.72.mlp.experts.65.down_proj", "model.layers.72.mlp.experts.66.down_proj", "model.layers.72.mlp.experts.67.down_proj", "model.layers.72.mlp.experts.68.down_proj", "model.layers.72.mlp.experts.69.down_proj", "model.layers.72.mlp.experts.70.down_proj", "model.layers.72.mlp.experts.71.down_proj", "model.layers.72.mlp.experts.72.down_proj", "model.layers.72.mlp.experts.73.down_proj", "model.layers.72.mlp.experts.74.down_proj", "model.layers.72.mlp.experts.75.down_proj", "model.layers.72.mlp.experts.76.down_proj", "model.layers.72.mlp.experts.77.down_proj", "model.layers.72.mlp.experts.78.down_proj", "model.layers.72.mlp.experts.79.down_proj", "model.layers.72.mlp.experts.80.down_proj", "model.layers.72.mlp.experts.81.down_proj", "model.layers.72.mlp.experts.82.down_proj", "model.layers.72.mlp.experts.83.down_proj", "model.layers.72.mlp.experts.84.down_proj", "model.layers.72.mlp.experts.85.down_proj", "model.layers.72.mlp.experts.86.down_proj", "model.layers.72.mlp.experts.87.down_proj", "model.layers.72.mlp.experts.88.down_proj", "model.layers.72.mlp.experts.89.down_proj", "model.layers.72.mlp.experts.90.down_proj", "model.layers.72.mlp.experts.91.down_proj", "model.layers.72.mlp.experts.92.down_proj", "model.layers.72.mlp.experts.93.down_proj", "model.layers.72.mlp.experts.94.down_proj", "model.layers.72.mlp.experts.95.down_proj", "model.layers.72.mlp.experts.96.down_proj", "model.layers.72.mlp.experts.97.down_proj", "model.layers.72.mlp.experts.98.down_proj", "model.layers.72.mlp.experts.99.down_proj", "model.layers.72.mlp.experts.100.down_proj", "model.layers.72.mlp.experts.101.down_proj", "model.layers.72.mlp.experts.102.down_proj", "model.layers.72.mlp.experts.103.down_proj", "model.layers.72.mlp.experts.104.down_proj", "model.layers.72.mlp.experts.105.down_proj", "model.layers.72.mlp.experts.106.down_proj", "model.layers.72.mlp.experts.107.down_proj", "model.layers.72.mlp.experts.108.down_proj", "model.layers.72.mlp.experts.109.down_proj", "model.layers.72.mlp.experts.110.down_proj", "model.layers.72.mlp.experts.111.down_proj", "model.layers.72.mlp.experts.112.down_proj", "model.layers.72.mlp.experts.113.down_proj", "model.layers.72.mlp.experts.114.down_proj", "model.layers.72.mlp.experts.115.down_proj", "model.layers.72.mlp.experts.116.down_proj", "model.layers.72.mlp.experts.117.down_proj", "model.layers.72.mlp.experts.118.down_proj", "model.layers.72.mlp.experts.119.down_proj", "model.layers.72.mlp.experts.120.down_proj", "model.layers.72.mlp.experts.121.down_proj", "model.layers.72.mlp.experts.122.down_proj", "model.layers.72.mlp.experts.123.down_proj", "model.layers.72.mlp.experts.124.down_proj", "model.layers.72.mlp.experts.125.down_proj", "model.layers.72.mlp.experts.126.down_proj", "model.layers.72.mlp.experts.127.down_proj", "model.layers.72.mlp.experts.128.down_proj", "model.layers.72.mlp.experts.129.down_proj", "model.layers.72.mlp.experts.130.down_proj", "model.layers.72.mlp.experts.131.down_proj", "model.layers.72.mlp.experts.132.down_proj", "model.layers.72.mlp.experts.133.down_proj", "model.layers.72.mlp.experts.134.down_proj", "model.layers.72.mlp.experts.135.down_proj", "model.layers.72.mlp.experts.136.down_proj", "model.layers.72.mlp.experts.137.down_proj", "model.layers.72.mlp.experts.138.down_proj", "model.layers.72.mlp.experts.139.down_proj", "model.layers.72.mlp.experts.140.down_proj", "model.layers.72.mlp.experts.141.down_proj", "model.layers.72.mlp.experts.142.down_proj", "model.layers.72.mlp.experts.143.down_proj", "model.layers.72.mlp.experts.144.down_proj", "model.layers.72.mlp.experts.145.down_proj", "model.layers.72.mlp.experts.146.down_proj", "model.layers.72.mlp.experts.147.down_proj", "model.layers.72.mlp.experts.148.down_proj", "model.layers.72.mlp.experts.149.down_proj", "model.layers.72.mlp.experts.150.down_proj", "model.layers.72.mlp.experts.151.down_proj", "model.layers.72.mlp.experts.152.down_proj", "model.layers.72.mlp.experts.153.down_proj", "model.layers.72.mlp.experts.154.down_proj", "model.layers.72.mlp.experts.155.down_proj", "model.layers.72.mlp.experts.156.down_proj", "model.layers.72.mlp.experts.157.down_proj", "model.layers.72.mlp.experts.158.down_proj", "model.layers.72.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00024662241339684643, "dbits": 1258291200 } ] }, { "idx": 435, "layers": [ "model.layers.73.self_attn.q_proj" ], "candidates": [ { "dkld": 6.252676248547084e-05, "dbits": 62914560 } ] }, { "idx": 436, "layers": [ "model.layers.73.self_attn.k_proj", "model.layers.73.self_attn.v_proj" ], "candidates": [ { "dkld": -0.00022127553820611157, "dbits": 10485760 } ] }, { "idx": 437, "layers": [ "model.layers.73.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0001638077199458965, "dbits": 62914560 } ] }, { "idx": 438, "layers": [ "model.layers.73.mlp.shared_experts.gate_proj", "model.layers.73.mlp.shared_experts.up_proj", "model.layers.73.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0001856379210948833, "dbits": 23592960 } ] }, { "idx": 439, "layers": [ "model.layers.73.mlp.experts.0.gate_proj", "model.layers.73.mlp.experts.1.gate_proj", "model.layers.73.mlp.experts.2.gate_proj", "model.layers.73.mlp.experts.3.gate_proj", "model.layers.73.mlp.experts.4.gate_proj", "model.layers.73.mlp.experts.5.gate_proj", "model.layers.73.mlp.experts.6.gate_proj", "model.layers.73.mlp.experts.7.gate_proj", "model.layers.73.mlp.experts.8.gate_proj", "model.layers.73.mlp.experts.9.gate_proj", "model.layers.73.mlp.experts.10.gate_proj", "model.layers.73.mlp.experts.11.gate_proj", "model.layers.73.mlp.experts.12.gate_proj", "model.layers.73.mlp.experts.13.gate_proj", "model.layers.73.mlp.experts.14.gate_proj", "model.layers.73.mlp.experts.15.gate_proj", "model.layers.73.mlp.experts.16.gate_proj", "model.layers.73.mlp.experts.17.gate_proj", "model.layers.73.mlp.experts.18.gate_proj", "model.layers.73.mlp.experts.19.gate_proj", "model.layers.73.mlp.experts.20.gate_proj", "model.layers.73.mlp.experts.21.gate_proj", "model.layers.73.mlp.experts.22.gate_proj", "model.layers.73.mlp.experts.23.gate_proj", "model.layers.73.mlp.experts.24.gate_proj", "model.layers.73.mlp.experts.25.gate_proj", "model.layers.73.mlp.experts.26.gate_proj", "model.layers.73.mlp.experts.27.gate_proj", "model.layers.73.mlp.experts.28.gate_proj", "model.layers.73.mlp.experts.29.gate_proj", "model.layers.73.mlp.experts.30.gate_proj", "model.layers.73.mlp.experts.31.gate_proj", "model.layers.73.mlp.experts.32.gate_proj", "model.layers.73.mlp.experts.33.gate_proj", "model.layers.73.mlp.experts.34.gate_proj", "model.layers.73.mlp.experts.35.gate_proj", "model.layers.73.mlp.experts.36.gate_proj", "model.layers.73.mlp.experts.37.gate_proj", "model.layers.73.mlp.experts.38.gate_proj", "model.layers.73.mlp.experts.39.gate_proj", "model.layers.73.mlp.experts.40.gate_proj", "model.layers.73.mlp.experts.41.gate_proj", "model.layers.73.mlp.experts.42.gate_proj", "model.layers.73.mlp.experts.43.gate_proj", "model.layers.73.mlp.experts.44.gate_proj", "model.layers.73.mlp.experts.45.gate_proj", "model.layers.73.mlp.experts.46.gate_proj", "model.layers.73.mlp.experts.47.gate_proj", "model.layers.73.mlp.experts.48.gate_proj", "model.layers.73.mlp.experts.49.gate_proj", "model.layers.73.mlp.experts.50.gate_proj", "model.layers.73.mlp.experts.51.gate_proj", "model.layers.73.mlp.experts.52.gate_proj", "model.layers.73.mlp.experts.53.gate_proj", "model.layers.73.mlp.experts.54.gate_proj", "model.layers.73.mlp.experts.55.gate_proj", "model.layers.73.mlp.experts.56.gate_proj", "model.layers.73.mlp.experts.57.gate_proj", "model.layers.73.mlp.experts.58.gate_proj", "model.layers.73.mlp.experts.59.gate_proj", "model.layers.73.mlp.experts.60.gate_proj", "model.layers.73.mlp.experts.61.gate_proj", "model.layers.73.mlp.experts.62.gate_proj", "model.layers.73.mlp.experts.63.gate_proj", "model.layers.73.mlp.experts.64.gate_proj", "model.layers.73.mlp.experts.65.gate_proj", "model.layers.73.mlp.experts.66.gate_proj", "model.layers.73.mlp.experts.67.gate_proj", "model.layers.73.mlp.experts.68.gate_proj", "model.layers.73.mlp.experts.69.gate_proj", "model.layers.73.mlp.experts.70.gate_proj", "model.layers.73.mlp.experts.71.gate_proj", "model.layers.73.mlp.experts.72.gate_proj", "model.layers.73.mlp.experts.73.gate_proj", "model.layers.73.mlp.experts.74.gate_proj", "model.layers.73.mlp.experts.75.gate_proj", "model.layers.73.mlp.experts.76.gate_proj", "model.layers.73.mlp.experts.77.gate_proj", "model.layers.73.mlp.experts.78.gate_proj", "model.layers.73.mlp.experts.79.gate_proj", "model.layers.73.mlp.experts.80.gate_proj", "model.layers.73.mlp.experts.81.gate_proj", "model.layers.73.mlp.experts.82.gate_proj", "model.layers.73.mlp.experts.83.gate_proj", "model.layers.73.mlp.experts.84.gate_proj", "model.layers.73.mlp.experts.85.gate_proj", "model.layers.73.mlp.experts.86.gate_proj", "model.layers.73.mlp.experts.87.gate_proj", "model.layers.73.mlp.experts.88.gate_proj", "model.layers.73.mlp.experts.89.gate_proj", "model.layers.73.mlp.experts.90.gate_proj", "model.layers.73.mlp.experts.91.gate_proj", "model.layers.73.mlp.experts.92.gate_proj", "model.layers.73.mlp.experts.93.gate_proj", "model.layers.73.mlp.experts.94.gate_proj", "model.layers.73.mlp.experts.95.gate_proj", "model.layers.73.mlp.experts.96.gate_proj", "model.layers.73.mlp.experts.97.gate_proj", "model.layers.73.mlp.experts.98.gate_proj", "model.layers.73.mlp.experts.99.gate_proj", "model.layers.73.mlp.experts.100.gate_proj", "model.layers.73.mlp.experts.101.gate_proj", "model.layers.73.mlp.experts.102.gate_proj", "model.layers.73.mlp.experts.103.gate_proj", "model.layers.73.mlp.experts.104.gate_proj", "model.layers.73.mlp.experts.105.gate_proj", "model.layers.73.mlp.experts.106.gate_proj", "model.layers.73.mlp.experts.107.gate_proj", "model.layers.73.mlp.experts.108.gate_proj", "model.layers.73.mlp.experts.109.gate_proj", "model.layers.73.mlp.experts.110.gate_proj", "model.layers.73.mlp.experts.111.gate_proj", "model.layers.73.mlp.experts.112.gate_proj", "model.layers.73.mlp.experts.113.gate_proj", "model.layers.73.mlp.experts.114.gate_proj", "model.layers.73.mlp.experts.115.gate_proj", "model.layers.73.mlp.experts.116.gate_proj", "model.layers.73.mlp.experts.117.gate_proj", "model.layers.73.mlp.experts.118.gate_proj", "model.layers.73.mlp.experts.119.gate_proj", "model.layers.73.mlp.experts.120.gate_proj", "model.layers.73.mlp.experts.121.gate_proj", "model.layers.73.mlp.experts.122.gate_proj", "model.layers.73.mlp.experts.123.gate_proj", "model.layers.73.mlp.experts.124.gate_proj", "model.layers.73.mlp.experts.125.gate_proj", "model.layers.73.mlp.experts.126.gate_proj", "model.layers.73.mlp.experts.127.gate_proj", "model.layers.73.mlp.experts.128.gate_proj", "model.layers.73.mlp.experts.129.gate_proj", "model.layers.73.mlp.experts.130.gate_proj", "model.layers.73.mlp.experts.131.gate_proj", "model.layers.73.mlp.experts.132.gate_proj", "model.layers.73.mlp.experts.133.gate_proj", "model.layers.73.mlp.experts.134.gate_proj", "model.layers.73.mlp.experts.135.gate_proj", "model.layers.73.mlp.experts.136.gate_proj", "model.layers.73.mlp.experts.137.gate_proj", "model.layers.73.mlp.experts.138.gate_proj", "model.layers.73.mlp.experts.139.gate_proj", "model.layers.73.mlp.experts.140.gate_proj", "model.layers.73.mlp.experts.141.gate_proj", "model.layers.73.mlp.experts.142.gate_proj", "model.layers.73.mlp.experts.143.gate_proj", "model.layers.73.mlp.experts.144.gate_proj", "model.layers.73.mlp.experts.145.gate_proj", "model.layers.73.mlp.experts.146.gate_proj", "model.layers.73.mlp.experts.147.gate_proj", "model.layers.73.mlp.experts.148.gate_proj", "model.layers.73.mlp.experts.149.gate_proj", "model.layers.73.mlp.experts.150.gate_proj", "model.layers.73.mlp.experts.151.gate_proj", "model.layers.73.mlp.experts.152.gate_proj", "model.layers.73.mlp.experts.153.gate_proj", "model.layers.73.mlp.experts.154.gate_proj", "model.layers.73.mlp.experts.155.gate_proj", "model.layers.73.mlp.experts.156.gate_proj", "model.layers.73.mlp.experts.157.gate_proj", "model.layers.73.mlp.experts.158.gate_proj", "model.layers.73.mlp.experts.159.gate_proj", "model.layers.73.mlp.experts.0.up_proj", "model.layers.73.mlp.experts.1.up_proj", "model.layers.73.mlp.experts.2.up_proj", "model.layers.73.mlp.experts.3.up_proj", "model.layers.73.mlp.experts.4.up_proj", "model.layers.73.mlp.experts.5.up_proj", "model.layers.73.mlp.experts.6.up_proj", "model.layers.73.mlp.experts.7.up_proj", "model.layers.73.mlp.experts.8.up_proj", "model.layers.73.mlp.experts.9.up_proj", "model.layers.73.mlp.experts.10.up_proj", "model.layers.73.mlp.experts.11.up_proj", "model.layers.73.mlp.experts.12.up_proj", "model.layers.73.mlp.experts.13.up_proj", "model.layers.73.mlp.experts.14.up_proj", "model.layers.73.mlp.experts.15.up_proj", "model.layers.73.mlp.experts.16.up_proj", "model.layers.73.mlp.experts.17.up_proj", "model.layers.73.mlp.experts.18.up_proj", "model.layers.73.mlp.experts.19.up_proj", "model.layers.73.mlp.experts.20.up_proj", "model.layers.73.mlp.experts.21.up_proj", "model.layers.73.mlp.experts.22.up_proj", "model.layers.73.mlp.experts.23.up_proj", "model.layers.73.mlp.experts.24.up_proj", "model.layers.73.mlp.experts.25.up_proj", "model.layers.73.mlp.experts.26.up_proj", "model.layers.73.mlp.experts.27.up_proj", "model.layers.73.mlp.experts.28.up_proj", "model.layers.73.mlp.experts.29.up_proj", "model.layers.73.mlp.experts.30.up_proj", "model.layers.73.mlp.experts.31.up_proj", "model.layers.73.mlp.experts.32.up_proj", "model.layers.73.mlp.experts.33.up_proj", "model.layers.73.mlp.experts.34.up_proj", "model.layers.73.mlp.experts.35.up_proj", "model.layers.73.mlp.experts.36.up_proj", "model.layers.73.mlp.experts.37.up_proj", "model.layers.73.mlp.experts.38.up_proj", "model.layers.73.mlp.experts.39.up_proj", "model.layers.73.mlp.experts.40.up_proj", "model.layers.73.mlp.experts.41.up_proj", "model.layers.73.mlp.experts.42.up_proj", "model.layers.73.mlp.experts.43.up_proj", "model.layers.73.mlp.experts.44.up_proj", "model.layers.73.mlp.experts.45.up_proj", "model.layers.73.mlp.experts.46.up_proj", "model.layers.73.mlp.experts.47.up_proj", "model.layers.73.mlp.experts.48.up_proj", "model.layers.73.mlp.experts.49.up_proj", "model.layers.73.mlp.experts.50.up_proj", "model.layers.73.mlp.experts.51.up_proj", "model.layers.73.mlp.experts.52.up_proj", "model.layers.73.mlp.experts.53.up_proj", "model.layers.73.mlp.experts.54.up_proj", "model.layers.73.mlp.experts.55.up_proj", "model.layers.73.mlp.experts.56.up_proj", "model.layers.73.mlp.experts.57.up_proj", "model.layers.73.mlp.experts.58.up_proj", "model.layers.73.mlp.experts.59.up_proj", "model.layers.73.mlp.experts.60.up_proj", "model.layers.73.mlp.experts.61.up_proj", "model.layers.73.mlp.experts.62.up_proj", "model.layers.73.mlp.experts.63.up_proj", "model.layers.73.mlp.experts.64.up_proj", "model.layers.73.mlp.experts.65.up_proj", "model.layers.73.mlp.experts.66.up_proj", "model.layers.73.mlp.experts.67.up_proj", "model.layers.73.mlp.experts.68.up_proj", "model.layers.73.mlp.experts.69.up_proj", "model.layers.73.mlp.experts.70.up_proj", "model.layers.73.mlp.experts.71.up_proj", "model.layers.73.mlp.experts.72.up_proj", "model.layers.73.mlp.experts.73.up_proj", "model.layers.73.mlp.experts.74.up_proj", "model.layers.73.mlp.experts.75.up_proj", "model.layers.73.mlp.experts.76.up_proj", "model.layers.73.mlp.experts.77.up_proj", "model.layers.73.mlp.experts.78.up_proj", "model.layers.73.mlp.experts.79.up_proj", "model.layers.73.mlp.experts.80.up_proj", "model.layers.73.mlp.experts.81.up_proj", "model.layers.73.mlp.experts.82.up_proj", "model.layers.73.mlp.experts.83.up_proj", "model.layers.73.mlp.experts.84.up_proj", "model.layers.73.mlp.experts.85.up_proj", "model.layers.73.mlp.experts.86.up_proj", "model.layers.73.mlp.experts.87.up_proj", "model.layers.73.mlp.experts.88.up_proj", "model.layers.73.mlp.experts.89.up_proj", "model.layers.73.mlp.experts.90.up_proj", "model.layers.73.mlp.experts.91.up_proj", "model.layers.73.mlp.experts.92.up_proj", "model.layers.73.mlp.experts.93.up_proj", "model.layers.73.mlp.experts.94.up_proj", "model.layers.73.mlp.experts.95.up_proj", "model.layers.73.mlp.experts.96.up_proj", "model.layers.73.mlp.experts.97.up_proj", "model.layers.73.mlp.experts.98.up_proj", "model.layers.73.mlp.experts.99.up_proj", "model.layers.73.mlp.experts.100.up_proj", "model.layers.73.mlp.experts.101.up_proj", "model.layers.73.mlp.experts.102.up_proj", "model.layers.73.mlp.experts.103.up_proj", "model.layers.73.mlp.experts.104.up_proj", "model.layers.73.mlp.experts.105.up_proj", "model.layers.73.mlp.experts.106.up_proj", "model.layers.73.mlp.experts.107.up_proj", "model.layers.73.mlp.experts.108.up_proj", "model.layers.73.mlp.experts.109.up_proj", "model.layers.73.mlp.experts.110.up_proj", "model.layers.73.mlp.experts.111.up_proj", "model.layers.73.mlp.experts.112.up_proj", "model.layers.73.mlp.experts.113.up_proj", "model.layers.73.mlp.experts.114.up_proj", "model.layers.73.mlp.experts.115.up_proj", "model.layers.73.mlp.experts.116.up_proj", "model.layers.73.mlp.experts.117.up_proj", "model.layers.73.mlp.experts.118.up_proj", "model.layers.73.mlp.experts.119.up_proj", "model.layers.73.mlp.experts.120.up_proj", "model.layers.73.mlp.experts.121.up_proj", "model.layers.73.mlp.experts.122.up_proj", "model.layers.73.mlp.experts.123.up_proj", "model.layers.73.mlp.experts.124.up_proj", "model.layers.73.mlp.experts.125.up_proj", "model.layers.73.mlp.experts.126.up_proj", "model.layers.73.mlp.experts.127.up_proj", "model.layers.73.mlp.experts.128.up_proj", "model.layers.73.mlp.experts.129.up_proj", "model.layers.73.mlp.experts.130.up_proj", "model.layers.73.mlp.experts.131.up_proj", "model.layers.73.mlp.experts.132.up_proj", "model.layers.73.mlp.experts.133.up_proj", "model.layers.73.mlp.experts.134.up_proj", "model.layers.73.mlp.experts.135.up_proj", "model.layers.73.mlp.experts.136.up_proj", "model.layers.73.mlp.experts.137.up_proj", "model.layers.73.mlp.experts.138.up_proj", "model.layers.73.mlp.experts.139.up_proj", "model.layers.73.mlp.experts.140.up_proj", "model.layers.73.mlp.experts.141.up_proj", "model.layers.73.mlp.experts.142.up_proj", "model.layers.73.mlp.experts.143.up_proj", "model.layers.73.mlp.experts.144.up_proj", "model.layers.73.mlp.experts.145.up_proj", "model.layers.73.mlp.experts.146.up_proj", "model.layers.73.mlp.experts.147.up_proj", "model.layers.73.mlp.experts.148.up_proj", "model.layers.73.mlp.experts.149.up_proj", "model.layers.73.mlp.experts.150.up_proj", "model.layers.73.mlp.experts.151.up_proj", "model.layers.73.mlp.experts.152.up_proj", "model.layers.73.mlp.experts.153.up_proj", "model.layers.73.mlp.experts.154.up_proj", "model.layers.73.mlp.experts.155.up_proj", "model.layers.73.mlp.experts.156.up_proj", "model.layers.73.mlp.experts.157.up_proj", "model.layers.73.mlp.experts.158.up_proj", "model.layers.73.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00012170746922496312, "dbits": 2516582400 } ] }, { "idx": 440, "layers": [ "model.layers.73.mlp.experts.0.down_proj", "model.layers.73.mlp.experts.1.down_proj", "model.layers.73.mlp.experts.2.down_proj", "model.layers.73.mlp.experts.3.down_proj", "model.layers.73.mlp.experts.4.down_proj", "model.layers.73.mlp.experts.5.down_proj", "model.layers.73.mlp.experts.6.down_proj", "model.layers.73.mlp.experts.7.down_proj", "model.layers.73.mlp.experts.8.down_proj", "model.layers.73.mlp.experts.9.down_proj", "model.layers.73.mlp.experts.10.down_proj", "model.layers.73.mlp.experts.11.down_proj", "model.layers.73.mlp.experts.12.down_proj", "model.layers.73.mlp.experts.13.down_proj", "model.layers.73.mlp.experts.14.down_proj", "model.layers.73.mlp.experts.15.down_proj", "model.layers.73.mlp.experts.16.down_proj", "model.layers.73.mlp.experts.17.down_proj", "model.layers.73.mlp.experts.18.down_proj", "model.layers.73.mlp.experts.19.down_proj", "model.layers.73.mlp.experts.20.down_proj", "model.layers.73.mlp.experts.21.down_proj", "model.layers.73.mlp.experts.22.down_proj", "model.layers.73.mlp.experts.23.down_proj", "model.layers.73.mlp.experts.24.down_proj", "model.layers.73.mlp.experts.25.down_proj", "model.layers.73.mlp.experts.26.down_proj", "model.layers.73.mlp.experts.27.down_proj", "model.layers.73.mlp.experts.28.down_proj", "model.layers.73.mlp.experts.29.down_proj", "model.layers.73.mlp.experts.30.down_proj", "model.layers.73.mlp.experts.31.down_proj", "model.layers.73.mlp.experts.32.down_proj", "model.layers.73.mlp.experts.33.down_proj", "model.layers.73.mlp.experts.34.down_proj", "model.layers.73.mlp.experts.35.down_proj", "model.layers.73.mlp.experts.36.down_proj", "model.layers.73.mlp.experts.37.down_proj", "model.layers.73.mlp.experts.38.down_proj", "model.layers.73.mlp.experts.39.down_proj", "model.layers.73.mlp.experts.40.down_proj", "model.layers.73.mlp.experts.41.down_proj", "model.layers.73.mlp.experts.42.down_proj", "model.layers.73.mlp.experts.43.down_proj", "model.layers.73.mlp.experts.44.down_proj", "model.layers.73.mlp.experts.45.down_proj", "model.layers.73.mlp.experts.46.down_proj", "model.layers.73.mlp.experts.47.down_proj", "model.layers.73.mlp.experts.48.down_proj", "model.layers.73.mlp.experts.49.down_proj", "model.layers.73.mlp.experts.50.down_proj", "model.layers.73.mlp.experts.51.down_proj", "model.layers.73.mlp.experts.52.down_proj", "model.layers.73.mlp.experts.53.down_proj", "model.layers.73.mlp.experts.54.down_proj", "model.layers.73.mlp.experts.55.down_proj", "model.layers.73.mlp.experts.56.down_proj", "model.layers.73.mlp.experts.57.down_proj", "model.layers.73.mlp.experts.58.down_proj", "model.layers.73.mlp.experts.59.down_proj", "model.layers.73.mlp.experts.60.down_proj", "model.layers.73.mlp.experts.61.down_proj", "model.layers.73.mlp.experts.62.down_proj", "model.layers.73.mlp.experts.63.down_proj", "model.layers.73.mlp.experts.64.down_proj", "model.layers.73.mlp.experts.65.down_proj", "model.layers.73.mlp.experts.66.down_proj", "model.layers.73.mlp.experts.67.down_proj", "model.layers.73.mlp.experts.68.down_proj", "model.layers.73.mlp.experts.69.down_proj", "model.layers.73.mlp.experts.70.down_proj", "model.layers.73.mlp.experts.71.down_proj", "model.layers.73.mlp.experts.72.down_proj", "model.layers.73.mlp.experts.73.down_proj", "model.layers.73.mlp.experts.74.down_proj", "model.layers.73.mlp.experts.75.down_proj", "model.layers.73.mlp.experts.76.down_proj", "model.layers.73.mlp.experts.77.down_proj", "model.layers.73.mlp.experts.78.down_proj", "model.layers.73.mlp.experts.79.down_proj", "model.layers.73.mlp.experts.80.down_proj", "model.layers.73.mlp.experts.81.down_proj", "model.layers.73.mlp.experts.82.down_proj", "model.layers.73.mlp.experts.83.down_proj", "model.layers.73.mlp.experts.84.down_proj", "model.layers.73.mlp.experts.85.down_proj", "model.layers.73.mlp.experts.86.down_proj", "model.layers.73.mlp.experts.87.down_proj", "model.layers.73.mlp.experts.88.down_proj", "model.layers.73.mlp.experts.89.down_proj", "model.layers.73.mlp.experts.90.down_proj", "model.layers.73.mlp.experts.91.down_proj", "model.layers.73.mlp.experts.92.down_proj", "model.layers.73.mlp.experts.93.down_proj", "model.layers.73.mlp.experts.94.down_proj", "model.layers.73.mlp.experts.95.down_proj", "model.layers.73.mlp.experts.96.down_proj", "model.layers.73.mlp.experts.97.down_proj", "model.layers.73.mlp.experts.98.down_proj", "model.layers.73.mlp.experts.99.down_proj", "model.layers.73.mlp.experts.100.down_proj", "model.layers.73.mlp.experts.101.down_proj", "model.layers.73.mlp.experts.102.down_proj", "model.layers.73.mlp.experts.103.down_proj", "model.layers.73.mlp.experts.104.down_proj", "model.layers.73.mlp.experts.105.down_proj", "model.layers.73.mlp.experts.106.down_proj", "model.layers.73.mlp.experts.107.down_proj", "model.layers.73.mlp.experts.108.down_proj", "model.layers.73.mlp.experts.109.down_proj", "model.layers.73.mlp.experts.110.down_proj", "model.layers.73.mlp.experts.111.down_proj", "model.layers.73.mlp.experts.112.down_proj", "model.layers.73.mlp.experts.113.down_proj", "model.layers.73.mlp.experts.114.down_proj", "model.layers.73.mlp.experts.115.down_proj", "model.layers.73.mlp.experts.116.down_proj", "model.layers.73.mlp.experts.117.down_proj", "model.layers.73.mlp.experts.118.down_proj", "model.layers.73.mlp.experts.119.down_proj", "model.layers.73.mlp.experts.120.down_proj", "model.layers.73.mlp.experts.121.down_proj", "model.layers.73.mlp.experts.122.down_proj", "model.layers.73.mlp.experts.123.down_proj", "model.layers.73.mlp.experts.124.down_proj", "model.layers.73.mlp.experts.125.down_proj", "model.layers.73.mlp.experts.126.down_proj", "model.layers.73.mlp.experts.127.down_proj", "model.layers.73.mlp.experts.128.down_proj", "model.layers.73.mlp.experts.129.down_proj", "model.layers.73.mlp.experts.130.down_proj", "model.layers.73.mlp.experts.131.down_proj", "model.layers.73.mlp.experts.132.down_proj", "model.layers.73.mlp.experts.133.down_proj", "model.layers.73.mlp.experts.134.down_proj", "model.layers.73.mlp.experts.135.down_proj", "model.layers.73.mlp.experts.136.down_proj", "model.layers.73.mlp.experts.137.down_proj", "model.layers.73.mlp.experts.138.down_proj", "model.layers.73.mlp.experts.139.down_proj", "model.layers.73.mlp.experts.140.down_proj", "model.layers.73.mlp.experts.141.down_proj", "model.layers.73.mlp.experts.142.down_proj", "model.layers.73.mlp.experts.143.down_proj", "model.layers.73.mlp.experts.144.down_proj", "model.layers.73.mlp.experts.145.down_proj", "model.layers.73.mlp.experts.146.down_proj", "model.layers.73.mlp.experts.147.down_proj", "model.layers.73.mlp.experts.148.down_proj", "model.layers.73.mlp.experts.149.down_proj", "model.layers.73.mlp.experts.150.down_proj", "model.layers.73.mlp.experts.151.down_proj", "model.layers.73.mlp.experts.152.down_proj", "model.layers.73.mlp.experts.153.down_proj", "model.layers.73.mlp.experts.154.down_proj", "model.layers.73.mlp.experts.155.down_proj", "model.layers.73.mlp.experts.156.down_proj", "model.layers.73.mlp.experts.157.down_proj", "model.layers.73.mlp.experts.158.down_proj", "model.layers.73.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": 9.273141622540004e-05, "dbits": 1258291200 } ] }, { "idx": 441, "layers": [ "model.layers.74.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00012597739696501575, "dbits": 62914560 } ] }, { "idx": 442, "layers": [ "model.layers.74.self_attn.k_proj", "model.layers.74.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00018924474716186523, "dbits": 10485760 } ] }, { "idx": 443, "layers": [ "model.layers.74.self_attn.o_proj" ], "candidates": [ { "dkld": -2.8738379478476794e-05, "dbits": 62914560 } ] }, { "idx": 444, "layers": [ "model.layers.74.mlp.shared_experts.gate_proj", "model.layers.74.mlp.shared_experts.up_proj", "model.layers.74.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 3.765597939492382e-05, "dbits": 23592960 } ] }, { "idx": 445, "layers": [ "model.layers.74.mlp.experts.0.gate_proj", "model.layers.74.mlp.experts.1.gate_proj", "model.layers.74.mlp.experts.2.gate_proj", "model.layers.74.mlp.experts.3.gate_proj", "model.layers.74.mlp.experts.4.gate_proj", "model.layers.74.mlp.experts.5.gate_proj", "model.layers.74.mlp.experts.6.gate_proj", "model.layers.74.mlp.experts.7.gate_proj", "model.layers.74.mlp.experts.8.gate_proj", "model.layers.74.mlp.experts.9.gate_proj", "model.layers.74.mlp.experts.10.gate_proj", "model.layers.74.mlp.experts.11.gate_proj", "model.layers.74.mlp.experts.12.gate_proj", "model.layers.74.mlp.experts.13.gate_proj", "model.layers.74.mlp.experts.14.gate_proj", "model.layers.74.mlp.experts.15.gate_proj", "model.layers.74.mlp.experts.16.gate_proj", "model.layers.74.mlp.experts.17.gate_proj", "model.layers.74.mlp.experts.18.gate_proj", "model.layers.74.mlp.experts.19.gate_proj", "model.layers.74.mlp.experts.20.gate_proj", "model.layers.74.mlp.experts.21.gate_proj", "model.layers.74.mlp.experts.22.gate_proj", "model.layers.74.mlp.experts.23.gate_proj", "model.layers.74.mlp.experts.24.gate_proj", "model.layers.74.mlp.experts.25.gate_proj", "model.layers.74.mlp.experts.26.gate_proj", "model.layers.74.mlp.experts.27.gate_proj", "model.layers.74.mlp.experts.28.gate_proj", "model.layers.74.mlp.experts.29.gate_proj", "model.layers.74.mlp.experts.30.gate_proj", "model.layers.74.mlp.experts.31.gate_proj", "model.layers.74.mlp.experts.32.gate_proj", "model.layers.74.mlp.experts.33.gate_proj", "model.layers.74.mlp.experts.34.gate_proj", "model.layers.74.mlp.experts.35.gate_proj", "model.layers.74.mlp.experts.36.gate_proj", "model.layers.74.mlp.experts.37.gate_proj", "model.layers.74.mlp.experts.38.gate_proj", "model.layers.74.mlp.experts.39.gate_proj", "model.layers.74.mlp.experts.40.gate_proj", "model.layers.74.mlp.experts.41.gate_proj", "model.layers.74.mlp.experts.42.gate_proj", "model.layers.74.mlp.experts.43.gate_proj", "model.layers.74.mlp.experts.44.gate_proj", "model.layers.74.mlp.experts.45.gate_proj", "model.layers.74.mlp.experts.46.gate_proj", "model.layers.74.mlp.experts.47.gate_proj", "model.layers.74.mlp.experts.48.gate_proj", "model.layers.74.mlp.experts.49.gate_proj", "model.layers.74.mlp.experts.50.gate_proj", "model.layers.74.mlp.experts.51.gate_proj", "model.layers.74.mlp.experts.52.gate_proj", "model.layers.74.mlp.experts.53.gate_proj", "model.layers.74.mlp.experts.54.gate_proj", "model.layers.74.mlp.experts.55.gate_proj", "model.layers.74.mlp.experts.56.gate_proj", "model.layers.74.mlp.experts.57.gate_proj", "model.layers.74.mlp.experts.58.gate_proj", "model.layers.74.mlp.experts.59.gate_proj", "model.layers.74.mlp.experts.60.gate_proj", "model.layers.74.mlp.experts.61.gate_proj", "model.layers.74.mlp.experts.62.gate_proj", "model.layers.74.mlp.experts.63.gate_proj", "model.layers.74.mlp.experts.64.gate_proj", "model.layers.74.mlp.experts.65.gate_proj", "model.layers.74.mlp.experts.66.gate_proj", "model.layers.74.mlp.experts.67.gate_proj", "model.layers.74.mlp.experts.68.gate_proj", "model.layers.74.mlp.experts.69.gate_proj", "model.layers.74.mlp.experts.70.gate_proj", "model.layers.74.mlp.experts.71.gate_proj", "model.layers.74.mlp.experts.72.gate_proj", "model.layers.74.mlp.experts.73.gate_proj", "model.layers.74.mlp.experts.74.gate_proj", "model.layers.74.mlp.experts.75.gate_proj", "model.layers.74.mlp.experts.76.gate_proj", "model.layers.74.mlp.experts.77.gate_proj", "model.layers.74.mlp.experts.78.gate_proj", "model.layers.74.mlp.experts.79.gate_proj", "model.layers.74.mlp.experts.80.gate_proj", "model.layers.74.mlp.experts.81.gate_proj", "model.layers.74.mlp.experts.82.gate_proj", "model.layers.74.mlp.experts.83.gate_proj", "model.layers.74.mlp.experts.84.gate_proj", "model.layers.74.mlp.experts.85.gate_proj", "model.layers.74.mlp.experts.86.gate_proj", "model.layers.74.mlp.experts.87.gate_proj", "model.layers.74.mlp.experts.88.gate_proj", "model.layers.74.mlp.experts.89.gate_proj", "model.layers.74.mlp.experts.90.gate_proj", "model.layers.74.mlp.experts.91.gate_proj", "model.layers.74.mlp.experts.92.gate_proj", "model.layers.74.mlp.experts.93.gate_proj", "model.layers.74.mlp.experts.94.gate_proj", "model.layers.74.mlp.experts.95.gate_proj", "model.layers.74.mlp.experts.96.gate_proj", "model.layers.74.mlp.experts.97.gate_proj", "model.layers.74.mlp.experts.98.gate_proj", "model.layers.74.mlp.experts.99.gate_proj", "model.layers.74.mlp.experts.100.gate_proj", "model.layers.74.mlp.experts.101.gate_proj", "model.layers.74.mlp.experts.102.gate_proj", "model.layers.74.mlp.experts.103.gate_proj", "model.layers.74.mlp.experts.104.gate_proj", "model.layers.74.mlp.experts.105.gate_proj", "model.layers.74.mlp.experts.106.gate_proj", "model.layers.74.mlp.experts.107.gate_proj", "model.layers.74.mlp.experts.108.gate_proj", "model.layers.74.mlp.experts.109.gate_proj", "model.layers.74.mlp.experts.110.gate_proj", "model.layers.74.mlp.experts.111.gate_proj", "model.layers.74.mlp.experts.112.gate_proj", "model.layers.74.mlp.experts.113.gate_proj", "model.layers.74.mlp.experts.114.gate_proj", "model.layers.74.mlp.experts.115.gate_proj", "model.layers.74.mlp.experts.116.gate_proj", "model.layers.74.mlp.experts.117.gate_proj", "model.layers.74.mlp.experts.118.gate_proj", "model.layers.74.mlp.experts.119.gate_proj", "model.layers.74.mlp.experts.120.gate_proj", "model.layers.74.mlp.experts.121.gate_proj", "model.layers.74.mlp.experts.122.gate_proj", "model.layers.74.mlp.experts.123.gate_proj", "model.layers.74.mlp.experts.124.gate_proj", "model.layers.74.mlp.experts.125.gate_proj", "model.layers.74.mlp.experts.126.gate_proj", "model.layers.74.mlp.experts.127.gate_proj", "model.layers.74.mlp.experts.128.gate_proj", "model.layers.74.mlp.experts.129.gate_proj", "model.layers.74.mlp.experts.130.gate_proj", "model.layers.74.mlp.experts.131.gate_proj", "model.layers.74.mlp.experts.132.gate_proj", "model.layers.74.mlp.experts.133.gate_proj", "model.layers.74.mlp.experts.134.gate_proj", "model.layers.74.mlp.experts.135.gate_proj", "model.layers.74.mlp.experts.136.gate_proj", "model.layers.74.mlp.experts.137.gate_proj", "model.layers.74.mlp.experts.138.gate_proj", "model.layers.74.mlp.experts.139.gate_proj", "model.layers.74.mlp.experts.140.gate_proj", "model.layers.74.mlp.experts.141.gate_proj", "model.layers.74.mlp.experts.142.gate_proj", "model.layers.74.mlp.experts.143.gate_proj", "model.layers.74.mlp.experts.144.gate_proj", "model.layers.74.mlp.experts.145.gate_proj", "model.layers.74.mlp.experts.146.gate_proj", "model.layers.74.mlp.experts.147.gate_proj", "model.layers.74.mlp.experts.148.gate_proj", "model.layers.74.mlp.experts.149.gate_proj", "model.layers.74.mlp.experts.150.gate_proj", "model.layers.74.mlp.experts.151.gate_proj", "model.layers.74.mlp.experts.152.gate_proj", "model.layers.74.mlp.experts.153.gate_proj", "model.layers.74.mlp.experts.154.gate_proj", "model.layers.74.mlp.experts.155.gate_proj", "model.layers.74.mlp.experts.156.gate_proj", "model.layers.74.mlp.experts.157.gate_proj", "model.layers.74.mlp.experts.158.gate_proj", "model.layers.74.mlp.experts.159.gate_proj", "model.layers.74.mlp.experts.0.up_proj", "model.layers.74.mlp.experts.1.up_proj", "model.layers.74.mlp.experts.2.up_proj", "model.layers.74.mlp.experts.3.up_proj", "model.layers.74.mlp.experts.4.up_proj", "model.layers.74.mlp.experts.5.up_proj", "model.layers.74.mlp.experts.6.up_proj", "model.layers.74.mlp.experts.7.up_proj", "model.layers.74.mlp.experts.8.up_proj", "model.layers.74.mlp.experts.9.up_proj", "model.layers.74.mlp.experts.10.up_proj", "model.layers.74.mlp.experts.11.up_proj", "model.layers.74.mlp.experts.12.up_proj", "model.layers.74.mlp.experts.13.up_proj", "model.layers.74.mlp.experts.14.up_proj", "model.layers.74.mlp.experts.15.up_proj", "model.layers.74.mlp.experts.16.up_proj", "model.layers.74.mlp.experts.17.up_proj", "model.layers.74.mlp.experts.18.up_proj", "model.layers.74.mlp.experts.19.up_proj", "model.layers.74.mlp.experts.20.up_proj", "model.layers.74.mlp.experts.21.up_proj", "model.layers.74.mlp.experts.22.up_proj", "model.layers.74.mlp.experts.23.up_proj", "model.layers.74.mlp.experts.24.up_proj", "model.layers.74.mlp.experts.25.up_proj", "model.layers.74.mlp.experts.26.up_proj", "model.layers.74.mlp.experts.27.up_proj", "model.layers.74.mlp.experts.28.up_proj", "model.layers.74.mlp.experts.29.up_proj", "model.layers.74.mlp.experts.30.up_proj", "model.layers.74.mlp.experts.31.up_proj", "model.layers.74.mlp.experts.32.up_proj", "model.layers.74.mlp.experts.33.up_proj", "model.layers.74.mlp.experts.34.up_proj", "model.layers.74.mlp.experts.35.up_proj", "model.layers.74.mlp.experts.36.up_proj", "model.layers.74.mlp.experts.37.up_proj", "model.layers.74.mlp.experts.38.up_proj", "model.layers.74.mlp.experts.39.up_proj", "model.layers.74.mlp.experts.40.up_proj", "model.layers.74.mlp.experts.41.up_proj", "model.layers.74.mlp.experts.42.up_proj", "model.layers.74.mlp.experts.43.up_proj", "model.layers.74.mlp.experts.44.up_proj", "model.layers.74.mlp.experts.45.up_proj", "model.layers.74.mlp.experts.46.up_proj", "model.layers.74.mlp.experts.47.up_proj", "model.layers.74.mlp.experts.48.up_proj", "model.layers.74.mlp.experts.49.up_proj", "model.layers.74.mlp.experts.50.up_proj", "model.layers.74.mlp.experts.51.up_proj", "model.layers.74.mlp.experts.52.up_proj", "model.layers.74.mlp.experts.53.up_proj", "model.layers.74.mlp.experts.54.up_proj", "model.layers.74.mlp.experts.55.up_proj", "model.layers.74.mlp.experts.56.up_proj", "model.layers.74.mlp.experts.57.up_proj", "model.layers.74.mlp.experts.58.up_proj", "model.layers.74.mlp.experts.59.up_proj", "model.layers.74.mlp.experts.60.up_proj", "model.layers.74.mlp.experts.61.up_proj", "model.layers.74.mlp.experts.62.up_proj", "model.layers.74.mlp.experts.63.up_proj", "model.layers.74.mlp.experts.64.up_proj", "model.layers.74.mlp.experts.65.up_proj", "model.layers.74.mlp.experts.66.up_proj", "model.layers.74.mlp.experts.67.up_proj", "model.layers.74.mlp.experts.68.up_proj", "model.layers.74.mlp.experts.69.up_proj", "model.layers.74.mlp.experts.70.up_proj", "model.layers.74.mlp.experts.71.up_proj", "model.layers.74.mlp.experts.72.up_proj", "model.layers.74.mlp.experts.73.up_proj", "model.layers.74.mlp.experts.74.up_proj", "model.layers.74.mlp.experts.75.up_proj", "model.layers.74.mlp.experts.76.up_proj", "model.layers.74.mlp.experts.77.up_proj", "model.layers.74.mlp.experts.78.up_proj", "model.layers.74.mlp.experts.79.up_proj", "model.layers.74.mlp.experts.80.up_proj", "model.layers.74.mlp.experts.81.up_proj", "model.layers.74.mlp.experts.82.up_proj", "model.layers.74.mlp.experts.83.up_proj", "model.layers.74.mlp.experts.84.up_proj", "model.layers.74.mlp.experts.85.up_proj", "model.layers.74.mlp.experts.86.up_proj", "model.layers.74.mlp.experts.87.up_proj", "model.layers.74.mlp.experts.88.up_proj", "model.layers.74.mlp.experts.89.up_proj", "model.layers.74.mlp.experts.90.up_proj", "model.layers.74.mlp.experts.91.up_proj", "model.layers.74.mlp.experts.92.up_proj", "model.layers.74.mlp.experts.93.up_proj", "model.layers.74.mlp.experts.94.up_proj", "model.layers.74.mlp.experts.95.up_proj", "model.layers.74.mlp.experts.96.up_proj", "model.layers.74.mlp.experts.97.up_proj", "model.layers.74.mlp.experts.98.up_proj", "model.layers.74.mlp.experts.99.up_proj", "model.layers.74.mlp.experts.100.up_proj", "model.layers.74.mlp.experts.101.up_proj", "model.layers.74.mlp.experts.102.up_proj", "model.layers.74.mlp.experts.103.up_proj", "model.layers.74.mlp.experts.104.up_proj", "model.layers.74.mlp.experts.105.up_proj", "model.layers.74.mlp.experts.106.up_proj", "model.layers.74.mlp.experts.107.up_proj", "model.layers.74.mlp.experts.108.up_proj", "model.layers.74.mlp.experts.109.up_proj", "model.layers.74.mlp.experts.110.up_proj", "model.layers.74.mlp.experts.111.up_proj", "model.layers.74.mlp.experts.112.up_proj", "model.layers.74.mlp.experts.113.up_proj", "model.layers.74.mlp.experts.114.up_proj", "model.layers.74.mlp.experts.115.up_proj", "model.layers.74.mlp.experts.116.up_proj", "model.layers.74.mlp.experts.117.up_proj", "model.layers.74.mlp.experts.118.up_proj", "model.layers.74.mlp.experts.119.up_proj", "model.layers.74.mlp.experts.120.up_proj", "model.layers.74.mlp.experts.121.up_proj", "model.layers.74.mlp.experts.122.up_proj", "model.layers.74.mlp.experts.123.up_proj", "model.layers.74.mlp.experts.124.up_proj", "model.layers.74.mlp.experts.125.up_proj", "model.layers.74.mlp.experts.126.up_proj", "model.layers.74.mlp.experts.127.up_proj", "model.layers.74.mlp.experts.128.up_proj", "model.layers.74.mlp.experts.129.up_proj", "model.layers.74.mlp.experts.130.up_proj", "model.layers.74.mlp.experts.131.up_proj", "model.layers.74.mlp.experts.132.up_proj", "model.layers.74.mlp.experts.133.up_proj", "model.layers.74.mlp.experts.134.up_proj", "model.layers.74.mlp.experts.135.up_proj", "model.layers.74.mlp.experts.136.up_proj", "model.layers.74.mlp.experts.137.up_proj", "model.layers.74.mlp.experts.138.up_proj", "model.layers.74.mlp.experts.139.up_proj", "model.layers.74.mlp.experts.140.up_proj", "model.layers.74.mlp.experts.141.up_proj", "model.layers.74.mlp.experts.142.up_proj", "model.layers.74.mlp.experts.143.up_proj", "model.layers.74.mlp.experts.144.up_proj", "model.layers.74.mlp.experts.145.up_proj", "model.layers.74.mlp.experts.146.up_proj", "model.layers.74.mlp.experts.147.up_proj", "model.layers.74.mlp.experts.148.up_proj", "model.layers.74.mlp.experts.149.up_proj", "model.layers.74.mlp.experts.150.up_proj", "model.layers.74.mlp.experts.151.up_proj", "model.layers.74.mlp.experts.152.up_proj", "model.layers.74.mlp.experts.153.up_proj", "model.layers.74.mlp.experts.154.up_proj", "model.layers.74.mlp.experts.155.up_proj", "model.layers.74.mlp.experts.156.up_proj", "model.layers.74.mlp.experts.157.up_proj", "model.layers.74.mlp.experts.158.up_proj", "model.layers.74.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00023014545440674938, "dbits": 2516582400 } ] }, { "idx": 446, "layers": [ "model.layers.74.mlp.experts.0.down_proj", "model.layers.74.mlp.experts.1.down_proj", "model.layers.74.mlp.experts.2.down_proj", "model.layers.74.mlp.experts.3.down_proj", "model.layers.74.mlp.experts.4.down_proj", "model.layers.74.mlp.experts.5.down_proj", "model.layers.74.mlp.experts.6.down_proj", "model.layers.74.mlp.experts.7.down_proj", "model.layers.74.mlp.experts.8.down_proj", "model.layers.74.mlp.experts.9.down_proj", "model.layers.74.mlp.experts.10.down_proj", "model.layers.74.mlp.experts.11.down_proj", "model.layers.74.mlp.experts.12.down_proj", "model.layers.74.mlp.experts.13.down_proj", "model.layers.74.mlp.experts.14.down_proj", "model.layers.74.mlp.experts.15.down_proj", "model.layers.74.mlp.experts.16.down_proj", "model.layers.74.mlp.experts.17.down_proj", "model.layers.74.mlp.experts.18.down_proj", "model.layers.74.mlp.experts.19.down_proj", "model.layers.74.mlp.experts.20.down_proj", "model.layers.74.mlp.experts.21.down_proj", "model.layers.74.mlp.experts.22.down_proj", "model.layers.74.mlp.experts.23.down_proj", "model.layers.74.mlp.experts.24.down_proj", "model.layers.74.mlp.experts.25.down_proj", "model.layers.74.mlp.experts.26.down_proj", "model.layers.74.mlp.experts.27.down_proj", "model.layers.74.mlp.experts.28.down_proj", "model.layers.74.mlp.experts.29.down_proj", "model.layers.74.mlp.experts.30.down_proj", "model.layers.74.mlp.experts.31.down_proj", "model.layers.74.mlp.experts.32.down_proj", "model.layers.74.mlp.experts.33.down_proj", "model.layers.74.mlp.experts.34.down_proj", "model.layers.74.mlp.experts.35.down_proj", "model.layers.74.mlp.experts.36.down_proj", "model.layers.74.mlp.experts.37.down_proj", "model.layers.74.mlp.experts.38.down_proj", "model.layers.74.mlp.experts.39.down_proj", "model.layers.74.mlp.experts.40.down_proj", "model.layers.74.mlp.experts.41.down_proj", "model.layers.74.mlp.experts.42.down_proj", "model.layers.74.mlp.experts.43.down_proj", "model.layers.74.mlp.experts.44.down_proj", "model.layers.74.mlp.experts.45.down_proj", "model.layers.74.mlp.experts.46.down_proj", "model.layers.74.mlp.experts.47.down_proj", "model.layers.74.mlp.experts.48.down_proj", "model.layers.74.mlp.experts.49.down_proj", "model.layers.74.mlp.experts.50.down_proj", "model.layers.74.mlp.experts.51.down_proj", "model.layers.74.mlp.experts.52.down_proj", "model.layers.74.mlp.experts.53.down_proj", "model.layers.74.mlp.experts.54.down_proj", "model.layers.74.mlp.experts.55.down_proj", "model.layers.74.mlp.experts.56.down_proj", "model.layers.74.mlp.experts.57.down_proj", "model.layers.74.mlp.experts.58.down_proj", "model.layers.74.mlp.experts.59.down_proj", "model.layers.74.mlp.experts.60.down_proj", "model.layers.74.mlp.experts.61.down_proj", "model.layers.74.mlp.experts.62.down_proj", "model.layers.74.mlp.experts.63.down_proj", "model.layers.74.mlp.experts.64.down_proj", "model.layers.74.mlp.experts.65.down_proj", "model.layers.74.mlp.experts.66.down_proj", "model.layers.74.mlp.experts.67.down_proj", "model.layers.74.mlp.experts.68.down_proj", "model.layers.74.mlp.experts.69.down_proj", "model.layers.74.mlp.experts.70.down_proj", "model.layers.74.mlp.experts.71.down_proj", "model.layers.74.mlp.experts.72.down_proj", "model.layers.74.mlp.experts.73.down_proj", "model.layers.74.mlp.experts.74.down_proj", "model.layers.74.mlp.experts.75.down_proj", "model.layers.74.mlp.experts.76.down_proj", "model.layers.74.mlp.experts.77.down_proj", "model.layers.74.mlp.experts.78.down_proj", "model.layers.74.mlp.experts.79.down_proj", "model.layers.74.mlp.experts.80.down_proj", "model.layers.74.mlp.experts.81.down_proj", "model.layers.74.mlp.experts.82.down_proj", "model.layers.74.mlp.experts.83.down_proj", "model.layers.74.mlp.experts.84.down_proj", "model.layers.74.mlp.experts.85.down_proj", "model.layers.74.mlp.experts.86.down_proj", "model.layers.74.mlp.experts.87.down_proj", "model.layers.74.mlp.experts.88.down_proj", "model.layers.74.mlp.experts.89.down_proj", "model.layers.74.mlp.experts.90.down_proj", "model.layers.74.mlp.experts.91.down_proj", "model.layers.74.mlp.experts.92.down_proj", "model.layers.74.mlp.experts.93.down_proj", "model.layers.74.mlp.experts.94.down_proj", "model.layers.74.mlp.experts.95.down_proj", "model.layers.74.mlp.experts.96.down_proj", "model.layers.74.mlp.experts.97.down_proj", "model.layers.74.mlp.experts.98.down_proj", "model.layers.74.mlp.experts.99.down_proj", "model.layers.74.mlp.experts.100.down_proj", "model.layers.74.mlp.experts.101.down_proj", "model.layers.74.mlp.experts.102.down_proj", "model.layers.74.mlp.experts.103.down_proj", "model.layers.74.mlp.experts.104.down_proj", "model.layers.74.mlp.experts.105.down_proj", "model.layers.74.mlp.experts.106.down_proj", "model.layers.74.mlp.experts.107.down_proj", "model.layers.74.mlp.experts.108.down_proj", "model.layers.74.mlp.experts.109.down_proj", "model.layers.74.mlp.experts.110.down_proj", "model.layers.74.mlp.experts.111.down_proj", "model.layers.74.mlp.experts.112.down_proj", "model.layers.74.mlp.experts.113.down_proj", "model.layers.74.mlp.experts.114.down_proj", "model.layers.74.mlp.experts.115.down_proj", "model.layers.74.mlp.experts.116.down_proj", "model.layers.74.mlp.experts.117.down_proj", "model.layers.74.mlp.experts.118.down_proj", "model.layers.74.mlp.experts.119.down_proj", "model.layers.74.mlp.experts.120.down_proj", "model.layers.74.mlp.experts.121.down_proj", "model.layers.74.mlp.experts.122.down_proj", "model.layers.74.mlp.experts.123.down_proj", "model.layers.74.mlp.experts.124.down_proj", "model.layers.74.mlp.experts.125.down_proj", "model.layers.74.mlp.experts.126.down_proj", "model.layers.74.mlp.experts.127.down_proj", "model.layers.74.mlp.experts.128.down_proj", "model.layers.74.mlp.experts.129.down_proj", "model.layers.74.mlp.experts.130.down_proj", "model.layers.74.mlp.experts.131.down_proj", "model.layers.74.mlp.experts.132.down_proj", "model.layers.74.mlp.experts.133.down_proj", "model.layers.74.mlp.experts.134.down_proj", "model.layers.74.mlp.experts.135.down_proj", "model.layers.74.mlp.experts.136.down_proj", "model.layers.74.mlp.experts.137.down_proj", "model.layers.74.mlp.experts.138.down_proj", "model.layers.74.mlp.experts.139.down_proj", "model.layers.74.mlp.experts.140.down_proj", "model.layers.74.mlp.experts.141.down_proj", "model.layers.74.mlp.experts.142.down_proj", "model.layers.74.mlp.experts.143.down_proj", "model.layers.74.mlp.experts.144.down_proj", "model.layers.74.mlp.experts.145.down_proj", "model.layers.74.mlp.experts.146.down_proj", "model.layers.74.mlp.experts.147.down_proj", "model.layers.74.mlp.experts.148.down_proj", "model.layers.74.mlp.experts.149.down_proj", "model.layers.74.mlp.experts.150.down_proj", "model.layers.74.mlp.experts.151.down_proj", "model.layers.74.mlp.experts.152.down_proj", "model.layers.74.mlp.experts.153.down_proj", "model.layers.74.mlp.experts.154.down_proj", "model.layers.74.mlp.experts.155.down_proj", "model.layers.74.mlp.experts.156.down_proj", "model.layers.74.mlp.experts.157.down_proj", "model.layers.74.mlp.experts.158.down_proj", "model.layers.74.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0004471302032471036, "dbits": 1258291200 } ] }, { "idx": 447, "layers": [ "model.layers.75.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00043270811438561596, "dbits": 62914560 } ] }, { "idx": 448, "layers": [ "model.layers.75.self_attn.k_proj", "model.layers.75.self_attn.v_proj" ], "candidates": [ { "dkld": -0.00034391358494761937, "dbits": 10485760 } ] }, { "idx": 449, "layers": [ "model.layers.75.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0007868409156799316, "dbits": 62914560 } ] }, { "idx": 450, "layers": [ "model.layers.75.mlp.shared_experts.gate_proj", "model.layers.75.mlp.shared_experts.up_proj", "model.layers.75.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0006715618073940277, "dbits": 23592960 } ] }, { "idx": 451, "layers": [ "model.layers.75.mlp.experts.0.gate_proj", "model.layers.75.mlp.experts.1.gate_proj", "model.layers.75.mlp.experts.2.gate_proj", "model.layers.75.mlp.experts.3.gate_proj", "model.layers.75.mlp.experts.4.gate_proj", "model.layers.75.mlp.experts.5.gate_proj", "model.layers.75.mlp.experts.6.gate_proj", "model.layers.75.mlp.experts.7.gate_proj", "model.layers.75.mlp.experts.8.gate_proj", "model.layers.75.mlp.experts.9.gate_proj", "model.layers.75.mlp.experts.10.gate_proj", "model.layers.75.mlp.experts.11.gate_proj", "model.layers.75.mlp.experts.12.gate_proj", "model.layers.75.mlp.experts.13.gate_proj", "model.layers.75.mlp.experts.14.gate_proj", "model.layers.75.mlp.experts.15.gate_proj", "model.layers.75.mlp.experts.16.gate_proj", "model.layers.75.mlp.experts.17.gate_proj", "model.layers.75.mlp.experts.18.gate_proj", "model.layers.75.mlp.experts.19.gate_proj", "model.layers.75.mlp.experts.20.gate_proj", "model.layers.75.mlp.experts.21.gate_proj", "model.layers.75.mlp.experts.22.gate_proj", "model.layers.75.mlp.experts.23.gate_proj", "model.layers.75.mlp.experts.24.gate_proj", "model.layers.75.mlp.experts.25.gate_proj", "model.layers.75.mlp.experts.26.gate_proj", "model.layers.75.mlp.experts.27.gate_proj", "model.layers.75.mlp.experts.28.gate_proj", "model.layers.75.mlp.experts.29.gate_proj", "model.layers.75.mlp.experts.30.gate_proj", "model.layers.75.mlp.experts.31.gate_proj", "model.layers.75.mlp.experts.32.gate_proj", "model.layers.75.mlp.experts.33.gate_proj", "model.layers.75.mlp.experts.34.gate_proj", "model.layers.75.mlp.experts.35.gate_proj", "model.layers.75.mlp.experts.36.gate_proj", "model.layers.75.mlp.experts.37.gate_proj", "model.layers.75.mlp.experts.38.gate_proj", "model.layers.75.mlp.experts.39.gate_proj", "model.layers.75.mlp.experts.40.gate_proj", "model.layers.75.mlp.experts.41.gate_proj", "model.layers.75.mlp.experts.42.gate_proj", "model.layers.75.mlp.experts.43.gate_proj", "model.layers.75.mlp.experts.44.gate_proj", "model.layers.75.mlp.experts.45.gate_proj", "model.layers.75.mlp.experts.46.gate_proj", "model.layers.75.mlp.experts.47.gate_proj", "model.layers.75.mlp.experts.48.gate_proj", "model.layers.75.mlp.experts.49.gate_proj", "model.layers.75.mlp.experts.50.gate_proj", "model.layers.75.mlp.experts.51.gate_proj", "model.layers.75.mlp.experts.52.gate_proj", "model.layers.75.mlp.experts.53.gate_proj", "model.layers.75.mlp.experts.54.gate_proj", "model.layers.75.mlp.experts.55.gate_proj", "model.layers.75.mlp.experts.56.gate_proj", "model.layers.75.mlp.experts.57.gate_proj", "model.layers.75.mlp.experts.58.gate_proj", "model.layers.75.mlp.experts.59.gate_proj", "model.layers.75.mlp.experts.60.gate_proj", "model.layers.75.mlp.experts.61.gate_proj", "model.layers.75.mlp.experts.62.gate_proj", "model.layers.75.mlp.experts.63.gate_proj", "model.layers.75.mlp.experts.64.gate_proj", "model.layers.75.mlp.experts.65.gate_proj", "model.layers.75.mlp.experts.66.gate_proj", "model.layers.75.mlp.experts.67.gate_proj", "model.layers.75.mlp.experts.68.gate_proj", "model.layers.75.mlp.experts.69.gate_proj", "model.layers.75.mlp.experts.70.gate_proj", "model.layers.75.mlp.experts.71.gate_proj", "model.layers.75.mlp.experts.72.gate_proj", "model.layers.75.mlp.experts.73.gate_proj", "model.layers.75.mlp.experts.74.gate_proj", "model.layers.75.mlp.experts.75.gate_proj", "model.layers.75.mlp.experts.76.gate_proj", "model.layers.75.mlp.experts.77.gate_proj", "model.layers.75.mlp.experts.78.gate_proj", "model.layers.75.mlp.experts.79.gate_proj", "model.layers.75.mlp.experts.80.gate_proj", "model.layers.75.mlp.experts.81.gate_proj", "model.layers.75.mlp.experts.82.gate_proj", "model.layers.75.mlp.experts.83.gate_proj", "model.layers.75.mlp.experts.84.gate_proj", "model.layers.75.mlp.experts.85.gate_proj", "model.layers.75.mlp.experts.86.gate_proj", "model.layers.75.mlp.experts.87.gate_proj", "model.layers.75.mlp.experts.88.gate_proj", "model.layers.75.mlp.experts.89.gate_proj", "model.layers.75.mlp.experts.90.gate_proj", "model.layers.75.mlp.experts.91.gate_proj", "model.layers.75.mlp.experts.92.gate_proj", "model.layers.75.mlp.experts.93.gate_proj", "model.layers.75.mlp.experts.94.gate_proj", "model.layers.75.mlp.experts.95.gate_proj", "model.layers.75.mlp.experts.96.gate_proj", "model.layers.75.mlp.experts.97.gate_proj", "model.layers.75.mlp.experts.98.gate_proj", "model.layers.75.mlp.experts.99.gate_proj", "model.layers.75.mlp.experts.100.gate_proj", "model.layers.75.mlp.experts.101.gate_proj", "model.layers.75.mlp.experts.102.gate_proj", "model.layers.75.mlp.experts.103.gate_proj", "model.layers.75.mlp.experts.104.gate_proj", "model.layers.75.mlp.experts.105.gate_proj", "model.layers.75.mlp.experts.106.gate_proj", "model.layers.75.mlp.experts.107.gate_proj", "model.layers.75.mlp.experts.108.gate_proj", "model.layers.75.mlp.experts.109.gate_proj", "model.layers.75.mlp.experts.110.gate_proj", "model.layers.75.mlp.experts.111.gate_proj", "model.layers.75.mlp.experts.112.gate_proj", "model.layers.75.mlp.experts.113.gate_proj", "model.layers.75.mlp.experts.114.gate_proj", "model.layers.75.mlp.experts.115.gate_proj", "model.layers.75.mlp.experts.116.gate_proj", "model.layers.75.mlp.experts.117.gate_proj", "model.layers.75.mlp.experts.118.gate_proj", "model.layers.75.mlp.experts.119.gate_proj", "model.layers.75.mlp.experts.120.gate_proj", "model.layers.75.mlp.experts.121.gate_proj", "model.layers.75.mlp.experts.122.gate_proj", "model.layers.75.mlp.experts.123.gate_proj", "model.layers.75.mlp.experts.124.gate_proj", "model.layers.75.mlp.experts.125.gate_proj", "model.layers.75.mlp.experts.126.gate_proj", "model.layers.75.mlp.experts.127.gate_proj", "model.layers.75.mlp.experts.128.gate_proj", "model.layers.75.mlp.experts.129.gate_proj", "model.layers.75.mlp.experts.130.gate_proj", "model.layers.75.mlp.experts.131.gate_proj", "model.layers.75.mlp.experts.132.gate_proj", "model.layers.75.mlp.experts.133.gate_proj", "model.layers.75.mlp.experts.134.gate_proj", "model.layers.75.mlp.experts.135.gate_proj", "model.layers.75.mlp.experts.136.gate_proj", "model.layers.75.mlp.experts.137.gate_proj", "model.layers.75.mlp.experts.138.gate_proj", "model.layers.75.mlp.experts.139.gate_proj", "model.layers.75.mlp.experts.140.gate_proj", "model.layers.75.mlp.experts.141.gate_proj", "model.layers.75.mlp.experts.142.gate_proj", "model.layers.75.mlp.experts.143.gate_proj", "model.layers.75.mlp.experts.144.gate_proj", "model.layers.75.mlp.experts.145.gate_proj", "model.layers.75.mlp.experts.146.gate_proj", "model.layers.75.mlp.experts.147.gate_proj", "model.layers.75.mlp.experts.148.gate_proj", "model.layers.75.mlp.experts.149.gate_proj", "model.layers.75.mlp.experts.150.gate_proj", "model.layers.75.mlp.experts.151.gate_proj", "model.layers.75.mlp.experts.152.gate_proj", "model.layers.75.mlp.experts.153.gate_proj", "model.layers.75.mlp.experts.154.gate_proj", "model.layers.75.mlp.experts.155.gate_proj", "model.layers.75.mlp.experts.156.gate_proj", "model.layers.75.mlp.experts.157.gate_proj", "model.layers.75.mlp.experts.158.gate_proj", "model.layers.75.mlp.experts.159.gate_proj", "model.layers.75.mlp.experts.0.up_proj", "model.layers.75.mlp.experts.1.up_proj", "model.layers.75.mlp.experts.2.up_proj", "model.layers.75.mlp.experts.3.up_proj", "model.layers.75.mlp.experts.4.up_proj", "model.layers.75.mlp.experts.5.up_proj", "model.layers.75.mlp.experts.6.up_proj", "model.layers.75.mlp.experts.7.up_proj", "model.layers.75.mlp.experts.8.up_proj", "model.layers.75.mlp.experts.9.up_proj", "model.layers.75.mlp.experts.10.up_proj", "model.layers.75.mlp.experts.11.up_proj", "model.layers.75.mlp.experts.12.up_proj", "model.layers.75.mlp.experts.13.up_proj", "model.layers.75.mlp.experts.14.up_proj", "model.layers.75.mlp.experts.15.up_proj", "model.layers.75.mlp.experts.16.up_proj", "model.layers.75.mlp.experts.17.up_proj", "model.layers.75.mlp.experts.18.up_proj", "model.layers.75.mlp.experts.19.up_proj", "model.layers.75.mlp.experts.20.up_proj", "model.layers.75.mlp.experts.21.up_proj", "model.layers.75.mlp.experts.22.up_proj", "model.layers.75.mlp.experts.23.up_proj", "model.layers.75.mlp.experts.24.up_proj", "model.layers.75.mlp.experts.25.up_proj", "model.layers.75.mlp.experts.26.up_proj", "model.layers.75.mlp.experts.27.up_proj", "model.layers.75.mlp.experts.28.up_proj", "model.layers.75.mlp.experts.29.up_proj", "model.layers.75.mlp.experts.30.up_proj", "model.layers.75.mlp.experts.31.up_proj", "model.layers.75.mlp.experts.32.up_proj", "model.layers.75.mlp.experts.33.up_proj", "model.layers.75.mlp.experts.34.up_proj", "model.layers.75.mlp.experts.35.up_proj", "model.layers.75.mlp.experts.36.up_proj", "model.layers.75.mlp.experts.37.up_proj", "model.layers.75.mlp.experts.38.up_proj", "model.layers.75.mlp.experts.39.up_proj", "model.layers.75.mlp.experts.40.up_proj", "model.layers.75.mlp.experts.41.up_proj", "model.layers.75.mlp.experts.42.up_proj", "model.layers.75.mlp.experts.43.up_proj", "model.layers.75.mlp.experts.44.up_proj", "model.layers.75.mlp.experts.45.up_proj", "model.layers.75.mlp.experts.46.up_proj", "model.layers.75.mlp.experts.47.up_proj", "model.layers.75.mlp.experts.48.up_proj", "model.layers.75.mlp.experts.49.up_proj", "model.layers.75.mlp.experts.50.up_proj", "model.layers.75.mlp.experts.51.up_proj", "model.layers.75.mlp.experts.52.up_proj", "model.layers.75.mlp.experts.53.up_proj", "model.layers.75.mlp.experts.54.up_proj", "model.layers.75.mlp.experts.55.up_proj", "model.layers.75.mlp.experts.56.up_proj", "model.layers.75.mlp.experts.57.up_proj", "model.layers.75.mlp.experts.58.up_proj", "model.layers.75.mlp.experts.59.up_proj", "model.layers.75.mlp.experts.60.up_proj", "model.layers.75.mlp.experts.61.up_proj", "model.layers.75.mlp.experts.62.up_proj", "model.layers.75.mlp.experts.63.up_proj", "model.layers.75.mlp.experts.64.up_proj", "model.layers.75.mlp.experts.65.up_proj", "model.layers.75.mlp.experts.66.up_proj", "model.layers.75.mlp.experts.67.up_proj", "model.layers.75.mlp.experts.68.up_proj", "model.layers.75.mlp.experts.69.up_proj", "model.layers.75.mlp.experts.70.up_proj", "model.layers.75.mlp.experts.71.up_proj", "model.layers.75.mlp.experts.72.up_proj", "model.layers.75.mlp.experts.73.up_proj", "model.layers.75.mlp.experts.74.up_proj", "model.layers.75.mlp.experts.75.up_proj", "model.layers.75.mlp.experts.76.up_proj", "model.layers.75.mlp.experts.77.up_proj", "model.layers.75.mlp.experts.78.up_proj", "model.layers.75.mlp.experts.79.up_proj", "model.layers.75.mlp.experts.80.up_proj", "model.layers.75.mlp.experts.81.up_proj", "model.layers.75.mlp.experts.82.up_proj", "model.layers.75.mlp.experts.83.up_proj", "model.layers.75.mlp.experts.84.up_proj", "model.layers.75.mlp.experts.85.up_proj", "model.layers.75.mlp.experts.86.up_proj", "model.layers.75.mlp.experts.87.up_proj", "model.layers.75.mlp.experts.88.up_proj", "model.layers.75.mlp.experts.89.up_proj", "model.layers.75.mlp.experts.90.up_proj", "model.layers.75.mlp.experts.91.up_proj", "model.layers.75.mlp.experts.92.up_proj", "model.layers.75.mlp.experts.93.up_proj", "model.layers.75.mlp.experts.94.up_proj", "model.layers.75.mlp.experts.95.up_proj", "model.layers.75.mlp.experts.96.up_proj", "model.layers.75.mlp.experts.97.up_proj", "model.layers.75.mlp.experts.98.up_proj", "model.layers.75.mlp.experts.99.up_proj", "model.layers.75.mlp.experts.100.up_proj", "model.layers.75.mlp.experts.101.up_proj", "model.layers.75.mlp.experts.102.up_proj", "model.layers.75.mlp.experts.103.up_proj", "model.layers.75.mlp.experts.104.up_proj", "model.layers.75.mlp.experts.105.up_proj", "model.layers.75.mlp.experts.106.up_proj", "model.layers.75.mlp.experts.107.up_proj", "model.layers.75.mlp.experts.108.up_proj", "model.layers.75.mlp.experts.109.up_proj", "model.layers.75.mlp.experts.110.up_proj", "model.layers.75.mlp.experts.111.up_proj", "model.layers.75.mlp.experts.112.up_proj", "model.layers.75.mlp.experts.113.up_proj", "model.layers.75.mlp.experts.114.up_proj", "model.layers.75.mlp.experts.115.up_proj", "model.layers.75.mlp.experts.116.up_proj", "model.layers.75.mlp.experts.117.up_proj", "model.layers.75.mlp.experts.118.up_proj", "model.layers.75.mlp.experts.119.up_proj", "model.layers.75.mlp.experts.120.up_proj", "model.layers.75.mlp.experts.121.up_proj", "model.layers.75.mlp.experts.122.up_proj", "model.layers.75.mlp.experts.123.up_proj", "model.layers.75.mlp.experts.124.up_proj", "model.layers.75.mlp.experts.125.up_proj", "model.layers.75.mlp.experts.126.up_proj", "model.layers.75.mlp.experts.127.up_proj", "model.layers.75.mlp.experts.128.up_proj", "model.layers.75.mlp.experts.129.up_proj", "model.layers.75.mlp.experts.130.up_proj", "model.layers.75.mlp.experts.131.up_proj", "model.layers.75.mlp.experts.132.up_proj", "model.layers.75.mlp.experts.133.up_proj", "model.layers.75.mlp.experts.134.up_proj", "model.layers.75.mlp.experts.135.up_proj", "model.layers.75.mlp.experts.136.up_proj", "model.layers.75.mlp.experts.137.up_proj", "model.layers.75.mlp.experts.138.up_proj", "model.layers.75.mlp.experts.139.up_proj", "model.layers.75.mlp.experts.140.up_proj", "model.layers.75.mlp.experts.141.up_proj", "model.layers.75.mlp.experts.142.up_proj", "model.layers.75.mlp.experts.143.up_proj", "model.layers.75.mlp.experts.144.up_proj", "model.layers.75.mlp.experts.145.up_proj", "model.layers.75.mlp.experts.146.up_proj", "model.layers.75.mlp.experts.147.up_proj", "model.layers.75.mlp.experts.148.up_proj", "model.layers.75.mlp.experts.149.up_proj", "model.layers.75.mlp.experts.150.up_proj", "model.layers.75.mlp.experts.151.up_proj", "model.layers.75.mlp.experts.152.up_proj", "model.layers.75.mlp.experts.153.up_proj", "model.layers.75.mlp.experts.154.up_proj", "model.layers.75.mlp.experts.155.up_proj", "model.layers.75.mlp.experts.156.up_proj", "model.layers.75.mlp.experts.157.up_proj", "model.layers.75.mlp.experts.158.up_proj", "model.layers.75.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00020393356680870056, "dbits": 2516582400 } ] }, { "idx": 452, "layers": [ "model.layers.75.mlp.experts.0.down_proj", "model.layers.75.mlp.experts.1.down_proj", "model.layers.75.mlp.experts.2.down_proj", "model.layers.75.mlp.experts.3.down_proj", "model.layers.75.mlp.experts.4.down_proj", "model.layers.75.mlp.experts.5.down_proj", "model.layers.75.mlp.experts.6.down_proj", "model.layers.75.mlp.experts.7.down_proj", "model.layers.75.mlp.experts.8.down_proj", "model.layers.75.mlp.experts.9.down_proj", "model.layers.75.mlp.experts.10.down_proj", "model.layers.75.mlp.experts.11.down_proj", "model.layers.75.mlp.experts.12.down_proj", "model.layers.75.mlp.experts.13.down_proj", "model.layers.75.mlp.experts.14.down_proj", "model.layers.75.mlp.experts.15.down_proj", "model.layers.75.mlp.experts.16.down_proj", "model.layers.75.mlp.experts.17.down_proj", "model.layers.75.mlp.experts.18.down_proj", "model.layers.75.mlp.experts.19.down_proj", "model.layers.75.mlp.experts.20.down_proj", "model.layers.75.mlp.experts.21.down_proj", "model.layers.75.mlp.experts.22.down_proj", "model.layers.75.mlp.experts.23.down_proj", "model.layers.75.mlp.experts.24.down_proj", "model.layers.75.mlp.experts.25.down_proj", "model.layers.75.mlp.experts.26.down_proj", "model.layers.75.mlp.experts.27.down_proj", "model.layers.75.mlp.experts.28.down_proj", "model.layers.75.mlp.experts.29.down_proj", "model.layers.75.mlp.experts.30.down_proj", "model.layers.75.mlp.experts.31.down_proj", "model.layers.75.mlp.experts.32.down_proj", "model.layers.75.mlp.experts.33.down_proj", "model.layers.75.mlp.experts.34.down_proj", "model.layers.75.mlp.experts.35.down_proj", "model.layers.75.mlp.experts.36.down_proj", "model.layers.75.mlp.experts.37.down_proj", "model.layers.75.mlp.experts.38.down_proj", "model.layers.75.mlp.experts.39.down_proj", "model.layers.75.mlp.experts.40.down_proj", "model.layers.75.mlp.experts.41.down_proj", "model.layers.75.mlp.experts.42.down_proj", "model.layers.75.mlp.experts.43.down_proj", "model.layers.75.mlp.experts.44.down_proj", "model.layers.75.mlp.experts.45.down_proj", "model.layers.75.mlp.experts.46.down_proj", "model.layers.75.mlp.experts.47.down_proj", "model.layers.75.mlp.experts.48.down_proj", "model.layers.75.mlp.experts.49.down_proj", "model.layers.75.mlp.experts.50.down_proj", "model.layers.75.mlp.experts.51.down_proj", "model.layers.75.mlp.experts.52.down_proj", "model.layers.75.mlp.experts.53.down_proj", "model.layers.75.mlp.experts.54.down_proj", "model.layers.75.mlp.experts.55.down_proj", "model.layers.75.mlp.experts.56.down_proj", "model.layers.75.mlp.experts.57.down_proj", "model.layers.75.mlp.experts.58.down_proj", "model.layers.75.mlp.experts.59.down_proj", "model.layers.75.mlp.experts.60.down_proj", "model.layers.75.mlp.experts.61.down_proj", "model.layers.75.mlp.experts.62.down_proj", "model.layers.75.mlp.experts.63.down_proj", "model.layers.75.mlp.experts.64.down_proj", "model.layers.75.mlp.experts.65.down_proj", "model.layers.75.mlp.experts.66.down_proj", "model.layers.75.mlp.experts.67.down_proj", "model.layers.75.mlp.experts.68.down_proj", "model.layers.75.mlp.experts.69.down_proj", "model.layers.75.mlp.experts.70.down_proj", "model.layers.75.mlp.experts.71.down_proj", "model.layers.75.mlp.experts.72.down_proj", "model.layers.75.mlp.experts.73.down_proj", "model.layers.75.mlp.experts.74.down_proj", "model.layers.75.mlp.experts.75.down_proj", "model.layers.75.mlp.experts.76.down_proj", "model.layers.75.mlp.experts.77.down_proj", "model.layers.75.mlp.experts.78.down_proj", "model.layers.75.mlp.experts.79.down_proj", "model.layers.75.mlp.experts.80.down_proj", "model.layers.75.mlp.experts.81.down_proj", "model.layers.75.mlp.experts.82.down_proj", "model.layers.75.mlp.experts.83.down_proj", "model.layers.75.mlp.experts.84.down_proj", "model.layers.75.mlp.experts.85.down_proj", "model.layers.75.mlp.experts.86.down_proj", "model.layers.75.mlp.experts.87.down_proj", "model.layers.75.mlp.experts.88.down_proj", "model.layers.75.mlp.experts.89.down_proj", "model.layers.75.mlp.experts.90.down_proj", "model.layers.75.mlp.experts.91.down_proj", "model.layers.75.mlp.experts.92.down_proj", "model.layers.75.mlp.experts.93.down_proj", "model.layers.75.mlp.experts.94.down_proj", "model.layers.75.mlp.experts.95.down_proj", "model.layers.75.mlp.experts.96.down_proj", "model.layers.75.mlp.experts.97.down_proj", "model.layers.75.mlp.experts.98.down_proj", "model.layers.75.mlp.experts.99.down_proj", "model.layers.75.mlp.experts.100.down_proj", "model.layers.75.mlp.experts.101.down_proj", "model.layers.75.mlp.experts.102.down_proj", "model.layers.75.mlp.experts.103.down_proj", "model.layers.75.mlp.experts.104.down_proj", "model.layers.75.mlp.experts.105.down_proj", "model.layers.75.mlp.experts.106.down_proj", "model.layers.75.mlp.experts.107.down_proj", "model.layers.75.mlp.experts.108.down_proj", "model.layers.75.mlp.experts.109.down_proj", "model.layers.75.mlp.experts.110.down_proj", "model.layers.75.mlp.experts.111.down_proj", "model.layers.75.mlp.experts.112.down_proj", "model.layers.75.mlp.experts.113.down_proj", "model.layers.75.mlp.experts.114.down_proj", "model.layers.75.mlp.experts.115.down_proj", "model.layers.75.mlp.experts.116.down_proj", "model.layers.75.mlp.experts.117.down_proj", "model.layers.75.mlp.experts.118.down_proj", "model.layers.75.mlp.experts.119.down_proj", "model.layers.75.mlp.experts.120.down_proj", "model.layers.75.mlp.experts.121.down_proj", "model.layers.75.mlp.experts.122.down_proj", "model.layers.75.mlp.experts.123.down_proj", "model.layers.75.mlp.experts.124.down_proj", "model.layers.75.mlp.experts.125.down_proj", "model.layers.75.mlp.experts.126.down_proj", "model.layers.75.mlp.experts.127.down_proj", "model.layers.75.mlp.experts.128.down_proj", "model.layers.75.mlp.experts.129.down_proj", "model.layers.75.mlp.experts.130.down_proj", "model.layers.75.mlp.experts.131.down_proj", "model.layers.75.mlp.experts.132.down_proj", "model.layers.75.mlp.experts.133.down_proj", "model.layers.75.mlp.experts.134.down_proj", "model.layers.75.mlp.experts.135.down_proj", "model.layers.75.mlp.experts.136.down_proj", "model.layers.75.mlp.experts.137.down_proj", "model.layers.75.mlp.experts.138.down_proj", "model.layers.75.mlp.experts.139.down_proj", "model.layers.75.mlp.experts.140.down_proj", "model.layers.75.mlp.experts.141.down_proj", "model.layers.75.mlp.experts.142.down_proj", "model.layers.75.mlp.experts.143.down_proj", "model.layers.75.mlp.experts.144.down_proj", "model.layers.75.mlp.experts.145.down_proj", "model.layers.75.mlp.experts.146.down_proj", "model.layers.75.mlp.experts.147.down_proj", "model.layers.75.mlp.experts.148.down_proj", "model.layers.75.mlp.experts.149.down_proj", "model.layers.75.mlp.experts.150.down_proj", "model.layers.75.mlp.experts.151.down_proj", "model.layers.75.mlp.experts.152.down_proj", "model.layers.75.mlp.experts.153.down_proj", "model.layers.75.mlp.experts.154.down_proj", "model.layers.75.mlp.experts.155.down_proj", "model.layers.75.mlp.experts.156.down_proj", "model.layers.75.mlp.experts.157.down_proj", "model.layers.75.mlp.experts.158.down_proj", "model.layers.75.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.000176618993282307, "dbits": 1258291200 } ] }, { "idx": 453, "layers": [ "model.layers.76.self_attn.q_proj" ], "candidates": [ { "dkld": 1.647621393202625e-05, "dbits": 62914560 } ] }, { "idx": 454, "layers": [ "model.layers.76.self_attn.k_proj", "model.layers.76.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00026733651757238075, "dbits": 10485760 } ] }, { "idx": 455, "layers": [ "model.layers.76.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0003391385078430065, "dbits": 62914560 } ] }, { "idx": 456, "layers": [ "model.layers.76.mlp.shared_experts.gate_proj", "model.layers.76.mlp.shared_experts.up_proj", "model.layers.76.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00014149323105810963, "dbits": 23592960 } ] }, { "idx": 457, "layers": [ "model.layers.76.mlp.experts.0.gate_proj", "model.layers.76.mlp.experts.1.gate_proj", "model.layers.76.mlp.experts.2.gate_proj", "model.layers.76.mlp.experts.3.gate_proj", "model.layers.76.mlp.experts.4.gate_proj", "model.layers.76.mlp.experts.5.gate_proj", "model.layers.76.mlp.experts.6.gate_proj", "model.layers.76.mlp.experts.7.gate_proj", "model.layers.76.mlp.experts.8.gate_proj", "model.layers.76.mlp.experts.9.gate_proj", "model.layers.76.mlp.experts.10.gate_proj", "model.layers.76.mlp.experts.11.gate_proj", "model.layers.76.mlp.experts.12.gate_proj", "model.layers.76.mlp.experts.13.gate_proj", "model.layers.76.mlp.experts.14.gate_proj", "model.layers.76.mlp.experts.15.gate_proj", "model.layers.76.mlp.experts.16.gate_proj", "model.layers.76.mlp.experts.17.gate_proj", "model.layers.76.mlp.experts.18.gate_proj", "model.layers.76.mlp.experts.19.gate_proj", "model.layers.76.mlp.experts.20.gate_proj", "model.layers.76.mlp.experts.21.gate_proj", "model.layers.76.mlp.experts.22.gate_proj", "model.layers.76.mlp.experts.23.gate_proj", "model.layers.76.mlp.experts.24.gate_proj", "model.layers.76.mlp.experts.25.gate_proj", "model.layers.76.mlp.experts.26.gate_proj", "model.layers.76.mlp.experts.27.gate_proj", "model.layers.76.mlp.experts.28.gate_proj", "model.layers.76.mlp.experts.29.gate_proj", "model.layers.76.mlp.experts.30.gate_proj", "model.layers.76.mlp.experts.31.gate_proj", "model.layers.76.mlp.experts.32.gate_proj", "model.layers.76.mlp.experts.33.gate_proj", "model.layers.76.mlp.experts.34.gate_proj", "model.layers.76.mlp.experts.35.gate_proj", "model.layers.76.mlp.experts.36.gate_proj", "model.layers.76.mlp.experts.37.gate_proj", "model.layers.76.mlp.experts.38.gate_proj", "model.layers.76.mlp.experts.39.gate_proj", "model.layers.76.mlp.experts.40.gate_proj", "model.layers.76.mlp.experts.41.gate_proj", "model.layers.76.mlp.experts.42.gate_proj", "model.layers.76.mlp.experts.43.gate_proj", "model.layers.76.mlp.experts.44.gate_proj", "model.layers.76.mlp.experts.45.gate_proj", "model.layers.76.mlp.experts.46.gate_proj", "model.layers.76.mlp.experts.47.gate_proj", "model.layers.76.mlp.experts.48.gate_proj", "model.layers.76.mlp.experts.49.gate_proj", "model.layers.76.mlp.experts.50.gate_proj", "model.layers.76.mlp.experts.51.gate_proj", "model.layers.76.mlp.experts.52.gate_proj", "model.layers.76.mlp.experts.53.gate_proj", "model.layers.76.mlp.experts.54.gate_proj", "model.layers.76.mlp.experts.55.gate_proj", "model.layers.76.mlp.experts.56.gate_proj", "model.layers.76.mlp.experts.57.gate_proj", "model.layers.76.mlp.experts.58.gate_proj", "model.layers.76.mlp.experts.59.gate_proj", "model.layers.76.mlp.experts.60.gate_proj", "model.layers.76.mlp.experts.61.gate_proj", "model.layers.76.mlp.experts.62.gate_proj", "model.layers.76.mlp.experts.63.gate_proj", "model.layers.76.mlp.experts.64.gate_proj", "model.layers.76.mlp.experts.65.gate_proj", "model.layers.76.mlp.experts.66.gate_proj", "model.layers.76.mlp.experts.67.gate_proj", "model.layers.76.mlp.experts.68.gate_proj", "model.layers.76.mlp.experts.69.gate_proj", "model.layers.76.mlp.experts.70.gate_proj", "model.layers.76.mlp.experts.71.gate_proj", "model.layers.76.mlp.experts.72.gate_proj", "model.layers.76.mlp.experts.73.gate_proj", "model.layers.76.mlp.experts.74.gate_proj", "model.layers.76.mlp.experts.75.gate_proj", "model.layers.76.mlp.experts.76.gate_proj", "model.layers.76.mlp.experts.77.gate_proj", "model.layers.76.mlp.experts.78.gate_proj", "model.layers.76.mlp.experts.79.gate_proj", "model.layers.76.mlp.experts.80.gate_proj", "model.layers.76.mlp.experts.81.gate_proj", "model.layers.76.mlp.experts.82.gate_proj", "model.layers.76.mlp.experts.83.gate_proj", "model.layers.76.mlp.experts.84.gate_proj", "model.layers.76.mlp.experts.85.gate_proj", "model.layers.76.mlp.experts.86.gate_proj", "model.layers.76.mlp.experts.87.gate_proj", "model.layers.76.mlp.experts.88.gate_proj", "model.layers.76.mlp.experts.89.gate_proj", "model.layers.76.mlp.experts.90.gate_proj", "model.layers.76.mlp.experts.91.gate_proj", "model.layers.76.mlp.experts.92.gate_proj", "model.layers.76.mlp.experts.93.gate_proj", "model.layers.76.mlp.experts.94.gate_proj", "model.layers.76.mlp.experts.95.gate_proj", "model.layers.76.mlp.experts.96.gate_proj", "model.layers.76.mlp.experts.97.gate_proj", "model.layers.76.mlp.experts.98.gate_proj", "model.layers.76.mlp.experts.99.gate_proj", "model.layers.76.mlp.experts.100.gate_proj", "model.layers.76.mlp.experts.101.gate_proj", "model.layers.76.mlp.experts.102.gate_proj", "model.layers.76.mlp.experts.103.gate_proj", "model.layers.76.mlp.experts.104.gate_proj", "model.layers.76.mlp.experts.105.gate_proj", "model.layers.76.mlp.experts.106.gate_proj", "model.layers.76.mlp.experts.107.gate_proj", "model.layers.76.mlp.experts.108.gate_proj", "model.layers.76.mlp.experts.109.gate_proj", "model.layers.76.mlp.experts.110.gate_proj", "model.layers.76.mlp.experts.111.gate_proj", "model.layers.76.mlp.experts.112.gate_proj", "model.layers.76.mlp.experts.113.gate_proj", "model.layers.76.mlp.experts.114.gate_proj", "model.layers.76.mlp.experts.115.gate_proj", "model.layers.76.mlp.experts.116.gate_proj", "model.layers.76.mlp.experts.117.gate_proj", "model.layers.76.mlp.experts.118.gate_proj", "model.layers.76.mlp.experts.119.gate_proj", "model.layers.76.mlp.experts.120.gate_proj", "model.layers.76.mlp.experts.121.gate_proj", "model.layers.76.mlp.experts.122.gate_proj", "model.layers.76.mlp.experts.123.gate_proj", "model.layers.76.mlp.experts.124.gate_proj", "model.layers.76.mlp.experts.125.gate_proj", "model.layers.76.mlp.experts.126.gate_proj", "model.layers.76.mlp.experts.127.gate_proj", "model.layers.76.mlp.experts.128.gate_proj", "model.layers.76.mlp.experts.129.gate_proj", "model.layers.76.mlp.experts.130.gate_proj", "model.layers.76.mlp.experts.131.gate_proj", "model.layers.76.mlp.experts.132.gate_proj", "model.layers.76.mlp.experts.133.gate_proj", "model.layers.76.mlp.experts.134.gate_proj", "model.layers.76.mlp.experts.135.gate_proj", "model.layers.76.mlp.experts.136.gate_proj", "model.layers.76.mlp.experts.137.gate_proj", "model.layers.76.mlp.experts.138.gate_proj", "model.layers.76.mlp.experts.139.gate_proj", "model.layers.76.mlp.experts.140.gate_proj", "model.layers.76.mlp.experts.141.gate_proj", "model.layers.76.mlp.experts.142.gate_proj", "model.layers.76.mlp.experts.143.gate_proj", "model.layers.76.mlp.experts.144.gate_proj", "model.layers.76.mlp.experts.145.gate_proj", "model.layers.76.mlp.experts.146.gate_proj", "model.layers.76.mlp.experts.147.gate_proj", "model.layers.76.mlp.experts.148.gate_proj", "model.layers.76.mlp.experts.149.gate_proj", "model.layers.76.mlp.experts.150.gate_proj", "model.layers.76.mlp.experts.151.gate_proj", "model.layers.76.mlp.experts.152.gate_proj", "model.layers.76.mlp.experts.153.gate_proj", "model.layers.76.mlp.experts.154.gate_proj", "model.layers.76.mlp.experts.155.gate_proj", "model.layers.76.mlp.experts.156.gate_proj", "model.layers.76.mlp.experts.157.gate_proj", "model.layers.76.mlp.experts.158.gate_proj", "model.layers.76.mlp.experts.159.gate_proj", "model.layers.76.mlp.experts.0.up_proj", "model.layers.76.mlp.experts.1.up_proj", "model.layers.76.mlp.experts.2.up_proj", "model.layers.76.mlp.experts.3.up_proj", "model.layers.76.mlp.experts.4.up_proj", "model.layers.76.mlp.experts.5.up_proj", "model.layers.76.mlp.experts.6.up_proj", "model.layers.76.mlp.experts.7.up_proj", "model.layers.76.mlp.experts.8.up_proj", "model.layers.76.mlp.experts.9.up_proj", "model.layers.76.mlp.experts.10.up_proj", "model.layers.76.mlp.experts.11.up_proj", "model.layers.76.mlp.experts.12.up_proj", "model.layers.76.mlp.experts.13.up_proj", "model.layers.76.mlp.experts.14.up_proj", "model.layers.76.mlp.experts.15.up_proj", "model.layers.76.mlp.experts.16.up_proj", "model.layers.76.mlp.experts.17.up_proj", "model.layers.76.mlp.experts.18.up_proj", "model.layers.76.mlp.experts.19.up_proj", "model.layers.76.mlp.experts.20.up_proj", "model.layers.76.mlp.experts.21.up_proj", "model.layers.76.mlp.experts.22.up_proj", "model.layers.76.mlp.experts.23.up_proj", "model.layers.76.mlp.experts.24.up_proj", "model.layers.76.mlp.experts.25.up_proj", "model.layers.76.mlp.experts.26.up_proj", "model.layers.76.mlp.experts.27.up_proj", "model.layers.76.mlp.experts.28.up_proj", "model.layers.76.mlp.experts.29.up_proj", "model.layers.76.mlp.experts.30.up_proj", "model.layers.76.mlp.experts.31.up_proj", "model.layers.76.mlp.experts.32.up_proj", "model.layers.76.mlp.experts.33.up_proj", "model.layers.76.mlp.experts.34.up_proj", "model.layers.76.mlp.experts.35.up_proj", "model.layers.76.mlp.experts.36.up_proj", "model.layers.76.mlp.experts.37.up_proj", "model.layers.76.mlp.experts.38.up_proj", "model.layers.76.mlp.experts.39.up_proj", "model.layers.76.mlp.experts.40.up_proj", "model.layers.76.mlp.experts.41.up_proj", "model.layers.76.mlp.experts.42.up_proj", "model.layers.76.mlp.experts.43.up_proj", "model.layers.76.mlp.experts.44.up_proj", "model.layers.76.mlp.experts.45.up_proj", "model.layers.76.mlp.experts.46.up_proj", "model.layers.76.mlp.experts.47.up_proj", "model.layers.76.mlp.experts.48.up_proj", "model.layers.76.mlp.experts.49.up_proj", "model.layers.76.mlp.experts.50.up_proj", "model.layers.76.mlp.experts.51.up_proj", "model.layers.76.mlp.experts.52.up_proj", "model.layers.76.mlp.experts.53.up_proj", "model.layers.76.mlp.experts.54.up_proj", "model.layers.76.mlp.experts.55.up_proj", "model.layers.76.mlp.experts.56.up_proj", "model.layers.76.mlp.experts.57.up_proj", "model.layers.76.mlp.experts.58.up_proj", "model.layers.76.mlp.experts.59.up_proj", "model.layers.76.mlp.experts.60.up_proj", "model.layers.76.mlp.experts.61.up_proj", "model.layers.76.mlp.experts.62.up_proj", "model.layers.76.mlp.experts.63.up_proj", "model.layers.76.mlp.experts.64.up_proj", "model.layers.76.mlp.experts.65.up_proj", "model.layers.76.mlp.experts.66.up_proj", "model.layers.76.mlp.experts.67.up_proj", "model.layers.76.mlp.experts.68.up_proj", "model.layers.76.mlp.experts.69.up_proj", "model.layers.76.mlp.experts.70.up_proj", "model.layers.76.mlp.experts.71.up_proj", "model.layers.76.mlp.experts.72.up_proj", "model.layers.76.mlp.experts.73.up_proj", "model.layers.76.mlp.experts.74.up_proj", "model.layers.76.mlp.experts.75.up_proj", "model.layers.76.mlp.experts.76.up_proj", "model.layers.76.mlp.experts.77.up_proj", "model.layers.76.mlp.experts.78.up_proj", "model.layers.76.mlp.experts.79.up_proj", "model.layers.76.mlp.experts.80.up_proj", "model.layers.76.mlp.experts.81.up_proj", "model.layers.76.mlp.experts.82.up_proj", "model.layers.76.mlp.experts.83.up_proj", "model.layers.76.mlp.experts.84.up_proj", "model.layers.76.mlp.experts.85.up_proj", "model.layers.76.mlp.experts.86.up_proj", "model.layers.76.mlp.experts.87.up_proj", "model.layers.76.mlp.experts.88.up_proj", "model.layers.76.mlp.experts.89.up_proj", "model.layers.76.mlp.experts.90.up_proj", "model.layers.76.mlp.experts.91.up_proj", "model.layers.76.mlp.experts.92.up_proj", "model.layers.76.mlp.experts.93.up_proj", "model.layers.76.mlp.experts.94.up_proj", "model.layers.76.mlp.experts.95.up_proj", "model.layers.76.mlp.experts.96.up_proj", "model.layers.76.mlp.experts.97.up_proj", "model.layers.76.mlp.experts.98.up_proj", "model.layers.76.mlp.experts.99.up_proj", "model.layers.76.mlp.experts.100.up_proj", "model.layers.76.mlp.experts.101.up_proj", "model.layers.76.mlp.experts.102.up_proj", "model.layers.76.mlp.experts.103.up_proj", "model.layers.76.mlp.experts.104.up_proj", "model.layers.76.mlp.experts.105.up_proj", "model.layers.76.mlp.experts.106.up_proj", "model.layers.76.mlp.experts.107.up_proj", "model.layers.76.mlp.experts.108.up_proj", "model.layers.76.mlp.experts.109.up_proj", "model.layers.76.mlp.experts.110.up_proj", "model.layers.76.mlp.experts.111.up_proj", "model.layers.76.mlp.experts.112.up_proj", "model.layers.76.mlp.experts.113.up_proj", "model.layers.76.mlp.experts.114.up_proj", "model.layers.76.mlp.experts.115.up_proj", "model.layers.76.mlp.experts.116.up_proj", "model.layers.76.mlp.experts.117.up_proj", "model.layers.76.mlp.experts.118.up_proj", "model.layers.76.mlp.experts.119.up_proj", "model.layers.76.mlp.experts.120.up_proj", "model.layers.76.mlp.experts.121.up_proj", "model.layers.76.mlp.experts.122.up_proj", "model.layers.76.mlp.experts.123.up_proj", "model.layers.76.mlp.experts.124.up_proj", "model.layers.76.mlp.experts.125.up_proj", "model.layers.76.mlp.experts.126.up_proj", "model.layers.76.mlp.experts.127.up_proj", "model.layers.76.mlp.experts.128.up_proj", "model.layers.76.mlp.experts.129.up_proj", "model.layers.76.mlp.experts.130.up_proj", "model.layers.76.mlp.experts.131.up_proj", "model.layers.76.mlp.experts.132.up_proj", "model.layers.76.mlp.experts.133.up_proj", "model.layers.76.mlp.experts.134.up_proj", "model.layers.76.mlp.experts.135.up_proj", "model.layers.76.mlp.experts.136.up_proj", "model.layers.76.mlp.experts.137.up_proj", "model.layers.76.mlp.experts.138.up_proj", "model.layers.76.mlp.experts.139.up_proj", "model.layers.76.mlp.experts.140.up_proj", "model.layers.76.mlp.experts.141.up_proj", "model.layers.76.mlp.experts.142.up_proj", "model.layers.76.mlp.experts.143.up_proj", "model.layers.76.mlp.experts.144.up_proj", "model.layers.76.mlp.experts.145.up_proj", "model.layers.76.mlp.experts.146.up_proj", "model.layers.76.mlp.experts.147.up_proj", "model.layers.76.mlp.experts.148.up_proj", "model.layers.76.mlp.experts.149.up_proj", "model.layers.76.mlp.experts.150.up_proj", "model.layers.76.mlp.experts.151.up_proj", "model.layers.76.mlp.experts.152.up_proj", "model.layers.76.mlp.experts.153.up_proj", "model.layers.76.mlp.experts.154.up_proj", "model.layers.76.mlp.experts.155.up_proj", "model.layers.76.mlp.experts.156.up_proj", "model.layers.76.mlp.experts.157.up_proj", "model.layers.76.mlp.experts.158.up_proj", "model.layers.76.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -5.3183734416983874e-05, "dbits": 2516582400 } ] }, { "idx": 458, "layers": [ "model.layers.76.mlp.experts.0.down_proj", "model.layers.76.mlp.experts.1.down_proj", "model.layers.76.mlp.experts.2.down_proj", "model.layers.76.mlp.experts.3.down_proj", "model.layers.76.mlp.experts.4.down_proj", "model.layers.76.mlp.experts.5.down_proj", "model.layers.76.mlp.experts.6.down_proj", "model.layers.76.mlp.experts.7.down_proj", "model.layers.76.mlp.experts.8.down_proj", "model.layers.76.mlp.experts.9.down_proj", "model.layers.76.mlp.experts.10.down_proj", "model.layers.76.mlp.experts.11.down_proj", "model.layers.76.mlp.experts.12.down_proj", "model.layers.76.mlp.experts.13.down_proj", "model.layers.76.mlp.experts.14.down_proj", "model.layers.76.mlp.experts.15.down_proj", "model.layers.76.mlp.experts.16.down_proj", "model.layers.76.mlp.experts.17.down_proj", "model.layers.76.mlp.experts.18.down_proj", "model.layers.76.mlp.experts.19.down_proj", "model.layers.76.mlp.experts.20.down_proj", "model.layers.76.mlp.experts.21.down_proj", "model.layers.76.mlp.experts.22.down_proj", "model.layers.76.mlp.experts.23.down_proj", "model.layers.76.mlp.experts.24.down_proj", "model.layers.76.mlp.experts.25.down_proj", "model.layers.76.mlp.experts.26.down_proj", "model.layers.76.mlp.experts.27.down_proj", "model.layers.76.mlp.experts.28.down_proj", "model.layers.76.mlp.experts.29.down_proj", "model.layers.76.mlp.experts.30.down_proj", "model.layers.76.mlp.experts.31.down_proj", "model.layers.76.mlp.experts.32.down_proj", "model.layers.76.mlp.experts.33.down_proj", "model.layers.76.mlp.experts.34.down_proj", "model.layers.76.mlp.experts.35.down_proj", "model.layers.76.mlp.experts.36.down_proj", "model.layers.76.mlp.experts.37.down_proj", "model.layers.76.mlp.experts.38.down_proj", "model.layers.76.mlp.experts.39.down_proj", "model.layers.76.mlp.experts.40.down_proj", "model.layers.76.mlp.experts.41.down_proj", "model.layers.76.mlp.experts.42.down_proj", "model.layers.76.mlp.experts.43.down_proj", "model.layers.76.mlp.experts.44.down_proj", "model.layers.76.mlp.experts.45.down_proj", "model.layers.76.mlp.experts.46.down_proj", "model.layers.76.mlp.experts.47.down_proj", "model.layers.76.mlp.experts.48.down_proj", "model.layers.76.mlp.experts.49.down_proj", "model.layers.76.mlp.experts.50.down_proj", "model.layers.76.mlp.experts.51.down_proj", "model.layers.76.mlp.experts.52.down_proj", "model.layers.76.mlp.experts.53.down_proj", "model.layers.76.mlp.experts.54.down_proj", "model.layers.76.mlp.experts.55.down_proj", "model.layers.76.mlp.experts.56.down_proj", "model.layers.76.mlp.experts.57.down_proj", "model.layers.76.mlp.experts.58.down_proj", "model.layers.76.mlp.experts.59.down_proj", "model.layers.76.mlp.experts.60.down_proj", "model.layers.76.mlp.experts.61.down_proj", "model.layers.76.mlp.experts.62.down_proj", "model.layers.76.mlp.experts.63.down_proj", "model.layers.76.mlp.experts.64.down_proj", "model.layers.76.mlp.experts.65.down_proj", "model.layers.76.mlp.experts.66.down_proj", "model.layers.76.mlp.experts.67.down_proj", "model.layers.76.mlp.experts.68.down_proj", "model.layers.76.mlp.experts.69.down_proj", "model.layers.76.mlp.experts.70.down_proj", "model.layers.76.mlp.experts.71.down_proj", "model.layers.76.mlp.experts.72.down_proj", "model.layers.76.mlp.experts.73.down_proj", "model.layers.76.mlp.experts.74.down_proj", "model.layers.76.mlp.experts.75.down_proj", "model.layers.76.mlp.experts.76.down_proj", "model.layers.76.mlp.experts.77.down_proj", "model.layers.76.mlp.experts.78.down_proj", "model.layers.76.mlp.experts.79.down_proj", "model.layers.76.mlp.experts.80.down_proj", "model.layers.76.mlp.experts.81.down_proj", "model.layers.76.mlp.experts.82.down_proj", "model.layers.76.mlp.experts.83.down_proj", "model.layers.76.mlp.experts.84.down_proj", "model.layers.76.mlp.experts.85.down_proj", "model.layers.76.mlp.experts.86.down_proj", "model.layers.76.mlp.experts.87.down_proj", "model.layers.76.mlp.experts.88.down_proj", "model.layers.76.mlp.experts.89.down_proj", "model.layers.76.mlp.experts.90.down_proj", "model.layers.76.mlp.experts.91.down_proj", "model.layers.76.mlp.experts.92.down_proj", "model.layers.76.mlp.experts.93.down_proj", "model.layers.76.mlp.experts.94.down_proj", "model.layers.76.mlp.experts.95.down_proj", "model.layers.76.mlp.experts.96.down_proj", "model.layers.76.mlp.experts.97.down_proj", "model.layers.76.mlp.experts.98.down_proj", "model.layers.76.mlp.experts.99.down_proj", "model.layers.76.mlp.experts.100.down_proj", "model.layers.76.mlp.experts.101.down_proj", "model.layers.76.mlp.experts.102.down_proj", "model.layers.76.mlp.experts.103.down_proj", "model.layers.76.mlp.experts.104.down_proj", "model.layers.76.mlp.experts.105.down_proj", "model.layers.76.mlp.experts.106.down_proj", "model.layers.76.mlp.experts.107.down_proj", "model.layers.76.mlp.experts.108.down_proj", "model.layers.76.mlp.experts.109.down_proj", "model.layers.76.mlp.experts.110.down_proj", "model.layers.76.mlp.experts.111.down_proj", "model.layers.76.mlp.experts.112.down_proj", "model.layers.76.mlp.experts.113.down_proj", "model.layers.76.mlp.experts.114.down_proj", "model.layers.76.mlp.experts.115.down_proj", "model.layers.76.mlp.experts.116.down_proj", "model.layers.76.mlp.experts.117.down_proj", "model.layers.76.mlp.experts.118.down_proj", "model.layers.76.mlp.experts.119.down_proj", "model.layers.76.mlp.experts.120.down_proj", "model.layers.76.mlp.experts.121.down_proj", "model.layers.76.mlp.experts.122.down_proj", "model.layers.76.mlp.experts.123.down_proj", "model.layers.76.mlp.experts.124.down_proj", "model.layers.76.mlp.experts.125.down_proj", "model.layers.76.mlp.experts.126.down_proj", "model.layers.76.mlp.experts.127.down_proj", "model.layers.76.mlp.experts.128.down_proj", "model.layers.76.mlp.experts.129.down_proj", "model.layers.76.mlp.experts.130.down_proj", "model.layers.76.mlp.experts.131.down_proj", "model.layers.76.mlp.experts.132.down_proj", "model.layers.76.mlp.experts.133.down_proj", "model.layers.76.mlp.experts.134.down_proj", "model.layers.76.mlp.experts.135.down_proj", "model.layers.76.mlp.experts.136.down_proj", "model.layers.76.mlp.experts.137.down_proj", "model.layers.76.mlp.experts.138.down_proj", "model.layers.76.mlp.experts.139.down_proj", "model.layers.76.mlp.experts.140.down_proj", "model.layers.76.mlp.experts.141.down_proj", "model.layers.76.mlp.experts.142.down_proj", "model.layers.76.mlp.experts.143.down_proj", "model.layers.76.mlp.experts.144.down_proj", "model.layers.76.mlp.experts.145.down_proj", "model.layers.76.mlp.experts.146.down_proj", "model.layers.76.mlp.experts.147.down_proj", "model.layers.76.mlp.experts.148.down_proj", "model.layers.76.mlp.experts.149.down_proj", "model.layers.76.mlp.experts.150.down_proj", "model.layers.76.mlp.experts.151.down_proj", "model.layers.76.mlp.experts.152.down_proj", "model.layers.76.mlp.experts.153.down_proj", "model.layers.76.mlp.experts.154.down_proj", "model.layers.76.mlp.experts.155.down_proj", "model.layers.76.mlp.experts.156.down_proj", "model.layers.76.mlp.experts.157.down_proj", "model.layers.76.mlp.experts.158.down_proj", "model.layers.76.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00030030906200412266, "dbits": 1258291200 } ] }, { "idx": 459, "layers": [ "model.layers.77.self_attn.q_proj" ], "candidates": [ { "dkld": 4.235655069351196e-05, "dbits": 62914560 } ] }, { "idx": 460, "layers": [ "model.layers.77.self_attn.k_proj", "model.layers.77.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00023401007056234047, "dbits": 10485760 } ] }, { "idx": 461, "layers": [ "model.layers.77.self_attn.o_proj" ], "candidates": [ { "dkld": -0.001055291295051608, "dbits": 62914560 } ] }, { "idx": 462, "layers": [ "model.layers.77.mlp.shared_experts.gate_proj", "model.layers.77.mlp.shared_experts.up_proj", "model.layers.77.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0012217067182064278, "dbits": 23592960 } ] }, { "idx": 463, "layers": [ "model.layers.77.mlp.experts.0.gate_proj", "model.layers.77.mlp.experts.1.gate_proj", "model.layers.77.mlp.experts.2.gate_proj", "model.layers.77.mlp.experts.3.gate_proj", "model.layers.77.mlp.experts.4.gate_proj", "model.layers.77.mlp.experts.5.gate_proj", "model.layers.77.mlp.experts.6.gate_proj", "model.layers.77.mlp.experts.7.gate_proj", "model.layers.77.mlp.experts.8.gate_proj", "model.layers.77.mlp.experts.9.gate_proj", "model.layers.77.mlp.experts.10.gate_proj", "model.layers.77.mlp.experts.11.gate_proj", "model.layers.77.mlp.experts.12.gate_proj", "model.layers.77.mlp.experts.13.gate_proj", "model.layers.77.mlp.experts.14.gate_proj", "model.layers.77.mlp.experts.15.gate_proj", "model.layers.77.mlp.experts.16.gate_proj", "model.layers.77.mlp.experts.17.gate_proj", "model.layers.77.mlp.experts.18.gate_proj", "model.layers.77.mlp.experts.19.gate_proj", "model.layers.77.mlp.experts.20.gate_proj", "model.layers.77.mlp.experts.21.gate_proj", "model.layers.77.mlp.experts.22.gate_proj", "model.layers.77.mlp.experts.23.gate_proj", "model.layers.77.mlp.experts.24.gate_proj", "model.layers.77.mlp.experts.25.gate_proj", "model.layers.77.mlp.experts.26.gate_proj", "model.layers.77.mlp.experts.27.gate_proj", "model.layers.77.mlp.experts.28.gate_proj", "model.layers.77.mlp.experts.29.gate_proj", "model.layers.77.mlp.experts.30.gate_proj", "model.layers.77.mlp.experts.31.gate_proj", "model.layers.77.mlp.experts.32.gate_proj", "model.layers.77.mlp.experts.33.gate_proj", "model.layers.77.mlp.experts.34.gate_proj", "model.layers.77.mlp.experts.35.gate_proj", "model.layers.77.mlp.experts.36.gate_proj", "model.layers.77.mlp.experts.37.gate_proj", "model.layers.77.mlp.experts.38.gate_proj", "model.layers.77.mlp.experts.39.gate_proj", "model.layers.77.mlp.experts.40.gate_proj", "model.layers.77.mlp.experts.41.gate_proj", "model.layers.77.mlp.experts.42.gate_proj", "model.layers.77.mlp.experts.43.gate_proj", "model.layers.77.mlp.experts.44.gate_proj", "model.layers.77.mlp.experts.45.gate_proj", "model.layers.77.mlp.experts.46.gate_proj", "model.layers.77.mlp.experts.47.gate_proj", "model.layers.77.mlp.experts.48.gate_proj", "model.layers.77.mlp.experts.49.gate_proj", "model.layers.77.mlp.experts.50.gate_proj", "model.layers.77.mlp.experts.51.gate_proj", "model.layers.77.mlp.experts.52.gate_proj", "model.layers.77.mlp.experts.53.gate_proj", "model.layers.77.mlp.experts.54.gate_proj", "model.layers.77.mlp.experts.55.gate_proj", "model.layers.77.mlp.experts.56.gate_proj", "model.layers.77.mlp.experts.57.gate_proj", "model.layers.77.mlp.experts.58.gate_proj", "model.layers.77.mlp.experts.59.gate_proj", "model.layers.77.mlp.experts.60.gate_proj", "model.layers.77.mlp.experts.61.gate_proj", "model.layers.77.mlp.experts.62.gate_proj", "model.layers.77.mlp.experts.63.gate_proj", "model.layers.77.mlp.experts.64.gate_proj", "model.layers.77.mlp.experts.65.gate_proj", "model.layers.77.mlp.experts.66.gate_proj", "model.layers.77.mlp.experts.67.gate_proj", "model.layers.77.mlp.experts.68.gate_proj", "model.layers.77.mlp.experts.69.gate_proj", "model.layers.77.mlp.experts.70.gate_proj", "model.layers.77.mlp.experts.71.gate_proj", "model.layers.77.mlp.experts.72.gate_proj", "model.layers.77.mlp.experts.73.gate_proj", "model.layers.77.mlp.experts.74.gate_proj", "model.layers.77.mlp.experts.75.gate_proj", "model.layers.77.mlp.experts.76.gate_proj", "model.layers.77.mlp.experts.77.gate_proj", "model.layers.77.mlp.experts.78.gate_proj", "model.layers.77.mlp.experts.79.gate_proj", "model.layers.77.mlp.experts.80.gate_proj", "model.layers.77.mlp.experts.81.gate_proj", "model.layers.77.mlp.experts.82.gate_proj", "model.layers.77.mlp.experts.83.gate_proj", "model.layers.77.mlp.experts.84.gate_proj", "model.layers.77.mlp.experts.85.gate_proj", "model.layers.77.mlp.experts.86.gate_proj", "model.layers.77.mlp.experts.87.gate_proj", "model.layers.77.mlp.experts.88.gate_proj", "model.layers.77.mlp.experts.89.gate_proj", "model.layers.77.mlp.experts.90.gate_proj", "model.layers.77.mlp.experts.91.gate_proj", "model.layers.77.mlp.experts.92.gate_proj", "model.layers.77.mlp.experts.93.gate_proj", "model.layers.77.mlp.experts.94.gate_proj", "model.layers.77.mlp.experts.95.gate_proj", "model.layers.77.mlp.experts.96.gate_proj", "model.layers.77.mlp.experts.97.gate_proj", "model.layers.77.mlp.experts.98.gate_proj", "model.layers.77.mlp.experts.99.gate_proj", "model.layers.77.mlp.experts.100.gate_proj", "model.layers.77.mlp.experts.101.gate_proj", "model.layers.77.mlp.experts.102.gate_proj", "model.layers.77.mlp.experts.103.gate_proj", "model.layers.77.mlp.experts.104.gate_proj", "model.layers.77.mlp.experts.105.gate_proj", "model.layers.77.mlp.experts.106.gate_proj", "model.layers.77.mlp.experts.107.gate_proj", "model.layers.77.mlp.experts.108.gate_proj", "model.layers.77.mlp.experts.109.gate_proj", "model.layers.77.mlp.experts.110.gate_proj", "model.layers.77.mlp.experts.111.gate_proj", "model.layers.77.mlp.experts.112.gate_proj", "model.layers.77.mlp.experts.113.gate_proj", "model.layers.77.mlp.experts.114.gate_proj", "model.layers.77.mlp.experts.115.gate_proj", "model.layers.77.mlp.experts.116.gate_proj", "model.layers.77.mlp.experts.117.gate_proj", "model.layers.77.mlp.experts.118.gate_proj", "model.layers.77.mlp.experts.119.gate_proj", "model.layers.77.mlp.experts.120.gate_proj", "model.layers.77.mlp.experts.121.gate_proj", "model.layers.77.mlp.experts.122.gate_proj", "model.layers.77.mlp.experts.123.gate_proj", "model.layers.77.mlp.experts.124.gate_proj", "model.layers.77.mlp.experts.125.gate_proj", "model.layers.77.mlp.experts.126.gate_proj", "model.layers.77.mlp.experts.127.gate_proj", "model.layers.77.mlp.experts.128.gate_proj", "model.layers.77.mlp.experts.129.gate_proj", "model.layers.77.mlp.experts.130.gate_proj", "model.layers.77.mlp.experts.131.gate_proj", "model.layers.77.mlp.experts.132.gate_proj", "model.layers.77.mlp.experts.133.gate_proj", "model.layers.77.mlp.experts.134.gate_proj", "model.layers.77.mlp.experts.135.gate_proj", "model.layers.77.mlp.experts.136.gate_proj", "model.layers.77.mlp.experts.137.gate_proj", "model.layers.77.mlp.experts.138.gate_proj", "model.layers.77.mlp.experts.139.gate_proj", "model.layers.77.mlp.experts.140.gate_proj", "model.layers.77.mlp.experts.141.gate_proj", "model.layers.77.mlp.experts.142.gate_proj", "model.layers.77.mlp.experts.143.gate_proj", "model.layers.77.mlp.experts.144.gate_proj", "model.layers.77.mlp.experts.145.gate_proj", "model.layers.77.mlp.experts.146.gate_proj", "model.layers.77.mlp.experts.147.gate_proj", "model.layers.77.mlp.experts.148.gate_proj", "model.layers.77.mlp.experts.149.gate_proj", "model.layers.77.mlp.experts.150.gate_proj", "model.layers.77.mlp.experts.151.gate_proj", "model.layers.77.mlp.experts.152.gate_proj", "model.layers.77.mlp.experts.153.gate_proj", "model.layers.77.mlp.experts.154.gate_proj", "model.layers.77.mlp.experts.155.gate_proj", "model.layers.77.mlp.experts.156.gate_proj", "model.layers.77.mlp.experts.157.gate_proj", "model.layers.77.mlp.experts.158.gate_proj", "model.layers.77.mlp.experts.159.gate_proj", "model.layers.77.mlp.experts.0.up_proj", "model.layers.77.mlp.experts.1.up_proj", "model.layers.77.mlp.experts.2.up_proj", "model.layers.77.mlp.experts.3.up_proj", "model.layers.77.mlp.experts.4.up_proj", "model.layers.77.mlp.experts.5.up_proj", "model.layers.77.mlp.experts.6.up_proj", "model.layers.77.mlp.experts.7.up_proj", "model.layers.77.mlp.experts.8.up_proj", "model.layers.77.mlp.experts.9.up_proj", "model.layers.77.mlp.experts.10.up_proj", "model.layers.77.mlp.experts.11.up_proj", "model.layers.77.mlp.experts.12.up_proj", "model.layers.77.mlp.experts.13.up_proj", "model.layers.77.mlp.experts.14.up_proj", "model.layers.77.mlp.experts.15.up_proj", "model.layers.77.mlp.experts.16.up_proj", "model.layers.77.mlp.experts.17.up_proj", "model.layers.77.mlp.experts.18.up_proj", "model.layers.77.mlp.experts.19.up_proj", "model.layers.77.mlp.experts.20.up_proj", "model.layers.77.mlp.experts.21.up_proj", "model.layers.77.mlp.experts.22.up_proj", "model.layers.77.mlp.experts.23.up_proj", "model.layers.77.mlp.experts.24.up_proj", "model.layers.77.mlp.experts.25.up_proj", "model.layers.77.mlp.experts.26.up_proj", "model.layers.77.mlp.experts.27.up_proj", "model.layers.77.mlp.experts.28.up_proj", "model.layers.77.mlp.experts.29.up_proj", "model.layers.77.mlp.experts.30.up_proj", "model.layers.77.mlp.experts.31.up_proj", "model.layers.77.mlp.experts.32.up_proj", "model.layers.77.mlp.experts.33.up_proj", "model.layers.77.mlp.experts.34.up_proj", "model.layers.77.mlp.experts.35.up_proj", "model.layers.77.mlp.experts.36.up_proj", "model.layers.77.mlp.experts.37.up_proj", "model.layers.77.mlp.experts.38.up_proj", "model.layers.77.mlp.experts.39.up_proj", "model.layers.77.mlp.experts.40.up_proj", "model.layers.77.mlp.experts.41.up_proj", "model.layers.77.mlp.experts.42.up_proj", "model.layers.77.mlp.experts.43.up_proj", "model.layers.77.mlp.experts.44.up_proj", "model.layers.77.mlp.experts.45.up_proj", "model.layers.77.mlp.experts.46.up_proj", "model.layers.77.mlp.experts.47.up_proj", "model.layers.77.mlp.experts.48.up_proj", "model.layers.77.mlp.experts.49.up_proj", "model.layers.77.mlp.experts.50.up_proj", "model.layers.77.mlp.experts.51.up_proj", "model.layers.77.mlp.experts.52.up_proj", "model.layers.77.mlp.experts.53.up_proj", "model.layers.77.mlp.experts.54.up_proj", "model.layers.77.mlp.experts.55.up_proj", "model.layers.77.mlp.experts.56.up_proj", "model.layers.77.mlp.experts.57.up_proj", "model.layers.77.mlp.experts.58.up_proj", "model.layers.77.mlp.experts.59.up_proj", "model.layers.77.mlp.experts.60.up_proj", "model.layers.77.mlp.experts.61.up_proj", "model.layers.77.mlp.experts.62.up_proj", "model.layers.77.mlp.experts.63.up_proj", "model.layers.77.mlp.experts.64.up_proj", "model.layers.77.mlp.experts.65.up_proj", "model.layers.77.mlp.experts.66.up_proj", "model.layers.77.mlp.experts.67.up_proj", "model.layers.77.mlp.experts.68.up_proj", "model.layers.77.mlp.experts.69.up_proj", "model.layers.77.mlp.experts.70.up_proj", "model.layers.77.mlp.experts.71.up_proj", "model.layers.77.mlp.experts.72.up_proj", "model.layers.77.mlp.experts.73.up_proj", "model.layers.77.mlp.experts.74.up_proj", "model.layers.77.mlp.experts.75.up_proj", "model.layers.77.mlp.experts.76.up_proj", "model.layers.77.mlp.experts.77.up_proj", "model.layers.77.mlp.experts.78.up_proj", "model.layers.77.mlp.experts.79.up_proj", "model.layers.77.mlp.experts.80.up_proj", "model.layers.77.mlp.experts.81.up_proj", "model.layers.77.mlp.experts.82.up_proj", "model.layers.77.mlp.experts.83.up_proj", "model.layers.77.mlp.experts.84.up_proj", "model.layers.77.mlp.experts.85.up_proj", "model.layers.77.mlp.experts.86.up_proj", "model.layers.77.mlp.experts.87.up_proj", "model.layers.77.mlp.experts.88.up_proj", "model.layers.77.mlp.experts.89.up_proj", "model.layers.77.mlp.experts.90.up_proj", "model.layers.77.mlp.experts.91.up_proj", "model.layers.77.mlp.experts.92.up_proj", "model.layers.77.mlp.experts.93.up_proj", "model.layers.77.mlp.experts.94.up_proj", "model.layers.77.mlp.experts.95.up_proj", "model.layers.77.mlp.experts.96.up_proj", "model.layers.77.mlp.experts.97.up_proj", "model.layers.77.mlp.experts.98.up_proj", "model.layers.77.mlp.experts.99.up_proj", "model.layers.77.mlp.experts.100.up_proj", "model.layers.77.mlp.experts.101.up_proj", "model.layers.77.mlp.experts.102.up_proj", "model.layers.77.mlp.experts.103.up_proj", "model.layers.77.mlp.experts.104.up_proj", "model.layers.77.mlp.experts.105.up_proj", "model.layers.77.mlp.experts.106.up_proj", "model.layers.77.mlp.experts.107.up_proj", "model.layers.77.mlp.experts.108.up_proj", "model.layers.77.mlp.experts.109.up_proj", "model.layers.77.mlp.experts.110.up_proj", "model.layers.77.mlp.experts.111.up_proj", "model.layers.77.mlp.experts.112.up_proj", "model.layers.77.mlp.experts.113.up_proj", "model.layers.77.mlp.experts.114.up_proj", "model.layers.77.mlp.experts.115.up_proj", "model.layers.77.mlp.experts.116.up_proj", "model.layers.77.mlp.experts.117.up_proj", "model.layers.77.mlp.experts.118.up_proj", "model.layers.77.mlp.experts.119.up_proj", "model.layers.77.mlp.experts.120.up_proj", "model.layers.77.mlp.experts.121.up_proj", "model.layers.77.mlp.experts.122.up_proj", "model.layers.77.mlp.experts.123.up_proj", "model.layers.77.mlp.experts.124.up_proj", "model.layers.77.mlp.experts.125.up_proj", "model.layers.77.mlp.experts.126.up_proj", "model.layers.77.mlp.experts.127.up_proj", "model.layers.77.mlp.experts.128.up_proj", "model.layers.77.mlp.experts.129.up_proj", "model.layers.77.mlp.experts.130.up_proj", "model.layers.77.mlp.experts.131.up_proj", "model.layers.77.mlp.experts.132.up_proj", "model.layers.77.mlp.experts.133.up_proj", "model.layers.77.mlp.experts.134.up_proj", "model.layers.77.mlp.experts.135.up_proj", "model.layers.77.mlp.experts.136.up_proj", "model.layers.77.mlp.experts.137.up_proj", "model.layers.77.mlp.experts.138.up_proj", "model.layers.77.mlp.experts.139.up_proj", "model.layers.77.mlp.experts.140.up_proj", "model.layers.77.mlp.experts.141.up_proj", "model.layers.77.mlp.experts.142.up_proj", "model.layers.77.mlp.experts.143.up_proj", "model.layers.77.mlp.experts.144.up_proj", "model.layers.77.mlp.experts.145.up_proj", "model.layers.77.mlp.experts.146.up_proj", "model.layers.77.mlp.experts.147.up_proj", "model.layers.77.mlp.experts.148.up_proj", "model.layers.77.mlp.experts.149.up_proj", "model.layers.77.mlp.experts.150.up_proj", "model.layers.77.mlp.experts.151.up_proj", "model.layers.77.mlp.experts.152.up_proj", "model.layers.77.mlp.experts.153.up_proj", "model.layers.77.mlp.experts.154.up_proj", "model.layers.77.mlp.experts.155.up_proj", "model.layers.77.mlp.experts.156.up_proj", "model.layers.77.mlp.experts.157.up_proj", "model.layers.77.mlp.experts.158.up_proj", "model.layers.77.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0004511997103691101, "dbits": 2516582400 } ] }, { "idx": 464, "layers": [ "model.layers.77.mlp.experts.0.down_proj", "model.layers.77.mlp.experts.1.down_proj", "model.layers.77.mlp.experts.2.down_proj", "model.layers.77.mlp.experts.3.down_proj", "model.layers.77.mlp.experts.4.down_proj", "model.layers.77.mlp.experts.5.down_proj", "model.layers.77.mlp.experts.6.down_proj", "model.layers.77.mlp.experts.7.down_proj", "model.layers.77.mlp.experts.8.down_proj", "model.layers.77.mlp.experts.9.down_proj", "model.layers.77.mlp.experts.10.down_proj", "model.layers.77.mlp.experts.11.down_proj", "model.layers.77.mlp.experts.12.down_proj", "model.layers.77.mlp.experts.13.down_proj", "model.layers.77.mlp.experts.14.down_proj", "model.layers.77.mlp.experts.15.down_proj", "model.layers.77.mlp.experts.16.down_proj", "model.layers.77.mlp.experts.17.down_proj", "model.layers.77.mlp.experts.18.down_proj", "model.layers.77.mlp.experts.19.down_proj", "model.layers.77.mlp.experts.20.down_proj", "model.layers.77.mlp.experts.21.down_proj", "model.layers.77.mlp.experts.22.down_proj", "model.layers.77.mlp.experts.23.down_proj", "model.layers.77.mlp.experts.24.down_proj", "model.layers.77.mlp.experts.25.down_proj", "model.layers.77.mlp.experts.26.down_proj", "model.layers.77.mlp.experts.27.down_proj", "model.layers.77.mlp.experts.28.down_proj", "model.layers.77.mlp.experts.29.down_proj", "model.layers.77.mlp.experts.30.down_proj", "model.layers.77.mlp.experts.31.down_proj", "model.layers.77.mlp.experts.32.down_proj", "model.layers.77.mlp.experts.33.down_proj", "model.layers.77.mlp.experts.34.down_proj", "model.layers.77.mlp.experts.35.down_proj", "model.layers.77.mlp.experts.36.down_proj", "model.layers.77.mlp.experts.37.down_proj", "model.layers.77.mlp.experts.38.down_proj", "model.layers.77.mlp.experts.39.down_proj", "model.layers.77.mlp.experts.40.down_proj", "model.layers.77.mlp.experts.41.down_proj", "model.layers.77.mlp.experts.42.down_proj", "model.layers.77.mlp.experts.43.down_proj", "model.layers.77.mlp.experts.44.down_proj", "model.layers.77.mlp.experts.45.down_proj", "model.layers.77.mlp.experts.46.down_proj", "model.layers.77.mlp.experts.47.down_proj", "model.layers.77.mlp.experts.48.down_proj", "model.layers.77.mlp.experts.49.down_proj", "model.layers.77.mlp.experts.50.down_proj", "model.layers.77.mlp.experts.51.down_proj", "model.layers.77.mlp.experts.52.down_proj", "model.layers.77.mlp.experts.53.down_proj", "model.layers.77.mlp.experts.54.down_proj", "model.layers.77.mlp.experts.55.down_proj", "model.layers.77.mlp.experts.56.down_proj", "model.layers.77.mlp.experts.57.down_proj", "model.layers.77.mlp.experts.58.down_proj", "model.layers.77.mlp.experts.59.down_proj", "model.layers.77.mlp.experts.60.down_proj", "model.layers.77.mlp.experts.61.down_proj", "model.layers.77.mlp.experts.62.down_proj", "model.layers.77.mlp.experts.63.down_proj", "model.layers.77.mlp.experts.64.down_proj", "model.layers.77.mlp.experts.65.down_proj", "model.layers.77.mlp.experts.66.down_proj", "model.layers.77.mlp.experts.67.down_proj", "model.layers.77.mlp.experts.68.down_proj", "model.layers.77.mlp.experts.69.down_proj", "model.layers.77.mlp.experts.70.down_proj", "model.layers.77.mlp.experts.71.down_proj", "model.layers.77.mlp.experts.72.down_proj", "model.layers.77.mlp.experts.73.down_proj", "model.layers.77.mlp.experts.74.down_proj", "model.layers.77.mlp.experts.75.down_proj", "model.layers.77.mlp.experts.76.down_proj", "model.layers.77.mlp.experts.77.down_proj", "model.layers.77.mlp.experts.78.down_proj", "model.layers.77.mlp.experts.79.down_proj", "model.layers.77.mlp.experts.80.down_proj", "model.layers.77.mlp.experts.81.down_proj", "model.layers.77.mlp.experts.82.down_proj", "model.layers.77.mlp.experts.83.down_proj", "model.layers.77.mlp.experts.84.down_proj", "model.layers.77.mlp.experts.85.down_proj", "model.layers.77.mlp.experts.86.down_proj", "model.layers.77.mlp.experts.87.down_proj", "model.layers.77.mlp.experts.88.down_proj", "model.layers.77.mlp.experts.89.down_proj", "model.layers.77.mlp.experts.90.down_proj", "model.layers.77.mlp.experts.91.down_proj", "model.layers.77.mlp.experts.92.down_proj", "model.layers.77.mlp.experts.93.down_proj", "model.layers.77.mlp.experts.94.down_proj", "model.layers.77.mlp.experts.95.down_proj", "model.layers.77.mlp.experts.96.down_proj", "model.layers.77.mlp.experts.97.down_proj", "model.layers.77.mlp.experts.98.down_proj", "model.layers.77.mlp.experts.99.down_proj", "model.layers.77.mlp.experts.100.down_proj", "model.layers.77.mlp.experts.101.down_proj", "model.layers.77.mlp.experts.102.down_proj", "model.layers.77.mlp.experts.103.down_proj", "model.layers.77.mlp.experts.104.down_proj", "model.layers.77.mlp.experts.105.down_proj", "model.layers.77.mlp.experts.106.down_proj", "model.layers.77.mlp.experts.107.down_proj", "model.layers.77.mlp.experts.108.down_proj", "model.layers.77.mlp.experts.109.down_proj", "model.layers.77.mlp.experts.110.down_proj", "model.layers.77.mlp.experts.111.down_proj", "model.layers.77.mlp.experts.112.down_proj", "model.layers.77.mlp.experts.113.down_proj", "model.layers.77.mlp.experts.114.down_proj", "model.layers.77.mlp.experts.115.down_proj", "model.layers.77.mlp.experts.116.down_proj", "model.layers.77.mlp.experts.117.down_proj", "model.layers.77.mlp.experts.118.down_proj", "model.layers.77.mlp.experts.119.down_proj", "model.layers.77.mlp.experts.120.down_proj", "model.layers.77.mlp.experts.121.down_proj", "model.layers.77.mlp.experts.122.down_proj", "model.layers.77.mlp.experts.123.down_proj", "model.layers.77.mlp.experts.124.down_proj", "model.layers.77.mlp.experts.125.down_proj", "model.layers.77.mlp.experts.126.down_proj", "model.layers.77.mlp.experts.127.down_proj", "model.layers.77.mlp.experts.128.down_proj", "model.layers.77.mlp.experts.129.down_proj", "model.layers.77.mlp.experts.130.down_proj", "model.layers.77.mlp.experts.131.down_proj", "model.layers.77.mlp.experts.132.down_proj", "model.layers.77.mlp.experts.133.down_proj", "model.layers.77.mlp.experts.134.down_proj", "model.layers.77.mlp.experts.135.down_proj", "model.layers.77.mlp.experts.136.down_proj", "model.layers.77.mlp.experts.137.down_proj", "model.layers.77.mlp.experts.138.down_proj", "model.layers.77.mlp.experts.139.down_proj", "model.layers.77.mlp.experts.140.down_proj", "model.layers.77.mlp.experts.141.down_proj", "model.layers.77.mlp.experts.142.down_proj", "model.layers.77.mlp.experts.143.down_proj", "model.layers.77.mlp.experts.144.down_proj", "model.layers.77.mlp.experts.145.down_proj", "model.layers.77.mlp.experts.146.down_proj", "model.layers.77.mlp.experts.147.down_proj", "model.layers.77.mlp.experts.148.down_proj", "model.layers.77.mlp.experts.149.down_proj", "model.layers.77.mlp.experts.150.down_proj", "model.layers.77.mlp.experts.151.down_proj", "model.layers.77.mlp.experts.152.down_proj", "model.layers.77.mlp.experts.153.down_proj", "model.layers.77.mlp.experts.154.down_proj", "model.layers.77.mlp.experts.155.down_proj", "model.layers.77.mlp.experts.156.down_proj", "model.layers.77.mlp.experts.157.down_proj", "model.layers.77.mlp.experts.158.down_proj", "model.layers.77.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00013960525393486023, "dbits": 1258291200 } ] }, { "idx": 465, "layers": [ "model.layers.78.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0015677012503146903, "dbits": 62914560 } ] }, { "idx": 466, "layers": [ "model.layers.78.self_attn.k_proj", "model.layers.78.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0012599535286426655, "dbits": 10485760 } ] }, { "idx": 467, "layers": [ "model.layers.78.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0004789017140865437, "dbits": 62914560 } ] }, { "idx": 468, "layers": [ "model.layers.78.mlp.shared_experts.gate_proj", "model.layers.78.mlp.shared_experts.up_proj", "model.layers.78.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0015905342996120342, "dbits": 23592960 } ] }, { "idx": 469, "layers": [ "model.layers.78.mlp.experts.0.gate_proj", "model.layers.78.mlp.experts.1.gate_proj", "model.layers.78.mlp.experts.2.gate_proj", "model.layers.78.mlp.experts.3.gate_proj", "model.layers.78.mlp.experts.4.gate_proj", "model.layers.78.mlp.experts.5.gate_proj", "model.layers.78.mlp.experts.6.gate_proj", "model.layers.78.mlp.experts.7.gate_proj", "model.layers.78.mlp.experts.8.gate_proj", "model.layers.78.mlp.experts.9.gate_proj", "model.layers.78.mlp.experts.10.gate_proj", "model.layers.78.mlp.experts.11.gate_proj", "model.layers.78.mlp.experts.12.gate_proj", "model.layers.78.mlp.experts.13.gate_proj", "model.layers.78.mlp.experts.14.gate_proj", "model.layers.78.mlp.experts.15.gate_proj", "model.layers.78.mlp.experts.16.gate_proj", "model.layers.78.mlp.experts.17.gate_proj", "model.layers.78.mlp.experts.18.gate_proj", "model.layers.78.mlp.experts.19.gate_proj", "model.layers.78.mlp.experts.20.gate_proj", "model.layers.78.mlp.experts.21.gate_proj", "model.layers.78.mlp.experts.22.gate_proj", "model.layers.78.mlp.experts.23.gate_proj", "model.layers.78.mlp.experts.24.gate_proj", "model.layers.78.mlp.experts.25.gate_proj", "model.layers.78.mlp.experts.26.gate_proj", "model.layers.78.mlp.experts.27.gate_proj", "model.layers.78.mlp.experts.28.gate_proj", "model.layers.78.mlp.experts.29.gate_proj", "model.layers.78.mlp.experts.30.gate_proj", "model.layers.78.mlp.experts.31.gate_proj", "model.layers.78.mlp.experts.32.gate_proj", "model.layers.78.mlp.experts.33.gate_proj", "model.layers.78.mlp.experts.34.gate_proj", "model.layers.78.mlp.experts.35.gate_proj", "model.layers.78.mlp.experts.36.gate_proj", "model.layers.78.mlp.experts.37.gate_proj", "model.layers.78.mlp.experts.38.gate_proj", "model.layers.78.mlp.experts.39.gate_proj", "model.layers.78.mlp.experts.40.gate_proj", "model.layers.78.mlp.experts.41.gate_proj", "model.layers.78.mlp.experts.42.gate_proj", "model.layers.78.mlp.experts.43.gate_proj", "model.layers.78.mlp.experts.44.gate_proj", "model.layers.78.mlp.experts.45.gate_proj", "model.layers.78.mlp.experts.46.gate_proj", "model.layers.78.mlp.experts.47.gate_proj", "model.layers.78.mlp.experts.48.gate_proj", "model.layers.78.mlp.experts.49.gate_proj", "model.layers.78.mlp.experts.50.gate_proj", "model.layers.78.mlp.experts.51.gate_proj", "model.layers.78.mlp.experts.52.gate_proj", "model.layers.78.mlp.experts.53.gate_proj", "model.layers.78.mlp.experts.54.gate_proj", "model.layers.78.mlp.experts.55.gate_proj", "model.layers.78.mlp.experts.56.gate_proj", "model.layers.78.mlp.experts.57.gate_proj", "model.layers.78.mlp.experts.58.gate_proj", "model.layers.78.mlp.experts.59.gate_proj", "model.layers.78.mlp.experts.60.gate_proj", "model.layers.78.mlp.experts.61.gate_proj", "model.layers.78.mlp.experts.62.gate_proj", "model.layers.78.mlp.experts.63.gate_proj", "model.layers.78.mlp.experts.64.gate_proj", "model.layers.78.mlp.experts.65.gate_proj", "model.layers.78.mlp.experts.66.gate_proj", "model.layers.78.mlp.experts.67.gate_proj", "model.layers.78.mlp.experts.68.gate_proj", "model.layers.78.mlp.experts.69.gate_proj", "model.layers.78.mlp.experts.70.gate_proj", "model.layers.78.mlp.experts.71.gate_proj", "model.layers.78.mlp.experts.72.gate_proj", "model.layers.78.mlp.experts.73.gate_proj", "model.layers.78.mlp.experts.74.gate_proj", "model.layers.78.mlp.experts.75.gate_proj", "model.layers.78.mlp.experts.76.gate_proj", "model.layers.78.mlp.experts.77.gate_proj", "model.layers.78.mlp.experts.78.gate_proj", "model.layers.78.mlp.experts.79.gate_proj", "model.layers.78.mlp.experts.80.gate_proj", "model.layers.78.mlp.experts.81.gate_proj", "model.layers.78.mlp.experts.82.gate_proj", "model.layers.78.mlp.experts.83.gate_proj", "model.layers.78.mlp.experts.84.gate_proj", "model.layers.78.mlp.experts.85.gate_proj", "model.layers.78.mlp.experts.86.gate_proj", "model.layers.78.mlp.experts.87.gate_proj", "model.layers.78.mlp.experts.88.gate_proj", "model.layers.78.mlp.experts.89.gate_proj", "model.layers.78.mlp.experts.90.gate_proj", "model.layers.78.mlp.experts.91.gate_proj", "model.layers.78.mlp.experts.92.gate_proj", "model.layers.78.mlp.experts.93.gate_proj", "model.layers.78.mlp.experts.94.gate_proj", "model.layers.78.mlp.experts.95.gate_proj", "model.layers.78.mlp.experts.96.gate_proj", "model.layers.78.mlp.experts.97.gate_proj", "model.layers.78.mlp.experts.98.gate_proj", "model.layers.78.mlp.experts.99.gate_proj", "model.layers.78.mlp.experts.100.gate_proj", "model.layers.78.mlp.experts.101.gate_proj", "model.layers.78.mlp.experts.102.gate_proj", "model.layers.78.mlp.experts.103.gate_proj", "model.layers.78.mlp.experts.104.gate_proj", "model.layers.78.mlp.experts.105.gate_proj", "model.layers.78.mlp.experts.106.gate_proj", "model.layers.78.mlp.experts.107.gate_proj", "model.layers.78.mlp.experts.108.gate_proj", "model.layers.78.mlp.experts.109.gate_proj", "model.layers.78.mlp.experts.110.gate_proj", "model.layers.78.mlp.experts.111.gate_proj", "model.layers.78.mlp.experts.112.gate_proj", "model.layers.78.mlp.experts.113.gate_proj", "model.layers.78.mlp.experts.114.gate_proj", "model.layers.78.mlp.experts.115.gate_proj", "model.layers.78.mlp.experts.116.gate_proj", "model.layers.78.mlp.experts.117.gate_proj", "model.layers.78.mlp.experts.118.gate_proj", "model.layers.78.mlp.experts.119.gate_proj", "model.layers.78.mlp.experts.120.gate_proj", "model.layers.78.mlp.experts.121.gate_proj", "model.layers.78.mlp.experts.122.gate_proj", "model.layers.78.mlp.experts.123.gate_proj", "model.layers.78.mlp.experts.124.gate_proj", "model.layers.78.mlp.experts.125.gate_proj", "model.layers.78.mlp.experts.126.gate_proj", "model.layers.78.mlp.experts.127.gate_proj", "model.layers.78.mlp.experts.128.gate_proj", "model.layers.78.mlp.experts.129.gate_proj", "model.layers.78.mlp.experts.130.gate_proj", "model.layers.78.mlp.experts.131.gate_proj", "model.layers.78.mlp.experts.132.gate_proj", "model.layers.78.mlp.experts.133.gate_proj", "model.layers.78.mlp.experts.134.gate_proj", "model.layers.78.mlp.experts.135.gate_proj", "model.layers.78.mlp.experts.136.gate_proj", "model.layers.78.mlp.experts.137.gate_proj", "model.layers.78.mlp.experts.138.gate_proj", "model.layers.78.mlp.experts.139.gate_proj", "model.layers.78.mlp.experts.140.gate_proj", "model.layers.78.mlp.experts.141.gate_proj", "model.layers.78.mlp.experts.142.gate_proj", "model.layers.78.mlp.experts.143.gate_proj", "model.layers.78.mlp.experts.144.gate_proj", "model.layers.78.mlp.experts.145.gate_proj", "model.layers.78.mlp.experts.146.gate_proj", "model.layers.78.mlp.experts.147.gate_proj", "model.layers.78.mlp.experts.148.gate_proj", "model.layers.78.mlp.experts.149.gate_proj", "model.layers.78.mlp.experts.150.gate_proj", "model.layers.78.mlp.experts.151.gate_proj", "model.layers.78.mlp.experts.152.gate_proj", "model.layers.78.mlp.experts.153.gate_proj", "model.layers.78.mlp.experts.154.gate_proj", "model.layers.78.mlp.experts.155.gate_proj", "model.layers.78.mlp.experts.156.gate_proj", "model.layers.78.mlp.experts.157.gate_proj", "model.layers.78.mlp.experts.158.gate_proj", "model.layers.78.mlp.experts.159.gate_proj", "model.layers.78.mlp.experts.0.up_proj", "model.layers.78.mlp.experts.1.up_proj", "model.layers.78.mlp.experts.2.up_proj", "model.layers.78.mlp.experts.3.up_proj", "model.layers.78.mlp.experts.4.up_proj", "model.layers.78.mlp.experts.5.up_proj", "model.layers.78.mlp.experts.6.up_proj", "model.layers.78.mlp.experts.7.up_proj", "model.layers.78.mlp.experts.8.up_proj", "model.layers.78.mlp.experts.9.up_proj", "model.layers.78.mlp.experts.10.up_proj", "model.layers.78.mlp.experts.11.up_proj", "model.layers.78.mlp.experts.12.up_proj", "model.layers.78.mlp.experts.13.up_proj", "model.layers.78.mlp.experts.14.up_proj", "model.layers.78.mlp.experts.15.up_proj", "model.layers.78.mlp.experts.16.up_proj", "model.layers.78.mlp.experts.17.up_proj", "model.layers.78.mlp.experts.18.up_proj", "model.layers.78.mlp.experts.19.up_proj", "model.layers.78.mlp.experts.20.up_proj", "model.layers.78.mlp.experts.21.up_proj", "model.layers.78.mlp.experts.22.up_proj", "model.layers.78.mlp.experts.23.up_proj", "model.layers.78.mlp.experts.24.up_proj", "model.layers.78.mlp.experts.25.up_proj", "model.layers.78.mlp.experts.26.up_proj", "model.layers.78.mlp.experts.27.up_proj", "model.layers.78.mlp.experts.28.up_proj", "model.layers.78.mlp.experts.29.up_proj", "model.layers.78.mlp.experts.30.up_proj", "model.layers.78.mlp.experts.31.up_proj", "model.layers.78.mlp.experts.32.up_proj", "model.layers.78.mlp.experts.33.up_proj", "model.layers.78.mlp.experts.34.up_proj", "model.layers.78.mlp.experts.35.up_proj", "model.layers.78.mlp.experts.36.up_proj", "model.layers.78.mlp.experts.37.up_proj", "model.layers.78.mlp.experts.38.up_proj", "model.layers.78.mlp.experts.39.up_proj", "model.layers.78.mlp.experts.40.up_proj", "model.layers.78.mlp.experts.41.up_proj", "model.layers.78.mlp.experts.42.up_proj", "model.layers.78.mlp.experts.43.up_proj", "model.layers.78.mlp.experts.44.up_proj", "model.layers.78.mlp.experts.45.up_proj", "model.layers.78.mlp.experts.46.up_proj", "model.layers.78.mlp.experts.47.up_proj", "model.layers.78.mlp.experts.48.up_proj", "model.layers.78.mlp.experts.49.up_proj", "model.layers.78.mlp.experts.50.up_proj", "model.layers.78.mlp.experts.51.up_proj", "model.layers.78.mlp.experts.52.up_proj", "model.layers.78.mlp.experts.53.up_proj", "model.layers.78.mlp.experts.54.up_proj", "model.layers.78.mlp.experts.55.up_proj", "model.layers.78.mlp.experts.56.up_proj", "model.layers.78.mlp.experts.57.up_proj", "model.layers.78.mlp.experts.58.up_proj", "model.layers.78.mlp.experts.59.up_proj", "model.layers.78.mlp.experts.60.up_proj", "model.layers.78.mlp.experts.61.up_proj", "model.layers.78.mlp.experts.62.up_proj", "model.layers.78.mlp.experts.63.up_proj", "model.layers.78.mlp.experts.64.up_proj", "model.layers.78.mlp.experts.65.up_proj", "model.layers.78.mlp.experts.66.up_proj", "model.layers.78.mlp.experts.67.up_proj", "model.layers.78.mlp.experts.68.up_proj", "model.layers.78.mlp.experts.69.up_proj", "model.layers.78.mlp.experts.70.up_proj", "model.layers.78.mlp.experts.71.up_proj", "model.layers.78.mlp.experts.72.up_proj", "model.layers.78.mlp.experts.73.up_proj", "model.layers.78.mlp.experts.74.up_proj", "model.layers.78.mlp.experts.75.up_proj", "model.layers.78.mlp.experts.76.up_proj", "model.layers.78.mlp.experts.77.up_proj", "model.layers.78.mlp.experts.78.up_proj", "model.layers.78.mlp.experts.79.up_proj", "model.layers.78.mlp.experts.80.up_proj", "model.layers.78.mlp.experts.81.up_proj", "model.layers.78.mlp.experts.82.up_proj", "model.layers.78.mlp.experts.83.up_proj", "model.layers.78.mlp.experts.84.up_proj", "model.layers.78.mlp.experts.85.up_proj", "model.layers.78.mlp.experts.86.up_proj", "model.layers.78.mlp.experts.87.up_proj", "model.layers.78.mlp.experts.88.up_proj", "model.layers.78.mlp.experts.89.up_proj", "model.layers.78.mlp.experts.90.up_proj", "model.layers.78.mlp.experts.91.up_proj", "model.layers.78.mlp.experts.92.up_proj", "model.layers.78.mlp.experts.93.up_proj", "model.layers.78.mlp.experts.94.up_proj", "model.layers.78.mlp.experts.95.up_proj", "model.layers.78.mlp.experts.96.up_proj", "model.layers.78.mlp.experts.97.up_proj", "model.layers.78.mlp.experts.98.up_proj", "model.layers.78.mlp.experts.99.up_proj", "model.layers.78.mlp.experts.100.up_proj", "model.layers.78.mlp.experts.101.up_proj", "model.layers.78.mlp.experts.102.up_proj", "model.layers.78.mlp.experts.103.up_proj", "model.layers.78.mlp.experts.104.up_proj", "model.layers.78.mlp.experts.105.up_proj", "model.layers.78.mlp.experts.106.up_proj", "model.layers.78.mlp.experts.107.up_proj", "model.layers.78.mlp.experts.108.up_proj", "model.layers.78.mlp.experts.109.up_proj", "model.layers.78.mlp.experts.110.up_proj", "model.layers.78.mlp.experts.111.up_proj", "model.layers.78.mlp.experts.112.up_proj", "model.layers.78.mlp.experts.113.up_proj", "model.layers.78.mlp.experts.114.up_proj", "model.layers.78.mlp.experts.115.up_proj", "model.layers.78.mlp.experts.116.up_proj", "model.layers.78.mlp.experts.117.up_proj", "model.layers.78.mlp.experts.118.up_proj", "model.layers.78.mlp.experts.119.up_proj", "model.layers.78.mlp.experts.120.up_proj", "model.layers.78.mlp.experts.121.up_proj", "model.layers.78.mlp.experts.122.up_proj", "model.layers.78.mlp.experts.123.up_proj", "model.layers.78.mlp.experts.124.up_proj", "model.layers.78.mlp.experts.125.up_proj", "model.layers.78.mlp.experts.126.up_proj", "model.layers.78.mlp.experts.127.up_proj", "model.layers.78.mlp.experts.128.up_proj", "model.layers.78.mlp.experts.129.up_proj", "model.layers.78.mlp.experts.130.up_proj", "model.layers.78.mlp.experts.131.up_proj", "model.layers.78.mlp.experts.132.up_proj", "model.layers.78.mlp.experts.133.up_proj", "model.layers.78.mlp.experts.134.up_proj", "model.layers.78.mlp.experts.135.up_proj", "model.layers.78.mlp.experts.136.up_proj", "model.layers.78.mlp.experts.137.up_proj", "model.layers.78.mlp.experts.138.up_proj", "model.layers.78.mlp.experts.139.up_proj", "model.layers.78.mlp.experts.140.up_proj", "model.layers.78.mlp.experts.141.up_proj", "model.layers.78.mlp.experts.142.up_proj", "model.layers.78.mlp.experts.143.up_proj", "model.layers.78.mlp.experts.144.up_proj", "model.layers.78.mlp.experts.145.up_proj", "model.layers.78.mlp.experts.146.up_proj", "model.layers.78.mlp.experts.147.up_proj", "model.layers.78.mlp.experts.148.up_proj", "model.layers.78.mlp.experts.149.up_proj", "model.layers.78.mlp.experts.150.up_proj", "model.layers.78.mlp.experts.151.up_proj", "model.layers.78.mlp.experts.152.up_proj", "model.layers.78.mlp.experts.153.up_proj", "model.layers.78.mlp.experts.154.up_proj", "model.layers.78.mlp.experts.155.up_proj", "model.layers.78.mlp.experts.156.up_proj", "model.layers.78.mlp.experts.157.up_proj", "model.layers.78.mlp.experts.158.up_proj", "model.layers.78.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 0.0002538129687308932, "dbits": 2516582400 } ] }, { "idx": 470, "layers": [ "model.layers.78.mlp.experts.0.down_proj", "model.layers.78.mlp.experts.1.down_proj", "model.layers.78.mlp.experts.2.down_proj", "model.layers.78.mlp.experts.3.down_proj", "model.layers.78.mlp.experts.4.down_proj", "model.layers.78.mlp.experts.5.down_proj", "model.layers.78.mlp.experts.6.down_proj", "model.layers.78.mlp.experts.7.down_proj", "model.layers.78.mlp.experts.8.down_proj", "model.layers.78.mlp.experts.9.down_proj", "model.layers.78.mlp.experts.10.down_proj", "model.layers.78.mlp.experts.11.down_proj", "model.layers.78.mlp.experts.12.down_proj", "model.layers.78.mlp.experts.13.down_proj", "model.layers.78.mlp.experts.14.down_proj", "model.layers.78.mlp.experts.15.down_proj", "model.layers.78.mlp.experts.16.down_proj", "model.layers.78.mlp.experts.17.down_proj", "model.layers.78.mlp.experts.18.down_proj", "model.layers.78.mlp.experts.19.down_proj", "model.layers.78.mlp.experts.20.down_proj", "model.layers.78.mlp.experts.21.down_proj", "model.layers.78.mlp.experts.22.down_proj", "model.layers.78.mlp.experts.23.down_proj", "model.layers.78.mlp.experts.24.down_proj", "model.layers.78.mlp.experts.25.down_proj", "model.layers.78.mlp.experts.26.down_proj", "model.layers.78.mlp.experts.27.down_proj", "model.layers.78.mlp.experts.28.down_proj", "model.layers.78.mlp.experts.29.down_proj", "model.layers.78.mlp.experts.30.down_proj", "model.layers.78.mlp.experts.31.down_proj", "model.layers.78.mlp.experts.32.down_proj", "model.layers.78.mlp.experts.33.down_proj", "model.layers.78.mlp.experts.34.down_proj", "model.layers.78.mlp.experts.35.down_proj", "model.layers.78.mlp.experts.36.down_proj", "model.layers.78.mlp.experts.37.down_proj", "model.layers.78.mlp.experts.38.down_proj", "model.layers.78.mlp.experts.39.down_proj", "model.layers.78.mlp.experts.40.down_proj", "model.layers.78.mlp.experts.41.down_proj", "model.layers.78.mlp.experts.42.down_proj", "model.layers.78.mlp.experts.43.down_proj", "model.layers.78.mlp.experts.44.down_proj", "model.layers.78.mlp.experts.45.down_proj", "model.layers.78.mlp.experts.46.down_proj", "model.layers.78.mlp.experts.47.down_proj", "model.layers.78.mlp.experts.48.down_proj", "model.layers.78.mlp.experts.49.down_proj", "model.layers.78.mlp.experts.50.down_proj", "model.layers.78.mlp.experts.51.down_proj", "model.layers.78.mlp.experts.52.down_proj", "model.layers.78.mlp.experts.53.down_proj", "model.layers.78.mlp.experts.54.down_proj", "model.layers.78.mlp.experts.55.down_proj", "model.layers.78.mlp.experts.56.down_proj", "model.layers.78.mlp.experts.57.down_proj", "model.layers.78.mlp.experts.58.down_proj", "model.layers.78.mlp.experts.59.down_proj", "model.layers.78.mlp.experts.60.down_proj", "model.layers.78.mlp.experts.61.down_proj", "model.layers.78.mlp.experts.62.down_proj", "model.layers.78.mlp.experts.63.down_proj", "model.layers.78.mlp.experts.64.down_proj", "model.layers.78.mlp.experts.65.down_proj", "model.layers.78.mlp.experts.66.down_proj", "model.layers.78.mlp.experts.67.down_proj", "model.layers.78.mlp.experts.68.down_proj", "model.layers.78.mlp.experts.69.down_proj", "model.layers.78.mlp.experts.70.down_proj", "model.layers.78.mlp.experts.71.down_proj", "model.layers.78.mlp.experts.72.down_proj", "model.layers.78.mlp.experts.73.down_proj", "model.layers.78.mlp.experts.74.down_proj", "model.layers.78.mlp.experts.75.down_proj", "model.layers.78.mlp.experts.76.down_proj", "model.layers.78.mlp.experts.77.down_proj", "model.layers.78.mlp.experts.78.down_proj", "model.layers.78.mlp.experts.79.down_proj", "model.layers.78.mlp.experts.80.down_proj", "model.layers.78.mlp.experts.81.down_proj", "model.layers.78.mlp.experts.82.down_proj", "model.layers.78.mlp.experts.83.down_proj", "model.layers.78.mlp.experts.84.down_proj", "model.layers.78.mlp.experts.85.down_proj", "model.layers.78.mlp.experts.86.down_proj", "model.layers.78.mlp.experts.87.down_proj", "model.layers.78.mlp.experts.88.down_proj", "model.layers.78.mlp.experts.89.down_proj", "model.layers.78.mlp.experts.90.down_proj", "model.layers.78.mlp.experts.91.down_proj", "model.layers.78.mlp.experts.92.down_proj", "model.layers.78.mlp.experts.93.down_proj", "model.layers.78.mlp.experts.94.down_proj", "model.layers.78.mlp.experts.95.down_proj", "model.layers.78.mlp.experts.96.down_proj", "model.layers.78.mlp.experts.97.down_proj", "model.layers.78.mlp.experts.98.down_proj", "model.layers.78.mlp.experts.99.down_proj", "model.layers.78.mlp.experts.100.down_proj", "model.layers.78.mlp.experts.101.down_proj", "model.layers.78.mlp.experts.102.down_proj", "model.layers.78.mlp.experts.103.down_proj", "model.layers.78.mlp.experts.104.down_proj", "model.layers.78.mlp.experts.105.down_proj", "model.layers.78.mlp.experts.106.down_proj", "model.layers.78.mlp.experts.107.down_proj", "model.layers.78.mlp.experts.108.down_proj", "model.layers.78.mlp.experts.109.down_proj", "model.layers.78.mlp.experts.110.down_proj", "model.layers.78.mlp.experts.111.down_proj", "model.layers.78.mlp.experts.112.down_proj", "model.layers.78.mlp.experts.113.down_proj", "model.layers.78.mlp.experts.114.down_proj", "model.layers.78.mlp.experts.115.down_proj", "model.layers.78.mlp.experts.116.down_proj", "model.layers.78.mlp.experts.117.down_proj", "model.layers.78.mlp.experts.118.down_proj", "model.layers.78.mlp.experts.119.down_proj", "model.layers.78.mlp.experts.120.down_proj", "model.layers.78.mlp.experts.121.down_proj", "model.layers.78.mlp.experts.122.down_proj", "model.layers.78.mlp.experts.123.down_proj", "model.layers.78.mlp.experts.124.down_proj", "model.layers.78.mlp.experts.125.down_proj", "model.layers.78.mlp.experts.126.down_proj", "model.layers.78.mlp.experts.127.down_proj", "model.layers.78.mlp.experts.128.down_proj", "model.layers.78.mlp.experts.129.down_proj", "model.layers.78.mlp.experts.130.down_proj", "model.layers.78.mlp.experts.131.down_proj", "model.layers.78.mlp.experts.132.down_proj", "model.layers.78.mlp.experts.133.down_proj", "model.layers.78.mlp.experts.134.down_proj", "model.layers.78.mlp.experts.135.down_proj", "model.layers.78.mlp.experts.136.down_proj", "model.layers.78.mlp.experts.137.down_proj", "model.layers.78.mlp.experts.138.down_proj", "model.layers.78.mlp.experts.139.down_proj", "model.layers.78.mlp.experts.140.down_proj", "model.layers.78.mlp.experts.141.down_proj", "model.layers.78.mlp.experts.142.down_proj", "model.layers.78.mlp.experts.143.down_proj", "model.layers.78.mlp.experts.144.down_proj", "model.layers.78.mlp.experts.145.down_proj", "model.layers.78.mlp.experts.146.down_proj", "model.layers.78.mlp.experts.147.down_proj", "model.layers.78.mlp.experts.148.down_proj", "model.layers.78.mlp.experts.149.down_proj", "model.layers.78.mlp.experts.150.down_proj", "model.layers.78.mlp.experts.151.down_proj", "model.layers.78.mlp.experts.152.down_proj", "model.layers.78.mlp.experts.153.down_proj", "model.layers.78.mlp.experts.154.down_proj", "model.layers.78.mlp.experts.155.down_proj", "model.layers.78.mlp.experts.156.down_proj", "model.layers.78.mlp.experts.157.down_proj", "model.layers.78.mlp.experts.158.down_proj", "model.layers.78.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00018852800130847447, "dbits": 1258291200 } ] }, { "idx": 471, "layers": [ "model.layers.79.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0001968100666999706, "dbits": 62914560 } ] }, { "idx": 472, "layers": [ "model.layers.79.self_attn.k_proj", "model.layers.79.self_attn.v_proj" ], "candidates": [ { "dkld": -0.005035873502492905, "dbits": 10485760 } ] }, { "idx": 473, "layers": [ "model.layers.79.self_attn.o_proj" ], "candidates": [ { "dkld": -7.084012031555176e-05, "dbits": 62914560 } ] }, { "idx": 474, "layers": [ "model.layers.79.mlp.shared_experts.gate_proj", "model.layers.79.mlp.shared_experts.up_proj", "model.layers.79.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 7.573813199995838e-05, "dbits": 23592960 } ] }, { "idx": 475, "layers": [ "model.layers.79.mlp.experts.0.gate_proj", "model.layers.79.mlp.experts.1.gate_proj", "model.layers.79.mlp.experts.2.gate_proj", "model.layers.79.mlp.experts.3.gate_proj", "model.layers.79.mlp.experts.4.gate_proj", "model.layers.79.mlp.experts.5.gate_proj", "model.layers.79.mlp.experts.6.gate_proj", "model.layers.79.mlp.experts.7.gate_proj", "model.layers.79.mlp.experts.8.gate_proj", "model.layers.79.mlp.experts.9.gate_proj", "model.layers.79.mlp.experts.10.gate_proj", "model.layers.79.mlp.experts.11.gate_proj", "model.layers.79.mlp.experts.12.gate_proj", "model.layers.79.mlp.experts.13.gate_proj", "model.layers.79.mlp.experts.14.gate_proj", "model.layers.79.mlp.experts.15.gate_proj", "model.layers.79.mlp.experts.16.gate_proj", "model.layers.79.mlp.experts.17.gate_proj", "model.layers.79.mlp.experts.18.gate_proj", "model.layers.79.mlp.experts.19.gate_proj", "model.layers.79.mlp.experts.20.gate_proj", "model.layers.79.mlp.experts.21.gate_proj", "model.layers.79.mlp.experts.22.gate_proj", "model.layers.79.mlp.experts.23.gate_proj", "model.layers.79.mlp.experts.24.gate_proj", "model.layers.79.mlp.experts.25.gate_proj", "model.layers.79.mlp.experts.26.gate_proj", "model.layers.79.mlp.experts.27.gate_proj", "model.layers.79.mlp.experts.28.gate_proj", "model.layers.79.mlp.experts.29.gate_proj", "model.layers.79.mlp.experts.30.gate_proj", "model.layers.79.mlp.experts.31.gate_proj", "model.layers.79.mlp.experts.32.gate_proj", "model.layers.79.mlp.experts.33.gate_proj", "model.layers.79.mlp.experts.34.gate_proj", "model.layers.79.mlp.experts.35.gate_proj", "model.layers.79.mlp.experts.36.gate_proj", "model.layers.79.mlp.experts.37.gate_proj", "model.layers.79.mlp.experts.38.gate_proj", "model.layers.79.mlp.experts.39.gate_proj", "model.layers.79.mlp.experts.40.gate_proj", "model.layers.79.mlp.experts.41.gate_proj", "model.layers.79.mlp.experts.42.gate_proj", "model.layers.79.mlp.experts.43.gate_proj", "model.layers.79.mlp.experts.44.gate_proj", "model.layers.79.mlp.experts.45.gate_proj", "model.layers.79.mlp.experts.46.gate_proj", "model.layers.79.mlp.experts.47.gate_proj", "model.layers.79.mlp.experts.48.gate_proj", "model.layers.79.mlp.experts.49.gate_proj", "model.layers.79.mlp.experts.50.gate_proj", "model.layers.79.mlp.experts.51.gate_proj", "model.layers.79.mlp.experts.52.gate_proj", "model.layers.79.mlp.experts.53.gate_proj", "model.layers.79.mlp.experts.54.gate_proj", "model.layers.79.mlp.experts.55.gate_proj", "model.layers.79.mlp.experts.56.gate_proj", "model.layers.79.mlp.experts.57.gate_proj", "model.layers.79.mlp.experts.58.gate_proj", "model.layers.79.mlp.experts.59.gate_proj", "model.layers.79.mlp.experts.60.gate_proj", "model.layers.79.mlp.experts.61.gate_proj", "model.layers.79.mlp.experts.62.gate_proj", "model.layers.79.mlp.experts.63.gate_proj", "model.layers.79.mlp.experts.64.gate_proj", "model.layers.79.mlp.experts.65.gate_proj", "model.layers.79.mlp.experts.66.gate_proj", "model.layers.79.mlp.experts.67.gate_proj", "model.layers.79.mlp.experts.68.gate_proj", "model.layers.79.mlp.experts.69.gate_proj", "model.layers.79.mlp.experts.70.gate_proj", "model.layers.79.mlp.experts.71.gate_proj", "model.layers.79.mlp.experts.72.gate_proj", "model.layers.79.mlp.experts.73.gate_proj", "model.layers.79.mlp.experts.74.gate_proj", "model.layers.79.mlp.experts.75.gate_proj", "model.layers.79.mlp.experts.76.gate_proj", "model.layers.79.mlp.experts.77.gate_proj", "model.layers.79.mlp.experts.78.gate_proj", "model.layers.79.mlp.experts.79.gate_proj", "model.layers.79.mlp.experts.80.gate_proj", "model.layers.79.mlp.experts.81.gate_proj", "model.layers.79.mlp.experts.82.gate_proj", "model.layers.79.mlp.experts.83.gate_proj", "model.layers.79.mlp.experts.84.gate_proj", "model.layers.79.mlp.experts.85.gate_proj", "model.layers.79.mlp.experts.86.gate_proj", "model.layers.79.mlp.experts.87.gate_proj", "model.layers.79.mlp.experts.88.gate_proj", "model.layers.79.mlp.experts.89.gate_proj", "model.layers.79.mlp.experts.90.gate_proj", "model.layers.79.mlp.experts.91.gate_proj", "model.layers.79.mlp.experts.92.gate_proj", "model.layers.79.mlp.experts.93.gate_proj", "model.layers.79.mlp.experts.94.gate_proj", "model.layers.79.mlp.experts.95.gate_proj", "model.layers.79.mlp.experts.96.gate_proj", "model.layers.79.mlp.experts.97.gate_proj", "model.layers.79.mlp.experts.98.gate_proj", "model.layers.79.mlp.experts.99.gate_proj", "model.layers.79.mlp.experts.100.gate_proj", "model.layers.79.mlp.experts.101.gate_proj", "model.layers.79.mlp.experts.102.gate_proj", "model.layers.79.mlp.experts.103.gate_proj", "model.layers.79.mlp.experts.104.gate_proj", "model.layers.79.mlp.experts.105.gate_proj", "model.layers.79.mlp.experts.106.gate_proj", "model.layers.79.mlp.experts.107.gate_proj", "model.layers.79.mlp.experts.108.gate_proj", "model.layers.79.mlp.experts.109.gate_proj", "model.layers.79.mlp.experts.110.gate_proj", "model.layers.79.mlp.experts.111.gate_proj", "model.layers.79.mlp.experts.112.gate_proj", "model.layers.79.mlp.experts.113.gate_proj", "model.layers.79.mlp.experts.114.gate_proj", "model.layers.79.mlp.experts.115.gate_proj", "model.layers.79.mlp.experts.116.gate_proj", "model.layers.79.mlp.experts.117.gate_proj", "model.layers.79.mlp.experts.118.gate_proj", "model.layers.79.mlp.experts.119.gate_proj", "model.layers.79.mlp.experts.120.gate_proj", "model.layers.79.mlp.experts.121.gate_proj", "model.layers.79.mlp.experts.122.gate_proj", "model.layers.79.mlp.experts.123.gate_proj", "model.layers.79.mlp.experts.124.gate_proj", "model.layers.79.mlp.experts.125.gate_proj", "model.layers.79.mlp.experts.126.gate_proj", "model.layers.79.mlp.experts.127.gate_proj", "model.layers.79.mlp.experts.128.gate_proj", "model.layers.79.mlp.experts.129.gate_proj", "model.layers.79.mlp.experts.130.gate_proj", "model.layers.79.mlp.experts.131.gate_proj", "model.layers.79.mlp.experts.132.gate_proj", "model.layers.79.mlp.experts.133.gate_proj", "model.layers.79.mlp.experts.134.gate_proj", "model.layers.79.mlp.experts.135.gate_proj", "model.layers.79.mlp.experts.136.gate_proj", "model.layers.79.mlp.experts.137.gate_proj", "model.layers.79.mlp.experts.138.gate_proj", "model.layers.79.mlp.experts.139.gate_proj", "model.layers.79.mlp.experts.140.gate_proj", "model.layers.79.mlp.experts.141.gate_proj", "model.layers.79.mlp.experts.142.gate_proj", "model.layers.79.mlp.experts.143.gate_proj", "model.layers.79.mlp.experts.144.gate_proj", "model.layers.79.mlp.experts.145.gate_proj", "model.layers.79.mlp.experts.146.gate_proj", "model.layers.79.mlp.experts.147.gate_proj", "model.layers.79.mlp.experts.148.gate_proj", "model.layers.79.mlp.experts.149.gate_proj", "model.layers.79.mlp.experts.150.gate_proj", "model.layers.79.mlp.experts.151.gate_proj", "model.layers.79.mlp.experts.152.gate_proj", "model.layers.79.mlp.experts.153.gate_proj", "model.layers.79.mlp.experts.154.gate_proj", "model.layers.79.mlp.experts.155.gate_proj", "model.layers.79.mlp.experts.156.gate_proj", "model.layers.79.mlp.experts.157.gate_proj", "model.layers.79.mlp.experts.158.gate_proj", "model.layers.79.mlp.experts.159.gate_proj", "model.layers.79.mlp.experts.0.up_proj", "model.layers.79.mlp.experts.1.up_proj", "model.layers.79.mlp.experts.2.up_proj", "model.layers.79.mlp.experts.3.up_proj", "model.layers.79.mlp.experts.4.up_proj", "model.layers.79.mlp.experts.5.up_proj", "model.layers.79.mlp.experts.6.up_proj", "model.layers.79.mlp.experts.7.up_proj", "model.layers.79.mlp.experts.8.up_proj", "model.layers.79.mlp.experts.9.up_proj", "model.layers.79.mlp.experts.10.up_proj", "model.layers.79.mlp.experts.11.up_proj", "model.layers.79.mlp.experts.12.up_proj", "model.layers.79.mlp.experts.13.up_proj", "model.layers.79.mlp.experts.14.up_proj", "model.layers.79.mlp.experts.15.up_proj", "model.layers.79.mlp.experts.16.up_proj", "model.layers.79.mlp.experts.17.up_proj", "model.layers.79.mlp.experts.18.up_proj", "model.layers.79.mlp.experts.19.up_proj", "model.layers.79.mlp.experts.20.up_proj", "model.layers.79.mlp.experts.21.up_proj", "model.layers.79.mlp.experts.22.up_proj", "model.layers.79.mlp.experts.23.up_proj", "model.layers.79.mlp.experts.24.up_proj", "model.layers.79.mlp.experts.25.up_proj", "model.layers.79.mlp.experts.26.up_proj", "model.layers.79.mlp.experts.27.up_proj", "model.layers.79.mlp.experts.28.up_proj", "model.layers.79.mlp.experts.29.up_proj", "model.layers.79.mlp.experts.30.up_proj", "model.layers.79.mlp.experts.31.up_proj", "model.layers.79.mlp.experts.32.up_proj", "model.layers.79.mlp.experts.33.up_proj", "model.layers.79.mlp.experts.34.up_proj", "model.layers.79.mlp.experts.35.up_proj", "model.layers.79.mlp.experts.36.up_proj", "model.layers.79.mlp.experts.37.up_proj", "model.layers.79.mlp.experts.38.up_proj", "model.layers.79.mlp.experts.39.up_proj", "model.layers.79.mlp.experts.40.up_proj", "model.layers.79.mlp.experts.41.up_proj", "model.layers.79.mlp.experts.42.up_proj", "model.layers.79.mlp.experts.43.up_proj", "model.layers.79.mlp.experts.44.up_proj", "model.layers.79.mlp.experts.45.up_proj", "model.layers.79.mlp.experts.46.up_proj", "model.layers.79.mlp.experts.47.up_proj", "model.layers.79.mlp.experts.48.up_proj", "model.layers.79.mlp.experts.49.up_proj", "model.layers.79.mlp.experts.50.up_proj", "model.layers.79.mlp.experts.51.up_proj", "model.layers.79.mlp.experts.52.up_proj", "model.layers.79.mlp.experts.53.up_proj", "model.layers.79.mlp.experts.54.up_proj", "model.layers.79.mlp.experts.55.up_proj", "model.layers.79.mlp.experts.56.up_proj", "model.layers.79.mlp.experts.57.up_proj", "model.layers.79.mlp.experts.58.up_proj", "model.layers.79.mlp.experts.59.up_proj", "model.layers.79.mlp.experts.60.up_proj", "model.layers.79.mlp.experts.61.up_proj", "model.layers.79.mlp.experts.62.up_proj", "model.layers.79.mlp.experts.63.up_proj", "model.layers.79.mlp.experts.64.up_proj", "model.layers.79.mlp.experts.65.up_proj", "model.layers.79.mlp.experts.66.up_proj", "model.layers.79.mlp.experts.67.up_proj", "model.layers.79.mlp.experts.68.up_proj", "model.layers.79.mlp.experts.69.up_proj", "model.layers.79.mlp.experts.70.up_proj", "model.layers.79.mlp.experts.71.up_proj", "model.layers.79.mlp.experts.72.up_proj", "model.layers.79.mlp.experts.73.up_proj", "model.layers.79.mlp.experts.74.up_proj", "model.layers.79.mlp.experts.75.up_proj", "model.layers.79.mlp.experts.76.up_proj", "model.layers.79.mlp.experts.77.up_proj", "model.layers.79.mlp.experts.78.up_proj", "model.layers.79.mlp.experts.79.up_proj", "model.layers.79.mlp.experts.80.up_proj", "model.layers.79.mlp.experts.81.up_proj", "model.layers.79.mlp.experts.82.up_proj", "model.layers.79.mlp.experts.83.up_proj", "model.layers.79.mlp.experts.84.up_proj", "model.layers.79.mlp.experts.85.up_proj", "model.layers.79.mlp.experts.86.up_proj", "model.layers.79.mlp.experts.87.up_proj", "model.layers.79.mlp.experts.88.up_proj", "model.layers.79.mlp.experts.89.up_proj", "model.layers.79.mlp.experts.90.up_proj", "model.layers.79.mlp.experts.91.up_proj", "model.layers.79.mlp.experts.92.up_proj", "model.layers.79.mlp.experts.93.up_proj", "model.layers.79.mlp.experts.94.up_proj", "model.layers.79.mlp.experts.95.up_proj", "model.layers.79.mlp.experts.96.up_proj", "model.layers.79.mlp.experts.97.up_proj", "model.layers.79.mlp.experts.98.up_proj", "model.layers.79.mlp.experts.99.up_proj", "model.layers.79.mlp.experts.100.up_proj", "model.layers.79.mlp.experts.101.up_proj", "model.layers.79.mlp.experts.102.up_proj", "model.layers.79.mlp.experts.103.up_proj", "model.layers.79.mlp.experts.104.up_proj", "model.layers.79.mlp.experts.105.up_proj", "model.layers.79.mlp.experts.106.up_proj", "model.layers.79.mlp.experts.107.up_proj", "model.layers.79.mlp.experts.108.up_proj", "model.layers.79.mlp.experts.109.up_proj", "model.layers.79.mlp.experts.110.up_proj", "model.layers.79.mlp.experts.111.up_proj", "model.layers.79.mlp.experts.112.up_proj", "model.layers.79.mlp.experts.113.up_proj", "model.layers.79.mlp.experts.114.up_proj", "model.layers.79.mlp.experts.115.up_proj", "model.layers.79.mlp.experts.116.up_proj", "model.layers.79.mlp.experts.117.up_proj", "model.layers.79.mlp.experts.118.up_proj", "model.layers.79.mlp.experts.119.up_proj", "model.layers.79.mlp.experts.120.up_proj", "model.layers.79.mlp.experts.121.up_proj", "model.layers.79.mlp.experts.122.up_proj", "model.layers.79.mlp.experts.123.up_proj", "model.layers.79.mlp.experts.124.up_proj", "model.layers.79.mlp.experts.125.up_proj", "model.layers.79.mlp.experts.126.up_proj", "model.layers.79.mlp.experts.127.up_proj", "model.layers.79.mlp.experts.128.up_proj", "model.layers.79.mlp.experts.129.up_proj", "model.layers.79.mlp.experts.130.up_proj", "model.layers.79.mlp.experts.131.up_proj", "model.layers.79.mlp.experts.132.up_proj", "model.layers.79.mlp.experts.133.up_proj", "model.layers.79.mlp.experts.134.up_proj", "model.layers.79.mlp.experts.135.up_proj", "model.layers.79.mlp.experts.136.up_proj", "model.layers.79.mlp.experts.137.up_proj", "model.layers.79.mlp.experts.138.up_proj", "model.layers.79.mlp.experts.139.up_proj", "model.layers.79.mlp.experts.140.up_proj", "model.layers.79.mlp.experts.141.up_proj", "model.layers.79.mlp.experts.142.up_proj", "model.layers.79.mlp.experts.143.up_proj", "model.layers.79.mlp.experts.144.up_proj", "model.layers.79.mlp.experts.145.up_proj", "model.layers.79.mlp.experts.146.up_proj", "model.layers.79.mlp.experts.147.up_proj", "model.layers.79.mlp.experts.148.up_proj", "model.layers.79.mlp.experts.149.up_proj", "model.layers.79.mlp.experts.150.up_proj", "model.layers.79.mlp.experts.151.up_proj", "model.layers.79.mlp.experts.152.up_proj", "model.layers.79.mlp.experts.153.up_proj", "model.layers.79.mlp.experts.154.up_proj", "model.layers.79.mlp.experts.155.up_proj", "model.layers.79.mlp.experts.156.up_proj", "model.layers.79.mlp.experts.157.up_proj", "model.layers.79.mlp.experts.158.up_proj", "model.layers.79.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": 8.620023727418102e-05, "dbits": 2516582400 } ] }, { "idx": 476, "layers": [ "model.layers.79.mlp.experts.0.down_proj", "model.layers.79.mlp.experts.1.down_proj", "model.layers.79.mlp.experts.2.down_proj", "model.layers.79.mlp.experts.3.down_proj", "model.layers.79.mlp.experts.4.down_proj", "model.layers.79.mlp.experts.5.down_proj", "model.layers.79.mlp.experts.6.down_proj", "model.layers.79.mlp.experts.7.down_proj", "model.layers.79.mlp.experts.8.down_proj", "model.layers.79.mlp.experts.9.down_proj", "model.layers.79.mlp.experts.10.down_proj", "model.layers.79.mlp.experts.11.down_proj", "model.layers.79.mlp.experts.12.down_proj", "model.layers.79.mlp.experts.13.down_proj", "model.layers.79.mlp.experts.14.down_proj", "model.layers.79.mlp.experts.15.down_proj", "model.layers.79.mlp.experts.16.down_proj", "model.layers.79.mlp.experts.17.down_proj", "model.layers.79.mlp.experts.18.down_proj", "model.layers.79.mlp.experts.19.down_proj", "model.layers.79.mlp.experts.20.down_proj", "model.layers.79.mlp.experts.21.down_proj", "model.layers.79.mlp.experts.22.down_proj", "model.layers.79.mlp.experts.23.down_proj", "model.layers.79.mlp.experts.24.down_proj", "model.layers.79.mlp.experts.25.down_proj", "model.layers.79.mlp.experts.26.down_proj", "model.layers.79.mlp.experts.27.down_proj", "model.layers.79.mlp.experts.28.down_proj", "model.layers.79.mlp.experts.29.down_proj", "model.layers.79.mlp.experts.30.down_proj", "model.layers.79.mlp.experts.31.down_proj", "model.layers.79.mlp.experts.32.down_proj", "model.layers.79.mlp.experts.33.down_proj", "model.layers.79.mlp.experts.34.down_proj", "model.layers.79.mlp.experts.35.down_proj", "model.layers.79.mlp.experts.36.down_proj", "model.layers.79.mlp.experts.37.down_proj", "model.layers.79.mlp.experts.38.down_proj", "model.layers.79.mlp.experts.39.down_proj", "model.layers.79.mlp.experts.40.down_proj", "model.layers.79.mlp.experts.41.down_proj", "model.layers.79.mlp.experts.42.down_proj", "model.layers.79.mlp.experts.43.down_proj", "model.layers.79.mlp.experts.44.down_proj", "model.layers.79.mlp.experts.45.down_proj", "model.layers.79.mlp.experts.46.down_proj", "model.layers.79.mlp.experts.47.down_proj", "model.layers.79.mlp.experts.48.down_proj", "model.layers.79.mlp.experts.49.down_proj", "model.layers.79.mlp.experts.50.down_proj", "model.layers.79.mlp.experts.51.down_proj", "model.layers.79.mlp.experts.52.down_proj", "model.layers.79.mlp.experts.53.down_proj", "model.layers.79.mlp.experts.54.down_proj", "model.layers.79.mlp.experts.55.down_proj", "model.layers.79.mlp.experts.56.down_proj", "model.layers.79.mlp.experts.57.down_proj", "model.layers.79.mlp.experts.58.down_proj", "model.layers.79.mlp.experts.59.down_proj", "model.layers.79.mlp.experts.60.down_proj", "model.layers.79.mlp.experts.61.down_proj", "model.layers.79.mlp.experts.62.down_proj", "model.layers.79.mlp.experts.63.down_proj", "model.layers.79.mlp.experts.64.down_proj", "model.layers.79.mlp.experts.65.down_proj", "model.layers.79.mlp.experts.66.down_proj", "model.layers.79.mlp.experts.67.down_proj", "model.layers.79.mlp.experts.68.down_proj", "model.layers.79.mlp.experts.69.down_proj", "model.layers.79.mlp.experts.70.down_proj", "model.layers.79.mlp.experts.71.down_proj", "model.layers.79.mlp.experts.72.down_proj", "model.layers.79.mlp.experts.73.down_proj", "model.layers.79.mlp.experts.74.down_proj", "model.layers.79.mlp.experts.75.down_proj", "model.layers.79.mlp.experts.76.down_proj", "model.layers.79.mlp.experts.77.down_proj", "model.layers.79.mlp.experts.78.down_proj", "model.layers.79.mlp.experts.79.down_proj", "model.layers.79.mlp.experts.80.down_proj", "model.layers.79.mlp.experts.81.down_proj", "model.layers.79.mlp.experts.82.down_proj", "model.layers.79.mlp.experts.83.down_proj", "model.layers.79.mlp.experts.84.down_proj", "model.layers.79.mlp.experts.85.down_proj", "model.layers.79.mlp.experts.86.down_proj", "model.layers.79.mlp.experts.87.down_proj", "model.layers.79.mlp.experts.88.down_proj", "model.layers.79.mlp.experts.89.down_proj", "model.layers.79.mlp.experts.90.down_proj", "model.layers.79.mlp.experts.91.down_proj", "model.layers.79.mlp.experts.92.down_proj", "model.layers.79.mlp.experts.93.down_proj", "model.layers.79.mlp.experts.94.down_proj", "model.layers.79.mlp.experts.95.down_proj", "model.layers.79.mlp.experts.96.down_proj", "model.layers.79.mlp.experts.97.down_proj", "model.layers.79.mlp.experts.98.down_proj", "model.layers.79.mlp.experts.99.down_proj", "model.layers.79.mlp.experts.100.down_proj", "model.layers.79.mlp.experts.101.down_proj", "model.layers.79.mlp.experts.102.down_proj", "model.layers.79.mlp.experts.103.down_proj", "model.layers.79.mlp.experts.104.down_proj", "model.layers.79.mlp.experts.105.down_proj", "model.layers.79.mlp.experts.106.down_proj", "model.layers.79.mlp.experts.107.down_proj", "model.layers.79.mlp.experts.108.down_proj", "model.layers.79.mlp.experts.109.down_proj", "model.layers.79.mlp.experts.110.down_proj", "model.layers.79.mlp.experts.111.down_proj", "model.layers.79.mlp.experts.112.down_proj", "model.layers.79.mlp.experts.113.down_proj", "model.layers.79.mlp.experts.114.down_proj", "model.layers.79.mlp.experts.115.down_proj", "model.layers.79.mlp.experts.116.down_proj", "model.layers.79.mlp.experts.117.down_proj", "model.layers.79.mlp.experts.118.down_proj", "model.layers.79.mlp.experts.119.down_proj", "model.layers.79.mlp.experts.120.down_proj", "model.layers.79.mlp.experts.121.down_proj", "model.layers.79.mlp.experts.122.down_proj", "model.layers.79.mlp.experts.123.down_proj", "model.layers.79.mlp.experts.124.down_proj", "model.layers.79.mlp.experts.125.down_proj", "model.layers.79.mlp.experts.126.down_proj", "model.layers.79.mlp.experts.127.down_proj", "model.layers.79.mlp.experts.128.down_proj", "model.layers.79.mlp.experts.129.down_proj", "model.layers.79.mlp.experts.130.down_proj", "model.layers.79.mlp.experts.131.down_proj", "model.layers.79.mlp.experts.132.down_proj", "model.layers.79.mlp.experts.133.down_proj", "model.layers.79.mlp.experts.134.down_proj", "model.layers.79.mlp.experts.135.down_proj", "model.layers.79.mlp.experts.136.down_proj", "model.layers.79.mlp.experts.137.down_proj", "model.layers.79.mlp.experts.138.down_proj", "model.layers.79.mlp.experts.139.down_proj", "model.layers.79.mlp.experts.140.down_proj", "model.layers.79.mlp.experts.141.down_proj", "model.layers.79.mlp.experts.142.down_proj", "model.layers.79.mlp.experts.143.down_proj", "model.layers.79.mlp.experts.144.down_proj", "model.layers.79.mlp.experts.145.down_proj", "model.layers.79.mlp.experts.146.down_proj", "model.layers.79.mlp.experts.147.down_proj", "model.layers.79.mlp.experts.148.down_proj", "model.layers.79.mlp.experts.149.down_proj", "model.layers.79.mlp.experts.150.down_proj", "model.layers.79.mlp.experts.151.down_proj", "model.layers.79.mlp.experts.152.down_proj", "model.layers.79.mlp.experts.153.down_proj", "model.layers.79.mlp.experts.154.down_proj", "model.layers.79.mlp.experts.155.down_proj", "model.layers.79.mlp.experts.156.down_proj", "model.layers.79.mlp.experts.157.down_proj", "model.layers.79.mlp.experts.158.down_proj", "model.layers.79.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0006876982748508786, "dbits": 1258291200 } ] }, { "idx": 477, "layers": [ "model.layers.80.self_attn.q_proj" ], "candidates": [ { "dkld": -0.011156686395406734, "dbits": 62914560 } ] }, { "idx": 478, "layers": [ "model.layers.80.self_attn.k_proj", "model.layers.80.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00012397989630696937, "dbits": 10485760 } ] }, { "idx": 479, "layers": [ "model.layers.80.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00017409250140193455, "dbits": 62914560 } ] }, { "idx": 480, "layers": [ "model.layers.80.mlp.shared_experts.gate_proj", "model.layers.80.mlp.shared_experts.up_proj", "model.layers.80.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.002746395021677006, "dbits": 23592960 } ] }, { "idx": 481, "layers": [ "model.layers.80.mlp.experts.0.gate_proj", "model.layers.80.mlp.experts.1.gate_proj", "model.layers.80.mlp.experts.2.gate_proj", "model.layers.80.mlp.experts.3.gate_proj", "model.layers.80.mlp.experts.4.gate_proj", "model.layers.80.mlp.experts.5.gate_proj", "model.layers.80.mlp.experts.6.gate_proj", "model.layers.80.mlp.experts.7.gate_proj", "model.layers.80.mlp.experts.8.gate_proj", "model.layers.80.mlp.experts.9.gate_proj", "model.layers.80.mlp.experts.10.gate_proj", "model.layers.80.mlp.experts.11.gate_proj", "model.layers.80.mlp.experts.12.gate_proj", "model.layers.80.mlp.experts.13.gate_proj", "model.layers.80.mlp.experts.14.gate_proj", "model.layers.80.mlp.experts.15.gate_proj", "model.layers.80.mlp.experts.16.gate_proj", "model.layers.80.mlp.experts.17.gate_proj", "model.layers.80.mlp.experts.18.gate_proj", "model.layers.80.mlp.experts.19.gate_proj", "model.layers.80.mlp.experts.20.gate_proj", "model.layers.80.mlp.experts.21.gate_proj", "model.layers.80.mlp.experts.22.gate_proj", "model.layers.80.mlp.experts.23.gate_proj", "model.layers.80.mlp.experts.24.gate_proj", "model.layers.80.mlp.experts.25.gate_proj", "model.layers.80.mlp.experts.26.gate_proj", "model.layers.80.mlp.experts.27.gate_proj", "model.layers.80.mlp.experts.28.gate_proj", "model.layers.80.mlp.experts.29.gate_proj", "model.layers.80.mlp.experts.30.gate_proj", "model.layers.80.mlp.experts.31.gate_proj", "model.layers.80.mlp.experts.32.gate_proj", "model.layers.80.mlp.experts.33.gate_proj", "model.layers.80.mlp.experts.34.gate_proj", "model.layers.80.mlp.experts.35.gate_proj", "model.layers.80.mlp.experts.36.gate_proj", "model.layers.80.mlp.experts.37.gate_proj", "model.layers.80.mlp.experts.38.gate_proj", "model.layers.80.mlp.experts.39.gate_proj", "model.layers.80.mlp.experts.40.gate_proj", "model.layers.80.mlp.experts.41.gate_proj", "model.layers.80.mlp.experts.42.gate_proj", "model.layers.80.mlp.experts.43.gate_proj", "model.layers.80.mlp.experts.44.gate_proj", "model.layers.80.mlp.experts.45.gate_proj", "model.layers.80.mlp.experts.46.gate_proj", "model.layers.80.mlp.experts.47.gate_proj", "model.layers.80.mlp.experts.48.gate_proj", "model.layers.80.mlp.experts.49.gate_proj", "model.layers.80.mlp.experts.50.gate_proj", "model.layers.80.mlp.experts.51.gate_proj", "model.layers.80.mlp.experts.52.gate_proj", "model.layers.80.mlp.experts.53.gate_proj", "model.layers.80.mlp.experts.54.gate_proj", "model.layers.80.mlp.experts.55.gate_proj", "model.layers.80.mlp.experts.56.gate_proj", "model.layers.80.mlp.experts.57.gate_proj", "model.layers.80.mlp.experts.58.gate_proj", "model.layers.80.mlp.experts.59.gate_proj", "model.layers.80.mlp.experts.60.gate_proj", "model.layers.80.mlp.experts.61.gate_proj", "model.layers.80.mlp.experts.62.gate_proj", "model.layers.80.mlp.experts.63.gate_proj", "model.layers.80.mlp.experts.64.gate_proj", "model.layers.80.mlp.experts.65.gate_proj", "model.layers.80.mlp.experts.66.gate_proj", "model.layers.80.mlp.experts.67.gate_proj", "model.layers.80.mlp.experts.68.gate_proj", "model.layers.80.mlp.experts.69.gate_proj", "model.layers.80.mlp.experts.70.gate_proj", "model.layers.80.mlp.experts.71.gate_proj", "model.layers.80.mlp.experts.72.gate_proj", "model.layers.80.mlp.experts.73.gate_proj", "model.layers.80.mlp.experts.74.gate_proj", "model.layers.80.mlp.experts.75.gate_proj", "model.layers.80.mlp.experts.76.gate_proj", "model.layers.80.mlp.experts.77.gate_proj", "model.layers.80.mlp.experts.78.gate_proj", "model.layers.80.mlp.experts.79.gate_proj", "model.layers.80.mlp.experts.80.gate_proj", "model.layers.80.mlp.experts.81.gate_proj", "model.layers.80.mlp.experts.82.gate_proj", "model.layers.80.mlp.experts.83.gate_proj", "model.layers.80.mlp.experts.84.gate_proj", "model.layers.80.mlp.experts.85.gate_proj", "model.layers.80.mlp.experts.86.gate_proj", "model.layers.80.mlp.experts.87.gate_proj", "model.layers.80.mlp.experts.88.gate_proj", "model.layers.80.mlp.experts.89.gate_proj", "model.layers.80.mlp.experts.90.gate_proj", "model.layers.80.mlp.experts.91.gate_proj", "model.layers.80.mlp.experts.92.gate_proj", "model.layers.80.mlp.experts.93.gate_proj", "model.layers.80.mlp.experts.94.gate_proj", "model.layers.80.mlp.experts.95.gate_proj", "model.layers.80.mlp.experts.96.gate_proj", "model.layers.80.mlp.experts.97.gate_proj", "model.layers.80.mlp.experts.98.gate_proj", "model.layers.80.mlp.experts.99.gate_proj", "model.layers.80.mlp.experts.100.gate_proj", "model.layers.80.mlp.experts.101.gate_proj", "model.layers.80.mlp.experts.102.gate_proj", "model.layers.80.mlp.experts.103.gate_proj", "model.layers.80.mlp.experts.104.gate_proj", "model.layers.80.mlp.experts.105.gate_proj", "model.layers.80.mlp.experts.106.gate_proj", "model.layers.80.mlp.experts.107.gate_proj", "model.layers.80.mlp.experts.108.gate_proj", "model.layers.80.mlp.experts.109.gate_proj", "model.layers.80.mlp.experts.110.gate_proj", "model.layers.80.mlp.experts.111.gate_proj", "model.layers.80.mlp.experts.112.gate_proj", "model.layers.80.mlp.experts.113.gate_proj", "model.layers.80.mlp.experts.114.gate_proj", "model.layers.80.mlp.experts.115.gate_proj", "model.layers.80.mlp.experts.116.gate_proj", "model.layers.80.mlp.experts.117.gate_proj", "model.layers.80.mlp.experts.118.gate_proj", "model.layers.80.mlp.experts.119.gate_proj", "model.layers.80.mlp.experts.120.gate_proj", "model.layers.80.mlp.experts.121.gate_proj", "model.layers.80.mlp.experts.122.gate_proj", "model.layers.80.mlp.experts.123.gate_proj", "model.layers.80.mlp.experts.124.gate_proj", "model.layers.80.mlp.experts.125.gate_proj", "model.layers.80.mlp.experts.126.gate_proj", "model.layers.80.mlp.experts.127.gate_proj", "model.layers.80.mlp.experts.128.gate_proj", "model.layers.80.mlp.experts.129.gate_proj", "model.layers.80.mlp.experts.130.gate_proj", "model.layers.80.mlp.experts.131.gate_proj", "model.layers.80.mlp.experts.132.gate_proj", "model.layers.80.mlp.experts.133.gate_proj", "model.layers.80.mlp.experts.134.gate_proj", "model.layers.80.mlp.experts.135.gate_proj", "model.layers.80.mlp.experts.136.gate_proj", "model.layers.80.mlp.experts.137.gate_proj", "model.layers.80.mlp.experts.138.gate_proj", "model.layers.80.mlp.experts.139.gate_proj", "model.layers.80.mlp.experts.140.gate_proj", "model.layers.80.mlp.experts.141.gate_proj", "model.layers.80.mlp.experts.142.gate_proj", "model.layers.80.mlp.experts.143.gate_proj", "model.layers.80.mlp.experts.144.gate_proj", "model.layers.80.mlp.experts.145.gate_proj", "model.layers.80.mlp.experts.146.gate_proj", "model.layers.80.mlp.experts.147.gate_proj", "model.layers.80.mlp.experts.148.gate_proj", "model.layers.80.mlp.experts.149.gate_proj", "model.layers.80.mlp.experts.150.gate_proj", "model.layers.80.mlp.experts.151.gate_proj", "model.layers.80.mlp.experts.152.gate_proj", "model.layers.80.mlp.experts.153.gate_proj", "model.layers.80.mlp.experts.154.gate_proj", "model.layers.80.mlp.experts.155.gate_proj", "model.layers.80.mlp.experts.156.gate_proj", "model.layers.80.mlp.experts.157.gate_proj", "model.layers.80.mlp.experts.158.gate_proj", "model.layers.80.mlp.experts.159.gate_proj", "model.layers.80.mlp.experts.0.up_proj", "model.layers.80.mlp.experts.1.up_proj", "model.layers.80.mlp.experts.2.up_proj", "model.layers.80.mlp.experts.3.up_proj", "model.layers.80.mlp.experts.4.up_proj", "model.layers.80.mlp.experts.5.up_proj", "model.layers.80.mlp.experts.6.up_proj", "model.layers.80.mlp.experts.7.up_proj", "model.layers.80.mlp.experts.8.up_proj", "model.layers.80.mlp.experts.9.up_proj", "model.layers.80.mlp.experts.10.up_proj", "model.layers.80.mlp.experts.11.up_proj", "model.layers.80.mlp.experts.12.up_proj", "model.layers.80.mlp.experts.13.up_proj", "model.layers.80.mlp.experts.14.up_proj", "model.layers.80.mlp.experts.15.up_proj", "model.layers.80.mlp.experts.16.up_proj", "model.layers.80.mlp.experts.17.up_proj", "model.layers.80.mlp.experts.18.up_proj", "model.layers.80.mlp.experts.19.up_proj", "model.layers.80.mlp.experts.20.up_proj", "model.layers.80.mlp.experts.21.up_proj", "model.layers.80.mlp.experts.22.up_proj", "model.layers.80.mlp.experts.23.up_proj", "model.layers.80.mlp.experts.24.up_proj", "model.layers.80.mlp.experts.25.up_proj", "model.layers.80.mlp.experts.26.up_proj", "model.layers.80.mlp.experts.27.up_proj", "model.layers.80.mlp.experts.28.up_proj", "model.layers.80.mlp.experts.29.up_proj", "model.layers.80.mlp.experts.30.up_proj", "model.layers.80.mlp.experts.31.up_proj", "model.layers.80.mlp.experts.32.up_proj", "model.layers.80.mlp.experts.33.up_proj", "model.layers.80.mlp.experts.34.up_proj", "model.layers.80.mlp.experts.35.up_proj", "model.layers.80.mlp.experts.36.up_proj", "model.layers.80.mlp.experts.37.up_proj", "model.layers.80.mlp.experts.38.up_proj", "model.layers.80.mlp.experts.39.up_proj", "model.layers.80.mlp.experts.40.up_proj", "model.layers.80.mlp.experts.41.up_proj", "model.layers.80.mlp.experts.42.up_proj", "model.layers.80.mlp.experts.43.up_proj", "model.layers.80.mlp.experts.44.up_proj", "model.layers.80.mlp.experts.45.up_proj", "model.layers.80.mlp.experts.46.up_proj", "model.layers.80.mlp.experts.47.up_proj", "model.layers.80.mlp.experts.48.up_proj", "model.layers.80.mlp.experts.49.up_proj", "model.layers.80.mlp.experts.50.up_proj", "model.layers.80.mlp.experts.51.up_proj", "model.layers.80.mlp.experts.52.up_proj", "model.layers.80.mlp.experts.53.up_proj", "model.layers.80.mlp.experts.54.up_proj", "model.layers.80.mlp.experts.55.up_proj", "model.layers.80.mlp.experts.56.up_proj", "model.layers.80.mlp.experts.57.up_proj", "model.layers.80.mlp.experts.58.up_proj", "model.layers.80.mlp.experts.59.up_proj", "model.layers.80.mlp.experts.60.up_proj", "model.layers.80.mlp.experts.61.up_proj", "model.layers.80.mlp.experts.62.up_proj", "model.layers.80.mlp.experts.63.up_proj", "model.layers.80.mlp.experts.64.up_proj", "model.layers.80.mlp.experts.65.up_proj", "model.layers.80.mlp.experts.66.up_proj", "model.layers.80.mlp.experts.67.up_proj", "model.layers.80.mlp.experts.68.up_proj", "model.layers.80.mlp.experts.69.up_proj", "model.layers.80.mlp.experts.70.up_proj", "model.layers.80.mlp.experts.71.up_proj", "model.layers.80.mlp.experts.72.up_proj", "model.layers.80.mlp.experts.73.up_proj", "model.layers.80.mlp.experts.74.up_proj", "model.layers.80.mlp.experts.75.up_proj", "model.layers.80.mlp.experts.76.up_proj", "model.layers.80.mlp.experts.77.up_proj", "model.layers.80.mlp.experts.78.up_proj", "model.layers.80.mlp.experts.79.up_proj", "model.layers.80.mlp.experts.80.up_proj", "model.layers.80.mlp.experts.81.up_proj", "model.layers.80.mlp.experts.82.up_proj", "model.layers.80.mlp.experts.83.up_proj", "model.layers.80.mlp.experts.84.up_proj", "model.layers.80.mlp.experts.85.up_proj", "model.layers.80.mlp.experts.86.up_proj", "model.layers.80.mlp.experts.87.up_proj", "model.layers.80.mlp.experts.88.up_proj", "model.layers.80.mlp.experts.89.up_proj", "model.layers.80.mlp.experts.90.up_proj", "model.layers.80.mlp.experts.91.up_proj", "model.layers.80.mlp.experts.92.up_proj", "model.layers.80.mlp.experts.93.up_proj", "model.layers.80.mlp.experts.94.up_proj", "model.layers.80.mlp.experts.95.up_proj", "model.layers.80.mlp.experts.96.up_proj", "model.layers.80.mlp.experts.97.up_proj", "model.layers.80.mlp.experts.98.up_proj", "model.layers.80.mlp.experts.99.up_proj", "model.layers.80.mlp.experts.100.up_proj", "model.layers.80.mlp.experts.101.up_proj", "model.layers.80.mlp.experts.102.up_proj", "model.layers.80.mlp.experts.103.up_proj", "model.layers.80.mlp.experts.104.up_proj", "model.layers.80.mlp.experts.105.up_proj", "model.layers.80.mlp.experts.106.up_proj", "model.layers.80.mlp.experts.107.up_proj", "model.layers.80.mlp.experts.108.up_proj", "model.layers.80.mlp.experts.109.up_proj", "model.layers.80.mlp.experts.110.up_proj", "model.layers.80.mlp.experts.111.up_proj", "model.layers.80.mlp.experts.112.up_proj", "model.layers.80.mlp.experts.113.up_proj", "model.layers.80.mlp.experts.114.up_proj", "model.layers.80.mlp.experts.115.up_proj", "model.layers.80.mlp.experts.116.up_proj", "model.layers.80.mlp.experts.117.up_proj", "model.layers.80.mlp.experts.118.up_proj", "model.layers.80.mlp.experts.119.up_proj", "model.layers.80.mlp.experts.120.up_proj", "model.layers.80.mlp.experts.121.up_proj", "model.layers.80.mlp.experts.122.up_proj", "model.layers.80.mlp.experts.123.up_proj", "model.layers.80.mlp.experts.124.up_proj", "model.layers.80.mlp.experts.125.up_proj", "model.layers.80.mlp.experts.126.up_proj", "model.layers.80.mlp.experts.127.up_proj", "model.layers.80.mlp.experts.128.up_proj", "model.layers.80.mlp.experts.129.up_proj", "model.layers.80.mlp.experts.130.up_proj", "model.layers.80.mlp.experts.131.up_proj", "model.layers.80.mlp.experts.132.up_proj", "model.layers.80.mlp.experts.133.up_proj", "model.layers.80.mlp.experts.134.up_proj", "model.layers.80.mlp.experts.135.up_proj", "model.layers.80.mlp.experts.136.up_proj", "model.layers.80.mlp.experts.137.up_proj", "model.layers.80.mlp.experts.138.up_proj", "model.layers.80.mlp.experts.139.up_proj", "model.layers.80.mlp.experts.140.up_proj", "model.layers.80.mlp.experts.141.up_proj", "model.layers.80.mlp.experts.142.up_proj", "model.layers.80.mlp.experts.143.up_proj", "model.layers.80.mlp.experts.144.up_proj", "model.layers.80.mlp.experts.145.up_proj", "model.layers.80.mlp.experts.146.up_proj", "model.layers.80.mlp.experts.147.up_proj", "model.layers.80.mlp.experts.148.up_proj", "model.layers.80.mlp.experts.149.up_proj", "model.layers.80.mlp.experts.150.up_proj", "model.layers.80.mlp.experts.151.up_proj", "model.layers.80.mlp.experts.152.up_proj", "model.layers.80.mlp.experts.153.up_proj", "model.layers.80.mlp.experts.154.up_proj", "model.layers.80.mlp.experts.155.up_proj", "model.layers.80.mlp.experts.156.up_proj", "model.layers.80.mlp.experts.157.up_proj", "model.layers.80.mlp.experts.158.up_proj", "model.layers.80.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0002874813973903767, "dbits": 2516582400 } ] }, { "idx": 482, "layers": [ "model.layers.80.mlp.experts.0.down_proj", "model.layers.80.mlp.experts.1.down_proj", "model.layers.80.mlp.experts.2.down_proj", "model.layers.80.mlp.experts.3.down_proj", "model.layers.80.mlp.experts.4.down_proj", "model.layers.80.mlp.experts.5.down_proj", "model.layers.80.mlp.experts.6.down_proj", "model.layers.80.mlp.experts.7.down_proj", "model.layers.80.mlp.experts.8.down_proj", "model.layers.80.mlp.experts.9.down_proj", "model.layers.80.mlp.experts.10.down_proj", "model.layers.80.mlp.experts.11.down_proj", "model.layers.80.mlp.experts.12.down_proj", "model.layers.80.mlp.experts.13.down_proj", "model.layers.80.mlp.experts.14.down_proj", "model.layers.80.mlp.experts.15.down_proj", "model.layers.80.mlp.experts.16.down_proj", "model.layers.80.mlp.experts.17.down_proj", "model.layers.80.mlp.experts.18.down_proj", "model.layers.80.mlp.experts.19.down_proj", "model.layers.80.mlp.experts.20.down_proj", "model.layers.80.mlp.experts.21.down_proj", "model.layers.80.mlp.experts.22.down_proj", "model.layers.80.mlp.experts.23.down_proj", "model.layers.80.mlp.experts.24.down_proj", "model.layers.80.mlp.experts.25.down_proj", "model.layers.80.mlp.experts.26.down_proj", "model.layers.80.mlp.experts.27.down_proj", "model.layers.80.mlp.experts.28.down_proj", "model.layers.80.mlp.experts.29.down_proj", "model.layers.80.mlp.experts.30.down_proj", "model.layers.80.mlp.experts.31.down_proj", "model.layers.80.mlp.experts.32.down_proj", "model.layers.80.mlp.experts.33.down_proj", "model.layers.80.mlp.experts.34.down_proj", "model.layers.80.mlp.experts.35.down_proj", "model.layers.80.mlp.experts.36.down_proj", "model.layers.80.mlp.experts.37.down_proj", "model.layers.80.mlp.experts.38.down_proj", "model.layers.80.mlp.experts.39.down_proj", "model.layers.80.mlp.experts.40.down_proj", "model.layers.80.mlp.experts.41.down_proj", "model.layers.80.mlp.experts.42.down_proj", "model.layers.80.mlp.experts.43.down_proj", "model.layers.80.mlp.experts.44.down_proj", "model.layers.80.mlp.experts.45.down_proj", "model.layers.80.mlp.experts.46.down_proj", "model.layers.80.mlp.experts.47.down_proj", "model.layers.80.mlp.experts.48.down_proj", "model.layers.80.mlp.experts.49.down_proj", "model.layers.80.mlp.experts.50.down_proj", "model.layers.80.mlp.experts.51.down_proj", "model.layers.80.mlp.experts.52.down_proj", "model.layers.80.mlp.experts.53.down_proj", "model.layers.80.mlp.experts.54.down_proj", "model.layers.80.mlp.experts.55.down_proj", "model.layers.80.mlp.experts.56.down_proj", "model.layers.80.mlp.experts.57.down_proj", "model.layers.80.mlp.experts.58.down_proj", "model.layers.80.mlp.experts.59.down_proj", "model.layers.80.mlp.experts.60.down_proj", "model.layers.80.mlp.experts.61.down_proj", "model.layers.80.mlp.experts.62.down_proj", "model.layers.80.mlp.experts.63.down_proj", "model.layers.80.mlp.experts.64.down_proj", "model.layers.80.mlp.experts.65.down_proj", "model.layers.80.mlp.experts.66.down_proj", "model.layers.80.mlp.experts.67.down_proj", "model.layers.80.mlp.experts.68.down_proj", "model.layers.80.mlp.experts.69.down_proj", "model.layers.80.mlp.experts.70.down_proj", "model.layers.80.mlp.experts.71.down_proj", "model.layers.80.mlp.experts.72.down_proj", "model.layers.80.mlp.experts.73.down_proj", "model.layers.80.mlp.experts.74.down_proj", "model.layers.80.mlp.experts.75.down_proj", "model.layers.80.mlp.experts.76.down_proj", "model.layers.80.mlp.experts.77.down_proj", "model.layers.80.mlp.experts.78.down_proj", "model.layers.80.mlp.experts.79.down_proj", "model.layers.80.mlp.experts.80.down_proj", "model.layers.80.mlp.experts.81.down_proj", "model.layers.80.mlp.experts.82.down_proj", "model.layers.80.mlp.experts.83.down_proj", "model.layers.80.mlp.experts.84.down_proj", "model.layers.80.mlp.experts.85.down_proj", "model.layers.80.mlp.experts.86.down_proj", "model.layers.80.mlp.experts.87.down_proj", "model.layers.80.mlp.experts.88.down_proj", "model.layers.80.mlp.experts.89.down_proj", "model.layers.80.mlp.experts.90.down_proj", "model.layers.80.mlp.experts.91.down_proj", "model.layers.80.mlp.experts.92.down_proj", "model.layers.80.mlp.experts.93.down_proj", "model.layers.80.mlp.experts.94.down_proj", "model.layers.80.mlp.experts.95.down_proj", "model.layers.80.mlp.experts.96.down_proj", "model.layers.80.mlp.experts.97.down_proj", "model.layers.80.mlp.experts.98.down_proj", "model.layers.80.mlp.experts.99.down_proj", "model.layers.80.mlp.experts.100.down_proj", "model.layers.80.mlp.experts.101.down_proj", "model.layers.80.mlp.experts.102.down_proj", "model.layers.80.mlp.experts.103.down_proj", "model.layers.80.mlp.experts.104.down_proj", "model.layers.80.mlp.experts.105.down_proj", "model.layers.80.mlp.experts.106.down_proj", "model.layers.80.mlp.experts.107.down_proj", "model.layers.80.mlp.experts.108.down_proj", "model.layers.80.mlp.experts.109.down_proj", "model.layers.80.mlp.experts.110.down_proj", "model.layers.80.mlp.experts.111.down_proj", "model.layers.80.mlp.experts.112.down_proj", "model.layers.80.mlp.experts.113.down_proj", "model.layers.80.mlp.experts.114.down_proj", "model.layers.80.mlp.experts.115.down_proj", "model.layers.80.mlp.experts.116.down_proj", "model.layers.80.mlp.experts.117.down_proj", "model.layers.80.mlp.experts.118.down_proj", "model.layers.80.mlp.experts.119.down_proj", "model.layers.80.mlp.experts.120.down_proj", "model.layers.80.mlp.experts.121.down_proj", "model.layers.80.mlp.experts.122.down_proj", "model.layers.80.mlp.experts.123.down_proj", "model.layers.80.mlp.experts.124.down_proj", "model.layers.80.mlp.experts.125.down_proj", "model.layers.80.mlp.experts.126.down_proj", "model.layers.80.mlp.experts.127.down_proj", "model.layers.80.mlp.experts.128.down_proj", "model.layers.80.mlp.experts.129.down_proj", "model.layers.80.mlp.experts.130.down_proj", "model.layers.80.mlp.experts.131.down_proj", "model.layers.80.mlp.experts.132.down_proj", "model.layers.80.mlp.experts.133.down_proj", "model.layers.80.mlp.experts.134.down_proj", "model.layers.80.mlp.experts.135.down_proj", "model.layers.80.mlp.experts.136.down_proj", "model.layers.80.mlp.experts.137.down_proj", "model.layers.80.mlp.experts.138.down_proj", "model.layers.80.mlp.experts.139.down_proj", "model.layers.80.mlp.experts.140.down_proj", "model.layers.80.mlp.experts.141.down_proj", "model.layers.80.mlp.experts.142.down_proj", "model.layers.80.mlp.experts.143.down_proj", "model.layers.80.mlp.experts.144.down_proj", "model.layers.80.mlp.experts.145.down_proj", "model.layers.80.mlp.experts.146.down_proj", "model.layers.80.mlp.experts.147.down_proj", "model.layers.80.mlp.experts.148.down_proj", "model.layers.80.mlp.experts.149.down_proj", "model.layers.80.mlp.experts.150.down_proj", "model.layers.80.mlp.experts.151.down_proj", "model.layers.80.mlp.experts.152.down_proj", "model.layers.80.mlp.experts.153.down_proj", "model.layers.80.mlp.experts.154.down_proj", "model.layers.80.mlp.experts.155.down_proj", "model.layers.80.mlp.experts.156.down_proj", "model.layers.80.mlp.experts.157.down_proj", "model.layers.80.mlp.experts.158.down_proj", "model.layers.80.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00014282837510110058, "dbits": 1258291200 } ] }, { "idx": 483, "layers": [ "model.layers.81.self_attn.q_proj" ], "candidates": [ { "dkld": -0.001481901854276657, "dbits": 62914560 } ] }, { "idx": 484, "layers": [ "model.layers.81.self_attn.k_proj", "model.layers.81.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0011574439704418182, "dbits": 10485760 } ] }, { "idx": 485, "layers": [ "model.layers.81.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0014045722782611847, "dbits": 62914560 } ] }, { "idx": 486, "layers": [ "model.layers.81.mlp.shared_experts.gate_proj", "model.layers.81.mlp.shared_experts.up_proj", "model.layers.81.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -2.904236316714215e-06, "dbits": 23592960 } ] }, { "idx": 487, "layers": [ "model.layers.81.mlp.experts.0.gate_proj", "model.layers.81.mlp.experts.1.gate_proj", "model.layers.81.mlp.experts.2.gate_proj", "model.layers.81.mlp.experts.3.gate_proj", "model.layers.81.mlp.experts.4.gate_proj", "model.layers.81.mlp.experts.5.gate_proj", "model.layers.81.mlp.experts.6.gate_proj", "model.layers.81.mlp.experts.7.gate_proj", "model.layers.81.mlp.experts.8.gate_proj", "model.layers.81.mlp.experts.9.gate_proj", "model.layers.81.mlp.experts.10.gate_proj", "model.layers.81.mlp.experts.11.gate_proj", "model.layers.81.mlp.experts.12.gate_proj", "model.layers.81.mlp.experts.13.gate_proj", "model.layers.81.mlp.experts.14.gate_proj", "model.layers.81.mlp.experts.15.gate_proj", "model.layers.81.mlp.experts.16.gate_proj", "model.layers.81.mlp.experts.17.gate_proj", "model.layers.81.mlp.experts.18.gate_proj", "model.layers.81.mlp.experts.19.gate_proj", "model.layers.81.mlp.experts.20.gate_proj", "model.layers.81.mlp.experts.21.gate_proj", "model.layers.81.mlp.experts.22.gate_proj", "model.layers.81.mlp.experts.23.gate_proj", "model.layers.81.mlp.experts.24.gate_proj", "model.layers.81.mlp.experts.25.gate_proj", "model.layers.81.mlp.experts.26.gate_proj", "model.layers.81.mlp.experts.27.gate_proj", "model.layers.81.mlp.experts.28.gate_proj", "model.layers.81.mlp.experts.29.gate_proj", "model.layers.81.mlp.experts.30.gate_proj", "model.layers.81.mlp.experts.31.gate_proj", "model.layers.81.mlp.experts.32.gate_proj", "model.layers.81.mlp.experts.33.gate_proj", "model.layers.81.mlp.experts.34.gate_proj", "model.layers.81.mlp.experts.35.gate_proj", "model.layers.81.mlp.experts.36.gate_proj", "model.layers.81.mlp.experts.37.gate_proj", "model.layers.81.mlp.experts.38.gate_proj", "model.layers.81.mlp.experts.39.gate_proj", "model.layers.81.mlp.experts.40.gate_proj", "model.layers.81.mlp.experts.41.gate_proj", "model.layers.81.mlp.experts.42.gate_proj", "model.layers.81.mlp.experts.43.gate_proj", "model.layers.81.mlp.experts.44.gate_proj", "model.layers.81.mlp.experts.45.gate_proj", "model.layers.81.mlp.experts.46.gate_proj", "model.layers.81.mlp.experts.47.gate_proj", "model.layers.81.mlp.experts.48.gate_proj", "model.layers.81.mlp.experts.49.gate_proj", "model.layers.81.mlp.experts.50.gate_proj", "model.layers.81.mlp.experts.51.gate_proj", "model.layers.81.mlp.experts.52.gate_proj", "model.layers.81.mlp.experts.53.gate_proj", "model.layers.81.mlp.experts.54.gate_proj", "model.layers.81.mlp.experts.55.gate_proj", "model.layers.81.mlp.experts.56.gate_proj", "model.layers.81.mlp.experts.57.gate_proj", "model.layers.81.mlp.experts.58.gate_proj", "model.layers.81.mlp.experts.59.gate_proj", "model.layers.81.mlp.experts.60.gate_proj", "model.layers.81.mlp.experts.61.gate_proj", "model.layers.81.mlp.experts.62.gate_proj", "model.layers.81.mlp.experts.63.gate_proj", "model.layers.81.mlp.experts.64.gate_proj", "model.layers.81.mlp.experts.65.gate_proj", "model.layers.81.mlp.experts.66.gate_proj", "model.layers.81.mlp.experts.67.gate_proj", "model.layers.81.mlp.experts.68.gate_proj", "model.layers.81.mlp.experts.69.gate_proj", "model.layers.81.mlp.experts.70.gate_proj", "model.layers.81.mlp.experts.71.gate_proj", "model.layers.81.mlp.experts.72.gate_proj", "model.layers.81.mlp.experts.73.gate_proj", "model.layers.81.mlp.experts.74.gate_proj", "model.layers.81.mlp.experts.75.gate_proj", "model.layers.81.mlp.experts.76.gate_proj", "model.layers.81.mlp.experts.77.gate_proj", "model.layers.81.mlp.experts.78.gate_proj", "model.layers.81.mlp.experts.79.gate_proj", "model.layers.81.mlp.experts.80.gate_proj", "model.layers.81.mlp.experts.81.gate_proj", "model.layers.81.mlp.experts.82.gate_proj", "model.layers.81.mlp.experts.83.gate_proj", "model.layers.81.mlp.experts.84.gate_proj", "model.layers.81.mlp.experts.85.gate_proj", "model.layers.81.mlp.experts.86.gate_proj", "model.layers.81.mlp.experts.87.gate_proj", "model.layers.81.mlp.experts.88.gate_proj", "model.layers.81.mlp.experts.89.gate_proj", "model.layers.81.mlp.experts.90.gate_proj", "model.layers.81.mlp.experts.91.gate_proj", "model.layers.81.mlp.experts.92.gate_proj", "model.layers.81.mlp.experts.93.gate_proj", "model.layers.81.mlp.experts.94.gate_proj", "model.layers.81.mlp.experts.95.gate_proj", "model.layers.81.mlp.experts.96.gate_proj", "model.layers.81.mlp.experts.97.gate_proj", "model.layers.81.mlp.experts.98.gate_proj", "model.layers.81.mlp.experts.99.gate_proj", "model.layers.81.mlp.experts.100.gate_proj", "model.layers.81.mlp.experts.101.gate_proj", "model.layers.81.mlp.experts.102.gate_proj", "model.layers.81.mlp.experts.103.gate_proj", "model.layers.81.mlp.experts.104.gate_proj", "model.layers.81.mlp.experts.105.gate_proj", "model.layers.81.mlp.experts.106.gate_proj", "model.layers.81.mlp.experts.107.gate_proj", "model.layers.81.mlp.experts.108.gate_proj", "model.layers.81.mlp.experts.109.gate_proj", "model.layers.81.mlp.experts.110.gate_proj", "model.layers.81.mlp.experts.111.gate_proj", "model.layers.81.mlp.experts.112.gate_proj", "model.layers.81.mlp.experts.113.gate_proj", "model.layers.81.mlp.experts.114.gate_proj", "model.layers.81.mlp.experts.115.gate_proj", "model.layers.81.mlp.experts.116.gate_proj", "model.layers.81.mlp.experts.117.gate_proj", "model.layers.81.mlp.experts.118.gate_proj", "model.layers.81.mlp.experts.119.gate_proj", "model.layers.81.mlp.experts.120.gate_proj", "model.layers.81.mlp.experts.121.gate_proj", "model.layers.81.mlp.experts.122.gate_proj", "model.layers.81.mlp.experts.123.gate_proj", "model.layers.81.mlp.experts.124.gate_proj", "model.layers.81.mlp.experts.125.gate_proj", "model.layers.81.mlp.experts.126.gate_proj", "model.layers.81.mlp.experts.127.gate_proj", "model.layers.81.mlp.experts.128.gate_proj", "model.layers.81.mlp.experts.129.gate_proj", "model.layers.81.mlp.experts.130.gate_proj", "model.layers.81.mlp.experts.131.gate_proj", "model.layers.81.mlp.experts.132.gate_proj", "model.layers.81.mlp.experts.133.gate_proj", "model.layers.81.mlp.experts.134.gate_proj", "model.layers.81.mlp.experts.135.gate_proj", "model.layers.81.mlp.experts.136.gate_proj", "model.layers.81.mlp.experts.137.gate_proj", "model.layers.81.mlp.experts.138.gate_proj", "model.layers.81.mlp.experts.139.gate_proj", "model.layers.81.mlp.experts.140.gate_proj", "model.layers.81.mlp.experts.141.gate_proj", "model.layers.81.mlp.experts.142.gate_proj", "model.layers.81.mlp.experts.143.gate_proj", "model.layers.81.mlp.experts.144.gate_proj", "model.layers.81.mlp.experts.145.gate_proj", "model.layers.81.mlp.experts.146.gate_proj", "model.layers.81.mlp.experts.147.gate_proj", "model.layers.81.mlp.experts.148.gate_proj", "model.layers.81.mlp.experts.149.gate_proj", "model.layers.81.mlp.experts.150.gate_proj", "model.layers.81.mlp.experts.151.gate_proj", "model.layers.81.mlp.experts.152.gate_proj", "model.layers.81.mlp.experts.153.gate_proj", "model.layers.81.mlp.experts.154.gate_proj", "model.layers.81.mlp.experts.155.gate_proj", "model.layers.81.mlp.experts.156.gate_proj", "model.layers.81.mlp.experts.157.gate_proj", "model.layers.81.mlp.experts.158.gate_proj", "model.layers.81.mlp.experts.159.gate_proj", "model.layers.81.mlp.experts.0.up_proj", "model.layers.81.mlp.experts.1.up_proj", "model.layers.81.mlp.experts.2.up_proj", "model.layers.81.mlp.experts.3.up_proj", "model.layers.81.mlp.experts.4.up_proj", "model.layers.81.mlp.experts.5.up_proj", "model.layers.81.mlp.experts.6.up_proj", "model.layers.81.mlp.experts.7.up_proj", "model.layers.81.mlp.experts.8.up_proj", "model.layers.81.mlp.experts.9.up_proj", "model.layers.81.mlp.experts.10.up_proj", "model.layers.81.mlp.experts.11.up_proj", "model.layers.81.mlp.experts.12.up_proj", "model.layers.81.mlp.experts.13.up_proj", "model.layers.81.mlp.experts.14.up_proj", "model.layers.81.mlp.experts.15.up_proj", "model.layers.81.mlp.experts.16.up_proj", "model.layers.81.mlp.experts.17.up_proj", "model.layers.81.mlp.experts.18.up_proj", "model.layers.81.mlp.experts.19.up_proj", "model.layers.81.mlp.experts.20.up_proj", "model.layers.81.mlp.experts.21.up_proj", "model.layers.81.mlp.experts.22.up_proj", "model.layers.81.mlp.experts.23.up_proj", "model.layers.81.mlp.experts.24.up_proj", "model.layers.81.mlp.experts.25.up_proj", "model.layers.81.mlp.experts.26.up_proj", "model.layers.81.mlp.experts.27.up_proj", "model.layers.81.mlp.experts.28.up_proj", "model.layers.81.mlp.experts.29.up_proj", "model.layers.81.mlp.experts.30.up_proj", "model.layers.81.mlp.experts.31.up_proj", "model.layers.81.mlp.experts.32.up_proj", "model.layers.81.mlp.experts.33.up_proj", "model.layers.81.mlp.experts.34.up_proj", "model.layers.81.mlp.experts.35.up_proj", "model.layers.81.mlp.experts.36.up_proj", "model.layers.81.mlp.experts.37.up_proj", "model.layers.81.mlp.experts.38.up_proj", "model.layers.81.mlp.experts.39.up_proj", "model.layers.81.mlp.experts.40.up_proj", "model.layers.81.mlp.experts.41.up_proj", "model.layers.81.mlp.experts.42.up_proj", "model.layers.81.mlp.experts.43.up_proj", "model.layers.81.mlp.experts.44.up_proj", "model.layers.81.mlp.experts.45.up_proj", "model.layers.81.mlp.experts.46.up_proj", "model.layers.81.mlp.experts.47.up_proj", "model.layers.81.mlp.experts.48.up_proj", "model.layers.81.mlp.experts.49.up_proj", "model.layers.81.mlp.experts.50.up_proj", "model.layers.81.mlp.experts.51.up_proj", "model.layers.81.mlp.experts.52.up_proj", "model.layers.81.mlp.experts.53.up_proj", "model.layers.81.mlp.experts.54.up_proj", "model.layers.81.mlp.experts.55.up_proj", "model.layers.81.mlp.experts.56.up_proj", "model.layers.81.mlp.experts.57.up_proj", "model.layers.81.mlp.experts.58.up_proj", "model.layers.81.mlp.experts.59.up_proj", "model.layers.81.mlp.experts.60.up_proj", "model.layers.81.mlp.experts.61.up_proj", "model.layers.81.mlp.experts.62.up_proj", "model.layers.81.mlp.experts.63.up_proj", "model.layers.81.mlp.experts.64.up_proj", "model.layers.81.mlp.experts.65.up_proj", "model.layers.81.mlp.experts.66.up_proj", "model.layers.81.mlp.experts.67.up_proj", "model.layers.81.mlp.experts.68.up_proj", "model.layers.81.mlp.experts.69.up_proj", "model.layers.81.mlp.experts.70.up_proj", "model.layers.81.mlp.experts.71.up_proj", "model.layers.81.mlp.experts.72.up_proj", "model.layers.81.mlp.experts.73.up_proj", "model.layers.81.mlp.experts.74.up_proj", "model.layers.81.mlp.experts.75.up_proj", "model.layers.81.mlp.experts.76.up_proj", "model.layers.81.mlp.experts.77.up_proj", "model.layers.81.mlp.experts.78.up_proj", "model.layers.81.mlp.experts.79.up_proj", "model.layers.81.mlp.experts.80.up_proj", "model.layers.81.mlp.experts.81.up_proj", "model.layers.81.mlp.experts.82.up_proj", "model.layers.81.mlp.experts.83.up_proj", "model.layers.81.mlp.experts.84.up_proj", "model.layers.81.mlp.experts.85.up_proj", "model.layers.81.mlp.experts.86.up_proj", "model.layers.81.mlp.experts.87.up_proj", "model.layers.81.mlp.experts.88.up_proj", "model.layers.81.mlp.experts.89.up_proj", "model.layers.81.mlp.experts.90.up_proj", "model.layers.81.mlp.experts.91.up_proj", "model.layers.81.mlp.experts.92.up_proj", "model.layers.81.mlp.experts.93.up_proj", "model.layers.81.mlp.experts.94.up_proj", "model.layers.81.mlp.experts.95.up_proj", "model.layers.81.mlp.experts.96.up_proj", "model.layers.81.mlp.experts.97.up_proj", "model.layers.81.mlp.experts.98.up_proj", "model.layers.81.mlp.experts.99.up_proj", "model.layers.81.mlp.experts.100.up_proj", "model.layers.81.mlp.experts.101.up_proj", "model.layers.81.mlp.experts.102.up_proj", "model.layers.81.mlp.experts.103.up_proj", "model.layers.81.mlp.experts.104.up_proj", "model.layers.81.mlp.experts.105.up_proj", "model.layers.81.mlp.experts.106.up_proj", "model.layers.81.mlp.experts.107.up_proj", "model.layers.81.mlp.experts.108.up_proj", "model.layers.81.mlp.experts.109.up_proj", "model.layers.81.mlp.experts.110.up_proj", "model.layers.81.mlp.experts.111.up_proj", "model.layers.81.mlp.experts.112.up_proj", "model.layers.81.mlp.experts.113.up_proj", "model.layers.81.mlp.experts.114.up_proj", "model.layers.81.mlp.experts.115.up_proj", "model.layers.81.mlp.experts.116.up_proj", "model.layers.81.mlp.experts.117.up_proj", "model.layers.81.mlp.experts.118.up_proj", "model.layers.81.mlp.experts.119.up_proj", "model.layers.81.mlp.experts.120.up_proj", "model.layers.81.mlp.experts.121.up_proj", "model.layers.81.mlp.experts.122.up_proj", "model.layers.81.mlp.experts.123.up_proj", "model.layers.81.mlp.experts.124.up_proj", "model.layers.81.mlp.experts.125.up_proj", "model.layers.81.mlp.experts.126.up_proj", "model.layers.81.mlp.experts.127.up_proj", "model.layers.81.mlp.experts.128.up_proj", "model.layers.81.mlp.experts.129.up_proj", "model.layers.81.mlp.experts.130.up_proj", "model.layers.81.mlp.experts.131.up_proj", "model.layers.81.mlp.experts.132.up_proj", "model.layers.81.mlp.experts.133.up_proj", "model.layers.81.mlp.experts.134.up_proj", "model.layers.81.mlp.experts.135.up_proj", "model.layers.81.mlp.experts.136.up_proj", "model.layers.81.mlp.experts.137.up_proj", "model.layers.81.mlp.experts.138.up_proj", "model.layers.81.mlp.experts.139.up_proj", "model.layers.81.mlp.experts.140.up_proj", "model.layers.81.mlp.experts.141.up_proj", "model.layers.81.mlp.experts.142.up_proj", "model.layers.81.mlp.experts.143.up_proj", "model.layers.81.mlp.experts.144.up_proj", "model.layers.81.mlp.experts.145.up_proj", "model.layers.81.mlp.experts.146.up_proj", "model.layers.81.mlp.experts.147.up_proj", "model.layers.81.mlp.experts.148.up_proj", "model.layers.81.mlp.experts.149.up_proj", "model.layers.81.mlp.experts.150.up_proj", "model.layers.81.mlp.experts.151.up_proj", "model.layers.81.mlp.experts.152.up_proj", "model.layers.81.mlp.experts.153.up_proj", "model.layers.81.mlp.experts.154.up_proj", "model.layers.81.mlp.experts.155.up_proj", "model.layers.81.mlp.experts.156.up_proj", "model.layers.81.mlp.experts.157.up_proj", "model.layers.81.mlp.experts.158.up_proj", "model.layers.81.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00010803341865539551, "dbits": 2516582400 } ] }, { "idx": 488, "layers": [ "model.layers.81.mlp.experts.0.down_proj", "model.layers.81.mlp.experts.1.down_proj", "model.layers.81.mlp.experts.2.down_proj", "model.layers.81.mlp.experts.3.down_proj", "model.layers.81.mlp.experts.4.down_proj", "model.layers.81.mlp.experts.5.down_proj", "model.layers.81.mlp.experts.6.down_proj", "model.layers.81.mlp.experts.7.down_proj", "model.layers.81.mlp.experts.8.down_proj", "model.layers.81.mlp.experts.9.down_proj", "model.layers.81.mlp.experts.10.down_proj", "model.layers.81.mlp.experts.11.down_proj", "model.layers.81.mlp.experts.12.down_proj", "model.layers.81.mlp.experts.13.down_proj", "model.layers.81.mlp.experts.14.down_proj", "model.layers.81.mlp.experts.15.down_proj", "model.layers.81.mlp.experts.16.down_proj", "model.layers.81.mlp.experts.17.down_proj", "model.layers.81.mlp.experts.18.down_proj", "model.layers.81.mlp.experts.19.down_proj", "model.layers.81.mlp.experts.20.down_proj", "model.layers.81.mlp.experts.21.down_proj", "model.layers.81.mlp.experts.22.down_proj", "model.layers.81.mlp.experts.23.down_proj", "model.layers.81.mlp.experts.24.down_proj", "model.layers.81.mlp.experts.25.down_proj", "model.layers.81.mlp.experts.26.down_proj", "model.layers.81.mlp.experts.27.down_proj", "model.layers.81.mlp.experts.28.down_proj", "model.layers.81.mlp.experts.29.down_proj", "model.layers.81.mlp.experts.30.down_proj", "model.layers.81.mlp.experts.31.down_proj", "model.layers.81.mlp.experts.32.down_proj", "model.layers.81.mlp.experts.33.down_proj", "model.layers.81.mlp.experts.34.down_proj", "model.layers.81.mlp.experts.35.down_proj", "model.layers.81.mlp.experts.36.down_proj", "model.layers.81.mlp.experts.37.down_proj", "model.layers.81.mlp.experts.38.down_proj", "model.layers.81.mlp.experts.39.down_proj", "model.layers.81.mlp.experts.40.down_proj", "model.layers.81.mlp.experts.41.down_proj", "model.layers.81.mlp.experts.42.down_proj", "model.layers.81.mlp.experts.43.down_proj", "model.layers.81.mlp.experts.44.down_proj", "model.layers.81.mlp.experts.45.down_proj", "model.layers.81.mlp.experts.46.down_proj", "model.layers.81.mlp.experts.47.down_proj", "model.layers.81.mlp.experts.48.down_proj", "model.layers.81.mlp.experts.49.down_proj", "model.layers.81.mlp.experts.50.down_proj", "model.layers.81.mlp.experts.51.down_proj", "model.layers.81.mlp.experts.52.down_proj", "model.layers.81.mlp.experts.53.down_proj", "model.layers.81.mlp.experts.54.down_proj", "model.layers.81.mlp.experts.55.down_proj", "model.layers.81.mlp.experts.56.down_proj", "model.layers.81.mlp.experts.57.down_proj", "model.layers.81.mlp.experts.58.down_proj", "model.layers.81.mlp.experts.59.down_proj", "model.layers.81.mlp.experts.60.down_proj", "model.layers.81.mlp.experts.61.down_proj", "model.layers.81.mlp.experts.62.down_proj", "model.layers.81.mlp.experts.63.down_proj", "model.layers.81.mlp.experts.64.down_proj", "model.layers.81.mlp.experts.65.down_proj", "model.layers.81.mlp.experts.66.down_proj", "model.layers.81.mlp.experts.67.down_proj", "model.layers.81.mlp.experts.68.down_proj", "model.layers.81.mlp.experts.69.down_proj", "model.layers.81.mlp.experts.70.down_proj", "model.layers.81.mlp.experts.71.down_proj", "model.layers.81.mlp.experts.72.down_proj", "model.layers.81.mlp.experts.73.down_proj", "model.layers.81.mlp.experts.74.down_proj", "model.layers.81.mlp.experts.75.down_proj", "model.layers.81.mlp.experts.76.down_proj", "model.layers.81.mlp.experts.77.down_proj", "model.layers.81.mlp.experts.78.down_proj", "model.layers.81.mlp.experts.79.down_proj", "model.layers.81.mlp.experts.80.down_proj", "model.layers.81.mlp.experts.81.down_proj", "model.layers.81.mlp.experts.82.down_proj", "model.layers.81.mlp.experts.83.down_proj", "model.layers.81.mlp.experts.84.down_proj", "model.layers.81.mlp.experts.85.down_proj", "model.layers.81.mlp.experts.86.down_proj", "model.layers.81.mlp.experts.87.down_proj", "model.layers.81.mlp.experts.88.down_proj", "model.layers.81.mlp.experts.89.down_proj", "model.layers.81.mlp.experts.90.down_proj", "model.layers.81.mlp.experts.91.down_proj", "model.layers.81.mlp.experts.92.down_proj", "model.layers.81.mlp.experts.93.down_proj", "model.layers.81.mlp.experts.94.down_proj", "model.layers.81.mlp.experts.95.down_proj", "model.layers.81.mlp.experts.96.down_proj", "model.layers.81.mlp.experts.97.down_proj", "model.layers.81.mlp.experts.98.down_proj", "model.layers.81.mlp.experts.99.down_proj", "model.layers.81.mlp.experts.100.down_proj", "model.layers.81.mlp.experts.101.down_proj", "model.layers.81.mlp.experts.102.down_proj", "model.layers.81.mlp.experts.103.down_proj", "model.layers.81.mlp.experts.104.down_proj", "model.layers.81.mlp.experts.105.down_proj", "model.layers.81.mlp.experts.106.down_proj", "model.layers.81.mlp.experts.107.down_proj", "model.layers.81.mlp.experts.108.down_proj", "model.layers.81.mlp.experts.109.down_proj", "model.layers.81.mlp.experts.110.down_proj", "model.layers.81.mlp.experts.111.down_proj", "model.layers.81.mlp.experts.112.down_proj", "model.layers.81.mlp.experts.113.down_proj", "model.layers.81.mlp.experts.114.down_proj", "model.layers.81.mlp.experts.115.down_proj", "model.layers.81.mlp.experts.116.down_proj", "model.layers.81.mlp.experts.117.down_proj", "model.layers.81.mlp.experts.118.down_proj", "model.layers.81.mlp.experts.119.down_proj", "model.layers.81.mlp.experts.120.down_proj", "model.layers.81.mlp.experts.121.down_proj", "model.layers.81.mlp.experts.122.down_proj", "model.layers.81.mlp.experts.123.down_proj", "model.layers.81.mlp.experts.124.down_proj", "model.layers.81.mlp.experts.125.down_proj", "model.layers.81.mlp.experts.126.down_proj", "model.layers.81.mlp.experts.127.down_proj", "model.layers.81.mlp.experts.128.down_proj", "model.layers.81.mlp.experts.129.down_proj", "model.layers.81.mlp.experts.130.down_proj", "model.layers.81.mlp.experts.131.down_proj", "model.layers.81.mlp.experts.132.down_proj", "model.layers.81.mlp.experts.133.down_proj", "model.layers.81.mlp.experts.134.down_proj", "model.layers.81.mlp.experts.135.down_proj", "model.layers.81.mlp.experts.136.down_proj", "model.layers.81.mlp.experts.137.down_proj", "model.layers.81.mlp.experts.138.down_proj", "model.layers.81.mlp.experts.139.down_proj", "model.layers.81.mlp.experts.140.down_proj", "model.layers.81.mlp.experts.141.down_proj", "model.layers.81.mlp.experts.142.down_proj", "model.layers.81.mlp.experts.143.down_proj", "model.layers.81.mlp.experts.144.down_proj", "model.layers.81.mlp.experts.145.down_proj", "model.layers.81.mlp.experts.146.down_proj", "model.layers.81.mlp.experts.147.down_proj", "model.layers.81.mlp.experts.148.down_proj", "model.layers.81.mlp.experts.149.down_proj", "model.layers.81.mlp.experts.150.down_proj", "model.layers.81.mlp.experts.151.down_proj", "model.layers.81.mlp.experts.152.down_proj", "model.layers.81.mlp.experts.153.down_proj", "model.layers.81.mlp.experts.154.down_proj", "model.layers.81.mlp.experts.155.down_proj", "model.layers.81.mlp.experts.156.down_proj", "model.layers.81.mlp.experts.157.down_proj", "model.layers.81.mlp.experts.158.down_proj", "model.layers.81.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -3.534406423572056e-05, "dbits": 1258291200 } ] }, { "idx": 489, "layers": [ "model.layers.82.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0004472002387047036, "dbits": 62914560 } ] }, { "idx": 490, "layers": [ "model.layers.82.self_attn.k_proj", "model.layers.82.self_attn.v_proj" ], "candidates": [ { "dkld": -0.001215592771768581, "dbits": 10485760 } ] }, { "idx": 491, "layers": [ "model.layers.82.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0010304033756255992, "dbits": 62914560 } ] }, { "idx": 492, "layers": [ "model.layers.82.mlp.shared_experts.gate_proj", "model.layers.82.mlp.shared_experts.up_proj", "model.layers.82.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0008356787264346965, "dbits": 23592960 } ] }, { "idx": 493, "layers": [ "model.layers.82.mlp.experts.0.gate_proj", "model.layers.82.mlp.experts.1.gate_proj", "model.layers.82.mlp.experts.2.gate_proj", "model.layers.82.mlp.experts.3.gate_proj", "model.layers.82.mlp.experts.4.gate_proj", "model.layers.82.mlp.experts.5.gate_proj", "model.layers.82.mlp.experts.6.gate_proj", "model.layers.82.mlp.experts.7.gate_proj", "model.layers.82.mlp.experts.8.gate_proj", "model.layers.82.mlp.experts.9.gate_proj", "model.layers.82.mlp.experts.10.gate_proj", "model.layers.82.mlp.experts.11.gate_proj", "model.layers.82.mlp.experts.12.gate_proj", "model.layers.82.mlp.experts.13.gate_proj", "model.layers.82.mlp.experts.14.gate_proj", "model.layers.82.mlp.experts.15.gate_proj", "model.layers.82.mlp.experts.16.gate_proj", "model.layers.82.mlp.experts.17.gate_proj", "model.layers.82.mlp.experts.18.gate_proj", "model.layers.82.mlp.experts.19.gate_proj", "model.layers.82.mlp.experts.20.gate_proj", "model.layers.82.mlp.experts.21.gate_proj", "model.layers.82.mlp.experts.22.gate_proj", "model.layers.82.mlp.experts.23.gate_proj", "model.layers.82.mlp.experts.24.gate_proj", "model.layers.82.mlp.experts.25.gate_proj", "model.layers.82.mlp.experts.26.gate_proj", "model.layers.82.mlp.experts.27.gate_proj", "model.layers.82.mlp.experts.28.gate_proj", "model.layers.82.mlp.experts.29.gate_proj", "model.layers.82.mlp.experts.30.gate_proj", "model.layers.82.mlp.experts.31.gate_proj", "model.layers.82.mlp.experts.32.gate_proj", "model.layers.82.mlp.experts.33.gate_proj", "model.layers.82.mlp.experts.34.gate_proj", "model.layers.82.mlp.experts.35.gate_proj", "model.layers.82.mlp.experts.36.gate_proj", "model.layers.82.mlp.experts.37.gate_proj", "model.layers.82.mlp.experts.38.gate_proj", "model.layers.82.mlp.experts.39.gate_proj", "model.layers.82.mlp.experts.40.gate_proj", "model.layers.82.mlp.experts.41.gate_proj", "model.layers.82.mlp.experts.42.gate_proj", "model.layers.82.mlp.experts.43.gate_proj", "model.layers.82.mlp.experts.44.gate_proj", "model.layers.82.mlp.experts.45.gate_proj", "model.layers.82.mlp.experts.46.gate_proj", "model.layers.82.mlp.experts.47.gate_proj", "model.layers.82.mlp.experts.48.gate_proj", "model.layers.82.mlp.experts.49.gate_proj", "model.layers.82.mlp.experts.50.gate_proj", "model.layers.82.mlp.experts.51.gate_proj", "model.layers.82.mlp.experts.52.gate_proj", "model.layers.82.mlp.experts.53.gate_proj", "model.layers.82.mlp.experts.54.gate_proj", "model.layers.82.mlp.experts.55.gate_proj", "model.layers.82.mlp.experts.56.gate_proj", "model.layers.82.mlp.experts.57.gate_proj", "model.layers.82.mlp.experts.58.gate_proj", "model.layers.82.mlp.experts.59.gate_proj", "model.layers.82.mlp.experts.60.gate_proj", "model.layers.82.mlp.experts.61.gate_proj", "model.layers.82.mlp.experts.62.gate_proj", "model.layers.82.mlp.experts.63.gate_proj", "model.layers.82.mlp.experts.64.gate_proj", "model.layers.82.mlp.experts.65.gate_proj", "model.layers.82.mlp.experts.66.gate_proj", "model.layers.82.mlp.experts.67.gate_proj", "model.layers.82.mlp.experts.68.gate_proj", "model.layers.82.mlp.experts.69.gate_proj", "model.layers.82.mlp.experts.70.gate_proj", "model.layers.82.mlp.experts.71.gate_proj", "model.layers.82.mlp.experts.72.gate_proj", "model.layers.82.mlp.experts.73.gate_proj", "model.layers.82.mlp.experts.74.gate_proj", "model.layers.82.mlp.experts.75.gate_proj", "model.layers.82.mlp.experts.76.gate_proj", "model.layers.82.mlp.experts.77.gate_proj", "model.layers.82.mlp.experts.78.gate_proj", "model.layers.82.mlp.experts.79.gate_proj", "model.layers.82.mlp.experts.80.gate_proj", "model.layers.82.mlp.experts.81.gate_proj", "model.layers.82.mlp.experts.82.gate_proj", "model.layers.82.mlp.experts.83.gate_proj", "model.layers.82.mlp.experts.84.gate_proj", "model.layers.82.mlp.experts.85.gate_proj", "model.layers.82.mlp.experts.86.gate_proj", "model.layers.82.mlp.experts.87.gate_proj", "model.layers.82.mlp.experts.88.gate_proj", "model.layers.82.mlp.experts.89.gate_proj", "model.layers.82.mlp.experts.90.gate_proj", "model.layers.82.mlp.experts.91.gate_proj", "model.layers.82.mlp.experts.92.gate_proj", "model.layers.82.mlp.experts.93.gate_proj", "model.layers.82.mlp.experts.94.gate_proj", "model.layers.82.mlp.experts.95.gate_proj", "model.layers.82.mlp.experts.96.gate_proj", "model.layers.82.mlp.experts.97.gate_proj", "model.layers.82.mlp.experts.98.gate_proj", "model.layers.82.mlp.experts.99.gate_proj", "model.layers.82.mlp.experts.100.gate_proj", "model.layers.82.mlp.experts.101.gate_proj", "model.layers.82.mlp.experts.102.gate_proj", "model.layers.82.mlp.experts.103.gate_proj", "model.layers.82.mlp.experts.104.gate_proj", "model.layers.82.mlp.experts.105.gate_proj", "model.layers.82.mlp.experts.106.gate_proj", "model.layers.82.mlp.experts.107.gate_proj", "model.layers.82.mlp.experts.108.gate_proj", "model.layers.82.mlp.experts.109.gate_proj", "model.layers.82.mlp.experts.110.gate_proj", "model.layers.82.mlp.experts.111.gate_proj", "model.layers.82.mlp.experts.112.gate_proj", "model.layers.82.mlp.experts.113.gate_proj", "model.layers.82.mlp.experts.114.gate_proj", "model.layers.82.mlp.experts.115.gate_proj", "model.layers.82.mlp.experts.116.gate_proj", "model.layers.82.mlp.experts.117.gate_proj", "model.layers.82.mlp.experts.118.gate_proj", "model.layers.82.mlp.experts.119.gate_proj", "model.layers.82.mlp.experts.120.gate_proj", "model.layers.82.mlp.experts.121.gate_proj", "model.layers.82.mlp.experts.122.gate_proj", "model.layers.82.mlp.experts.123.gate_proj", "model.layers.82.mlp.experts.124.gate_proj", "model.layers.82.mlp.experts.125.gate_proj", "model.layers.82.mlp.experts.126.gate_proj", "model.layers.82.mlp.experts.127.gate_proj", "model.layers.82.mlp.experts.128.gate_proj", "model.layers.82.mlp.experts.129.gate_proj", "model.layers.82.mlp.experts.130.gate_proj", "model.layers.82.mlp.experts.131.gate_proj", "model.layers.82.mlp.experts.132.gate_proj", "model.layers.82.mlp.experts.133.gate_proj", "model.layers.82.mlp.experts.134.gate_proj", "model.layers.82.mlp.experts.135.gate_proj", "model.layers.82.mlp.experts.136.gate_proj", "model.layers.82.mlp.experts.137.gate_proj", "model.layers.82.mlp.experts.138.gate_proj", "model.layers.82.mlp.experts.139.gate_proj", "model.layers.82.mlp.experts.140.gate_proj", "model.layers.82.mlp.experts.141.gate_proj", "model.layers.82.mlp.experts.142.gate_proj", "model.layers.82.mlp.experts.143.gate_proj", "model.layers.82.mlp.experts.144.gate_proj", "model.layers.82.mlp.experts.145.gate_proj", "model.layers.82.mlp.experts.146.gate_proj", "model.layers.82.mlp.experts.147.gate_proj", "model.layers.82.mlp.experts.148.gate_proj", "model.layers.82.mlp.experts.149.gate_proj", "model.layers.82.mlp.experts.150.gate_proj", "model.layers.82.mlp.experts.151.gate_proj", "model.layers.82.mlp.experts.152.gate_proj", "model.layers.82.mlp.experts.153.gate_proj", "model.layers.82.mlp.experts.154.gate_proj", "model.layers.82.mlp.experts.155.gate_proj", "model.layers.82.mlp.experts.156.gate_proj", "model.layers.82.mlp.experts.157.gate_proj", "model.layers.82.mlp.experts.158.gate_proj", "model.layers.82.mlp.experts.159.gate_proj", "model.layers.82.mlp.experts.0.up_proj", "model.layers.82.mlp.experts.1.up_proj", "model.layers.82.mlp.experts.2.up_proj", "model.layers.82.mlp.experts.3.up_proj", "model.layers.82.mlp.experts.4.up_proj", "model.layers.82.mlp.experts.5.up_proj", "model.layers.82.mlp.experts.6.up_proj", "model.layers.82.mlp.experts.7.up_proj", "model.layers.82.mlp.experts.8.up_proj", "model.layers.82.mlp.experts.9.up_proj", "model.layers.82.mlp.experts.10.up_proj", "model.layers.82.mlp.experts.11.up_proj", "model.layers.82.mlp.experts.12.up_proj", "model.layers.82.mlp.experts.13.up_proj", "model.layers.82.mlp.experts.14.up_proj", "model.layers.82.mlp.experts.15.up_proj", "model.layers.82.mlp.experts.16.up_proj", "model.layers.82.mlp.experts.17.up_proj", "model.layers.82.mlp.experts.18.up_proj", "model.layers.82.mlp.experts.19.up_proj", "model.layers.82.mlp.experts.20.up_proj", "model.layers.82.mlp.experts.21.up_proj", "model.layers.82.mlp.experts.22.up_proj", "model.layers.82.mlp.experts.23.up_proj", "model.layers.82.mlp.experts.24.up_proj", "model.layers.82.mlp.experts.25.up_proj", "model.layers.82.mlp.experts.26.up_proj", "model.layers.82.mlp.experts.27.up_proj", "model.layers.82.mlp.experts.28.up_proj", "model.layers.82.mlp.experts.29.up_proj", "model.layers.82.mlp.experts.30.up_proj", "model.layers.82.mlp.experts.31.up_proj", "model.layers.82.mlp.experts.32.up_proj", "model.layers.82.mlp.experts.33.up_proj", "model.layers.82.mlp.experts.34.up_proj", "model.layers.82.mlp.experts.35.up_proj", "model.layers.82.mlp.experts.36.up_proj", "model.layers.82.mlp.experts.37.up_proj", "model.layers.82.mlp.experts.38.up_proj", "model.layers.82.mlp.experts.39.up_proj", "model.layers.82.mlp.experts.40.up_proj", "model.layers.82.mlp.experts.41.up_proj", "model.layers.82.mlp.experts.42.up_proj", "model.layers.82.mlp.experts.43.up_proj", "model.layers.82.mlp.experts.44.up_proj", "model.layers.82.mlp.experts.45.up_proj", "model.layers.82.mlp.experts.46.up_proj", "model.layers.82.mlp.experts.47.up_proj", "model.layers.82.mlp.experts.48.up_proj", "model.layers.82.mlp.experts.49.up_proj", "model.layers.82.mlp.experts.50.up_proj", "model.layers.82.mlp.experts.51.up_proj", "model.layers.82.mlp.experts.52.up_proj", "model.layers.82.mlp.experts.53.up_proj", "model.layers.82.mlp.experts.54.up_proj", "model.layers.82.mlp.experts.55.up_proj", "model.layers.82.mlp.experts.56.up_proj", "model.layers.82.mlp.experts.57.up_proj", "model.layers.82.mlp.experts.58.up_proj", "model.layers.82.mlp.experts.59.up_proj", "model.layers.82.mlp.experts.60.up_proj", "model.layers.82.mlp.experts.61.up_proj", "model.layers.82.mlp.experts.62.up_proj", "model.layers.82.mlp.experts.63.up_proj", "model.layers.82.mlp.experts.64.up_proj", "model.layers.82.mlp.experts.65.up_proj", "model.layers.82.mlp.experts.66.up_proj", "model.layers.82.mlp.experts.67.up_proj", "model.layers.82.mlp.experts.68.up_proj", "model.layers.82.mlp.experts.69.up_proj", "model.layers.82.mlp.experts.70.up_proj", "model.layers.82.mlp.experts.71.up_proj", "model.layers.82.mlp.experts.72.up_proj", "model.layers.82.mlp.experts.73.up_proj", "model.layers.82.mlp.experts.74.up_proj", "model.layers.82.mlp.experts.75.up_proj", "model.layers.82.mlp.experts.76.up_proj", "model.layers.82.mlp.experts.77.up_proj", "model.layers.82.mlp.experts.78.up_proj", "model.layers.82.mlp.experts.79.up_proj", "model.layers.82.mlp.experts.80.up_proj", "model.layers.82.mlp.experts.81.up_proj", "model.layers.82.mlp.experts.82.up_proj", "model.layers.82.mlp.experts.83.up_proj", "model.layers.82.mlp.experts.84.up_proj", "model.layers.82.mlp.experts.85.up_proj", "model.layers.82.mlp.experts.86.up_proj", "model.layers.82.mlp.experts.87.up_proj", "model.layers.82.mlp.experts.88.up_proj", "model.layers.82.mlp.experts.89.up_proj", "model.layers.82.mlp.experts.90.up_proj", "model.layers.82.mlp.experts.91.up_proj", "model.layers.82.mlp.experts.92.up_proj", "model.layers.82.mlp.experts.93.up_proj", "model.layers.82.mlp.experts.94.up_proj", "model.layers.82.mlp.experts.95.up_proj", "model.layers.82.mlp.experts.96.up_proj", "model.layers.82.mlp.experts.97.up_proj", "model.layers.82.mlp.experts.98.up_proj", "model.layers.82.mlp.experts.99.up_proj", "model.layers.82.mlp.experts.100.up_proj", "model.layers.82.mlp.experts.101.up_proj", "model.layers.82.mlp.experts.102.up_proj", "model.layers.82.mlp.experts.103.up_proj", "model.layers.82.mlp.experts.104.up_proj", "model.layers.82.mlp.experts.105.up_proj", "model.layers.82.mlp.experts.106.up_proj", "model.layers.82.mlp.experts.107.up_proj", "model.layers.82.mlp.experts.108.up_proj", "model.layers.82.mlp.experts.109.up_proj", "model.layers.82.mlp.experts.110.up_proj", "model.layers.82.mlp.experts.111.up_proj", "model.layers.82.mlp.experts.112.up_proj", "model.layers.82.mlp.experts.113.up_proj", "model.layers.82.mlp.experts.114.up_proj", "model.layers.82.mlp.experts.115.up_proj", "model.layers.82.mlp.experts.116.up_proj", "model.layers.82.mlp.experts.117.up_proj", "model.layers.82.mlp.experts.118.up_proj", "model.layers.82.mlp.experts.119.up_proj", "model.layers.82.mlp.experts.120.up_proj", "model.layers.82.mlp.experts.121.up_proj", "model.layers.82.mlp.experts.122.up_proj", "model.layers.82.mlp.experts.123.up_proj", "model.layers.82.mlp.experts.124.up_proj", "model.layers.82.mlp.experts.125.up_proj", "model.layers.82.mlp.experts.126.up_proj", "model.layers.82.mlp.experts.127.up_proj", "model.layers.82.mlp.experts.128.up_proj", "model.layers.82.mlp.experts.129.up_proj", "model.layers.82.mlp.experts.130.up_proj", "model.layers.82.mlp.experts.131.up_proj", "model.layers.82.mlp.experts.132.up_proj", "model.layers.82.mlp.experts.133.up_proj", "model.layers.82.mlp.experts.134.up_proj", "model.layers.82.mlp.experts.135.up_proj", "model.layers.82.mlp.experts.136.up_proj", "model.layers.82.mlp.experts.137.up_proj", "model.layers.82.mlp.experts.138.up_proj", "model.layers.82.mlp.experts.139.up_proj", "model.layers.82.mlp.experts.140.up_proj", "model.layers.82.mlp.experts.141.up_proj", "model.layers.82.mlp.experts.142.up_proj", "model.layers.82.mlp.experts.143.up_proj", "model.layers.82.mlp.experts.144.up_proj", "model.layers.82.mlp.experts.145.up_proj", "model.layers.82.mlp.experts.146.up_proj", "model.layers.82.mlp.experts.147.up_proj", "model.layers.82.mlp.experts.148.up_proj", "model.layers.82.mlp.experts.149.up_proj", "model.layers.82.mlp.experts.150.up_proj", "model.layers.82.mlp.experts.151.up_proj", "model.layers.82.mlp.experts.152.up_proj", "model.layers.82.mlp.experts.153.up_proj", "model.layers.82.mlp.experts.154.up_proj", "model.layers.82.mlp.experts.155.up_proj", "model.layers.82.mlp.experts.156.up_proj", "model.layers.82.mlp.experts.157.up_proj", "model.layers.82.mlp.experts.158.up_proj", "model.layers.82.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.000916718691587437, "dbits": 2516582400 } ] }, { "idx": 494, "layers": [ "model.layers.82.mlp.experts.0.down_proj", "model.layers.82.mlp.experts.1.down_proj", "model.layers.82.mlp.experts.2.down_proj", "model.layers.82.mlp.experts.3.down_proj", "model.layers.82.mlp.experts.4.down_proj", "model.layers.82.mlp.experts.5.down_proj", "model.layers.82.mlp.experts.6.down_proj", "model.layers.82.mlp.experts.7.down_proj", "model.layers.82.mlp.experts.8.down_proj", "model.layers.82.mlp.experts.9.down_proj", "model.layers.82.mlp.experts.10.down_proj", "model.layers.82.mlp.experts.11.down_proj", "model.layers.82.mlp.experts.12.down_proj", "model.layers.82.mlp.experts.13.down_proj", "model.layers.82.mlp.experts.14.down_proj", "model.layers.82.mlp.experts.15.down_proj", "model.layers.82.mlp.experts.16.down_proj", "model.layers.82.mlp.experts.17.down_proj", "model.layers.82.mlp.experts.18.down_proj", "model.layers.82.mlp.experts.19.down_proj", "model.layers.82.mlp.experts.20.down_proj", "model.layers.82.mlp.experts.21.down_proj", "model.layers.82.mlp.experts.22.down_proj", "model.layers.82.mlp.experts.23.down_proj", "model.layers.82.mlp.experts.24.down_proj", "model.layers.82.mlp.experts.25.down_proj", "model.layers.82.mlp.experts.26.down_proj", "model.layers.82.mlp.experts.27.down_proj", "model.layers.82.mlp.experts.28.down_proj", "model.layers.82.mlp.experts.29.down_proj", "model.layers.82.mlp.experts.30.down_proj", "model.layers.82.mlp.experts.31.down_proj", "model.layers.82.mlp.experts.32.down_proj", "model.layers.82.mlp.experts.33.down_proj", "model.layers.82.mlp.experts.34.down_proj", "model.layers.82.mlp.experts.35.down_proj", "model.layers.82.mlp.experts.36.down_proj", "model.layers.82.mlp.experts.37.down_proj", "model.layers.82.mlp.experts.38.down_proj", "model.layers.82.mlp.experts.39.down_proj", "model.layers.82.mlp.experts.40.down_proj", "model.layers.82.mlp.experts.41.down_proj", "model.layers.82.mlp.experts.42.down_proj", "model.layers.82.mlp.experts.43.down_proj", "model.layers.82.mlp.experts.44.down_proj", "model.layers.82.mlp.experts.45.down_proj", "model.layers.82.mlp.experts.46.down_proj", "model.layers.82.mlp.experts.47.down_proj", "model.layers.82.mlp.experts.48.down_proj", "model.layers.82.mlp.experts.49.down_proj", "model.layers.82.mlp.experts.50.down_proj", "model.layers.82.mlp.experts.51.down_proj", "model.layers.82.mlp.experts.52.down_proj", "model.layers.82.mlp.experts.53.down_proj", "model.layers.82.mlp.experts.54.down_proj", "model.layers.82.mlp.experts.55.down_proj", "model.layers.82.mlp.experts.56.down_proj", "model.layers.82.mlp.experts.57.down_proj", "model.layers.82.mlp.experts.58.down_proj", "model.layers.82.mlp.experts.59.down_proj", "model.layers.82.mlp.experts.60.down_proj", "model.layers.82.mlp.experts.61.down_proj", "model.layers.82.mlp.experts.62.down_proj", "model.layers.82.mlp.experts.63.down_proj", "model.layers.82.mlp.experts.64.down_proj", "model.layers.82.mlp.experts.65.down_proj", "model.layers.82.mlp.experts.66.down_proj", "model.layers.82.mlp.experts.67.down_proj", "model.layers.82.mlp.experts.68.down_proj", "model.layers.82.mlp.experts.69.down_proj", "model.layers.82.mlp.experts.70.down_proj", "model.layers.82.mlp.experts.71.down_proj", "model.layers.82.mlp.experts.72.down_proj", "model.layers.82.mlp.experts.73.down_proj", "model.layers.82.mlp.experts.74.down_proj", "model.layers.82.mlp.experts.75.down_proj", "model.layers.82.mlp.experts.76.down_proj", "model.layers.82.mlp.experts.77.down_proj", "model.layers.82.mlp.experts.78.down_proj", "model.layers.82.mlp.experts.79.down_proj", "model.layers.82.mlp.experts.80.down_proj", "model.layers.82.mlp.experts.81.down_proj", "model.layers.82.mlp.experts.82.down_proj", "model.layers.82.mlp.experts.83.down_proj", "model.layers.82.mlp.experts.84.down_proj", "model.layers.82.mlp.experts.85.down_proj", "model.layers.82.mlp.experts.86.down_proj", "model.layers.82.mlp.experts.87.down_proj", "model.layers.82.mlp.experts.88.down_proj", "model.layers.82.mlp.experts.89.down_proj", "model.layers.82.mlp.experts.90.down_proj", "model.layers.82.mlp.experts.91.down_proj", "model.layers.82.mlp.experts.92.down_proj", "model.layers.82.mlp.experts.93.down_proj", "model.layers.82.mlp.experts.94.down_proj", "model.layers.82.mlp.experts.95.down_proj", "model.layers.82.mlp.experts.96.down_proj", "model.layers.82.mlp.experts.97.down_proj", "model.layers.82.mlp.experts.98.down_proj", "model.layers.82.mlp.experts.99.down_proj", "model.layers.82.mlp.experts.100.down_proj", "model.layers.82.mlp.experts.101.down_proj", "model.layers.82.mlp.experts.102.down_proj", "model.layers.82.mlp.experts.103.down_proj", "model.layers.82.mlp.experts.104.down_proj", "model.layers.82.mlp.experts.105.down_proj", "model.layers.82.mlp.experts.106.down_proj", "model.layers.82.mlp.experts.107.down_proj", "model.layers.82.mlp.experts.108.down_proj", "model.layers.82.mlp.experts.109.down_proj", "model.layers.82.mlp.experts.110.down_proj", "model.layers.82.mlp.experts.111.down_proj", "model.layers.82.mlp.experts.112.down_proj", "model.layers.82.mlp.experts.113.down_proj", "model.layers.82.mlp.experts.114.down_proj", "model.layers.82.mlp.experts.115.down_proj", "model.layers.82.mlp.experts.116.down_proj", "model.layers.82.mlp.experts.117.down_proj", "model.layers.82.mlp.experts.118.down_proj", "model.layers.82.mlp.experts.119.down_proj", "model.layers.82.mlp.experts.120.down_proj", "model.layers.82.mlp.experts.121.down_proj", "model.layers.82.mlp.experts.122.down_proj", "model.layers.82.mlp.experts.123.down_proj", "model.layers.82.mlp.experts.124.down_proj", "model.layers.82.mlp.experts.125.down_proj", "model.layers.82.mlp.experts.126.down_proj", "model.layers.82.mlp.experts.127.down_proj", "model.layers.82.mlp.experts.128.down_proj", "model.layers.82.mlp.experts.129.down_proj", "model.layers.82.mlp.experts.130.down_proj", "model.layers.82.mlp.experts.131.down_proj", "model.layers.82.mlp.experts.132.down_proj", "model.layers.82.mlp.experts.133.down_proj", "model.layers.82.mlp.experts.134.down_proj", "model.layers.82.mlp.experts.135.down_proj", "model.layers.82.mlp.experts.136.down_proj", "model.layers.82.mlp.experts.137.down_proj", "model.layers.82.mlp.experts.138.down_proj", "model.layers.82.mlp.experts.139.down_proj", "model.layers.82.mlp.experts.140.down_proj", "model.layers.82.mlp.experts.141.down_proj", "model.layers.82.mlp.experts.142.down_proj", "model.layers.82.mlp.experts.143.down_proj", "model.layers.82.mlp.experts.144.down_proj", "model.layers.82.mlp.experts.145.down_proj", "model.layers.82.mlp.experts.146.down_proj", "model.layers.82.mlp.experts.147.down_proj", "model.layers.82.mlp.experts.148.down_proj", "model.layers.82.mlp.experts.149.down_proj", "model.layers.82.mlp.experts.150.down_proj", "model.layers.82.mlp.experts.151.down_proj", "model.layers.82.mlp.experts.152.down_proj", "model.layers.82.mlp.experts.153.down_proj", "model.layers.82.mlp.experts.154.down_proj", "model.layers.82.mlp.experts.155.down_proj", "model.layers.82.mlp.experts.156.down_proj", "model.layers.82.mlp.experts.157.down_proj", "model.layers.82.mlp.experts.158.down_proj", "model.layers.82.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0002566717565059551, "dbits": 1258291200 } ] }, { "idx": 495, "layers": [ "model.layers.83.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0009335704147815926, "dbits": 62914560 } ] }, { "idx": 496, "layers": [ "model.layers.83.self_attn.k_proj", "model.layers.83.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0013775855302810558, "dbits": 10485760 } ] }, { "idx": 497, "layers": [ "model.layers.83.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00027591437101365246, "dbits": 62914560 } ] }, { "idx": 498, "layers": [ "model.layers.83.mlp.shared_experts.gate_proj", "model.layers.83.mlp.shared_experts.up_proj", "model.layers.83.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00012225285172462463, "dbits": 23592960 } ] }, { "idx": 499, "layers": [ "model.layers.83.mlp.experts.0.gate_proj", "model.layers.83.mlp.experts.1.gate_proj", "model.layers.83.mlp.experts.2.gate_proj", "model.layers.83.mlp.experts.3.gate_proj", "model.layers.83.mlp.experts.4.gate_proj", "model.layers.83.mlp.experts.5.gate_proj", "model.layers.83.mlp.experts.6.gate_proj", "model.layers.83.mlp.experts.7.gate_proj", "model.layers.83.mlp.experts.8.gate_proj", "model.layers.83.mlp.experts.9.gate_proj", "model.layers.83.mlp.experts.10.gate_proj", "model.layers.83.mlp.experts.11.gate_proj", "model.layers.83.mlp.experts.12.gate_proj", "model.layers.83.mlp.experts.13.gate_proj", "model.layers.83.mlp.experts.14.gate_proj", "model.layers.83.mlp.experts.15.gate_proj", "model.layers.83.mlp.experts.16.gate_proj", "model.layers.83.mlp.experts.17.gate_proj", "model.layers.83.mlp.experts.18.gate_proj", "model.layers.83.mlp.experts.19.gate_proj", "model.layers.83.mlp.experts.20.gate_proj", "model.layers.83.mlp.experts.21.gate_proj", "model.layers.83.mlp.experts.22.gate_proj", "model.layers.83.mlp.experts.23.gate_proj", "model.layers.83.mlp.experts.24.gate_proj", "model.layers.83.mlp.experts.25.gate_proj", "model.layers.83.mlp.experts.26.gate_proj", "model.layers.83.mlp.experts.27.gate_proj", "model.layers.83.mlp.experts.28.gate_proj", "model.layers.83.mlp.experts.29.gate_proj", "model.layers.83.mlp.experts.30.gate_proj", "model.layers.83.mlp.experts.31.gate_proj", "model.layers.83.mlp.experts.32.gate_proj", "model.layers.83.mlp.experts.33.gate_proj", "model.layers.83.mlp.experts.34.gate_proj", "model.layers.83.mlp.experts.35.gate_proj", "model.layers.83.mlp.experts.36.gate_proj", "model.layers.83.mlp.experts.37.gate_proj", "model.layers.83.mlp.experts.38.gate_proj", "model.layers.83.mlp.experts.39.gate_proj", "model.layers.83.mlp.experts.40.gate_proj", "model.layers.83.mlp.experts.41.gate_proj", "model.layers.83.mlp.experts.42.gate_proj", "model.layers.83.mlp.experts.43.gate_proj", "model.layers.83.mlp.experts.44.gate_proj", "model.layers.83.mlp.experts.45.gate_proj", "model.layers.83.mlp.experts.46.gate_proj", "model.layers.83.mlp.experts.47.gate_proj", "model.layers.83.mlp.experts.48.gate_proj", "model.layers.83.mlp.experts.49.gate_proj", "model.layers.83.mlp.experts.50.gate_proj", "model.layers.83.mlp.experts.51.gate_proj", "model.layers.83.mlp.experts.52.gate_proj", "model.layers.83.mlp.experts.53.gate_proj", "model.layers.83.mlp.experts.54.gate_proj", "model.layers.83.mlp.experts.55.gate_proj", "model.layers.83.mlp.experts.56.gate_proj", "model.layers.83.mlp.experts.57.gate_proj", "model.layers.83.mlp.experts.58.gate_proj", "model.layers.83.mlp.experts.59.gate_proj", "model.layers.83.mlp.experts.60.gate_proj", "model.layers.83.mlp.experts.61.gate_proj", "model.layers.83.mlp.experts.62.gate_proj", "model.layers.83.mlp.experts.63.gate_proj", "model.layers.83.mlp.experts.64.gate_proj", "model.layers.83.mlp.experts.65.gate_proj", "model.layers.83.mlp.experts.66.gate_proj", "model.layers.83.mlp.experts.67.gate_proj", "model.layers.83.mlp.experts.68.gate_proj", "model.layers.83.mlp.experts.69.gate_proj", "model.layers.83.mlp.experts.70.gate_proj", "model.layers.83.mlp.experts.71.gate_proj", "model.layers.83.mlp.experts.72.gate_proj", "model.layers.83.mlp.experts.73.gate_proj", "model.layers.83.mlp.experts.74.gate_proj", "model.layers.83.mlp.experts.75.gate_proj", "model.layers.83.mlp.experts.76.gate_proj", "model.layers.83.mlp.experts.77.gate_proj", "model.layers.83.mlp.experts.78.gate_proj", "model.layers.83.mlp.experts.79.gate_proj", "model.layers.83.mlp.experts.80.gate_proj", "model.layers.83.mlp.experts.81.gate_proj", "model.layers.83.mlp.experts.82.gate_proj", "model.layers.83.mlp.experts.83.gate_proj", "model.layers.83.mlp.experts.84.gate_proj", "model.layers.83.mlp.experts.85.gate_proj", "model.layers.83.mlp.experts.86.gate_proj", "model.layers.83.mlp.experts.87.gate_proj", "model.layers.83.mlp.experts.88.gate_proj", "model.layers.83.mlp.experts.89.gate_proj", "model.layers.83.mlp.experts.90.gate_proj", "model.layers.83.mlp.experts.91.gate_proj", "model.layers.83.mlp.experts.92.gate_proj", "model.layers.83.mlp.experts.93.gate_proj", "model.layers.83.mlp.experts.94.gate_proj", "model.layers.83.mlp.experts.95.gate_proj", "model.layers.83.mlp.experts.96.gate_proj", "model.layers.83.mlp.experts.97.gate_proj", "model.layers.83.mlp.experts.98.gate_proj", "model.layers.83.mlp.experts.99.gate_proj", "model.layers.83.mlp.experts.100.gate_proj", "model.layers.83.mlp.experts.101.gate_proj", "model.layers.83.mlp.experts.102.gate_proj", "model.layers.83.mlp.experts.103.gate_proj", "model.layers.83.mlp.experts.104.gate_proj", "model.layers.83.mlp.experts.105.gate_proj", "model.layers.83.mlp.experts.106.gate_proj", "model.layers.83.mlp.experts.107.gate_proj", "model.layers.83.mlp.experts.108.gate_proj", "model.layers.83.mlp.experts.109.gate_proj", "model.layers.83.mlp.experts.110.gate_proj", "model.layers.83.mlp.experts.111.gate_proj", "model.layers.83.mlp.experts.112.gate_proj", "model.layers.83.mlp.experts.113.gate_proj", "model.layers.83.mlp.experts.114.gate_proj", "model.layers.83.mlp.experts.115.gate_proj", "model.layers.83.mlp.experts.116.gate_proj", "model.layers.83.mlp.experts.117.gate_proj", "model.layers.83.mlp.experts.118.gate_proj", "model.layers.83.mlp.experts.119.gate_proj", "model.layers.83.mlp.experts.120.gate_proj", "model.layers.83.mlp.experts.121.gate_proj", "model.layers.83.mlp.experts.122.gate_proj", "model.layers.83.mlp.experts.123.gate_proj", "model.layers.83.mlp.experts.124.gate_proj", "model.layers.83.mlp.experts.125.gate_proj", "model.layers.83.mlp.experts.126.gate_proj", "model.layers.83.mlp.experts.127.gate_proj", "model.layers.83.mlp.experts.128.gate_proj", "model.layers.83.mlp.experts.129.gate_proj", "model.layers.83.mlp.experts.130.gate_proj", "model.layers.83.mlp.experts.131.gate_proj", "model.layers.83.mlp.experts.132.gate_proj", "model.layers.83.mlp.experts.133.gate_proj", "model.layers.83.mlp.experts.134.gate_proj", "model.layers.83.mlp.experts.135.gate_proj", "model.layers.83.mlp.experts.136.gate_proj", "model.layers.83.mlp.experts.137.gate_proj", "model.layers.83.mlp.experts.138.gate_proj", "model.layers.83.mlp.experts.139.gate_proj", "model.layers.83.mlp.experts.140.gate_proj", "model.layers.83.mlp.experts.141.gate_proj", "model.layers.83.mlp.experts.142.gate_proj", "model.layers.83.mlp.experts.143.gate_proj", "model.layers.83.mlp.experts.144.gate_proj", "model.layers.83.mlp.experts.145.gate_proj", "model.layers.83.mlp.experts.146.gate_proj", "model.layers.83.mlp.experts.147.gate_proj", "model.layers.83.mlp.experts.148.gate_proj", "model.layers.83.mlp.experts.149.gate_proj", "model.layers.83.mlp.experts.150.gate_proj", "model.layers.83.mlp.experts.151.gate_proj", "model.layers.83.mlp.experts.152.gate_proj", "model.layers.83.mlp.experts.153.gate_proj", "model.layers.83.mlp.experts.154.gate_proj", "model.layers.83.mlp.experts.155.gate_proj", "model.layers.83.mlp.experts.156.gate_proj", "model.layers.83.mlp.experts.157.gate_proj", "model.layers.83.mlp.experts.158.gate_proj", "model.layers.83.mlp.experts.159.gate_proj", "model.layers.83.mlp.experts.0.up_proj", "model.layers.83.mlp.experts.1.up_proj", "model.layers.83.mlp.experts.2.up_proj", "model.layers.83.mlp.experts.3.up_proj", "model.layers.83.mlp.experts.4.up_proj", "model.layers.83.mlp.experts.5.up_proj", "model.layers.83.mlp.experts.6.up_proj", "model.layers.83.mlp.experts.7.up_proj", "model.layers.83.mlp.experts.8.up_proj", "model.layers.83.mlp.experts.9.up_proj", "model.layers.83.mlp.experts.10.up_proj", "model.layers.83.mlp.experts.11.up_proj", "model.layers.83.mlp.experts.12.up_proj", "model.layers.83.mlp.experts.13.up_proj", "model.layers.83.mlp.experts.14.up_proj", "model.layers.83.mlp.experts.15.up_proj", "model.layers.83.mlp.experts.16.up_proj", "model.layers.83.mlp.experts.17.up_proj", "model.layers.83.mlp.experts.18.up_proj", "model.layers.83.mlp.experts.19.up_proj", "model.layers.83.mlp.experts.20.up_proj", "model.layers.83.mlp.experts.21.up_proj", "model.layers.83.mlp.experts.22.up_proj", "model.layers.83.mlp.experts.23.up_proj", "model.layers.83.mlp.experts.24.up_proj", "model.layers.83.mlp.experts.25.up_proj", "model.layers.83.mlp.experts.26.up_proj", "model.layers.83.mlp.experts.27.up_proj", "model.layers.83.mlp.experts.28.up_proj", "model.layers.83.mlp.experts.29.up_proj", "model.layers.83.mlp.experts.30.up_proj", "model.layers.83.mlp.experts.31.up_proj", "model.layers.83.mlp.experts.32.up_proj", "model.layers.83.mlp.experts.33.up_proj", "model.layers.83.mlp.experts.34.up_proj", "model.layers.83.mlp.experts.35.up_proj", "model.layers.83.mlp.experts.36.up_proj", "model.layers.83.mlp.experts.37.up_proj", "model.layers.83.mlp.experts.38.up_proj", "model.layers.83.mlp.experts.39.up_proj", "model.layers.83.mlp.experts.40.up_proj", "model.layers.83.mlp.experts.41.up_proj", "model.layers.83.mlp.experts.42.up_proj", "model.layers.83.mlp.experts.43.up_proj", "model.layers.83.mlp.experts.44.up_proj", "model.layers.83.mlp.experts.45.up_proj", "model.layers.83.mlp.experts.46.up_proj", "model.layers.83.mlp.experts.47.up_proj", "model.layers.83.mlp.experts.48.up_proj", "model.layers.83.mlp.experts.49.up_proj", "model.layers.83.mlp.experts.50.up_proj", "model.layers.83.mlp.experts.51.up_proj", "model.layers.83.mlp.experts.52.up_proj", "model.layers.83.mlp.experts.53.up_proj", "model.layers.83.mlp.experts.54.up_proj", "model.layers.83.mlp.experts.55.up_proj", "model.layers.83.mlp.experts.56.up_proj", "model.layers.83.mlp.experts.57.up_proj", "model.layers.83.mlp.experts.58.up_proj", "model.layers.83.mlp.experts.59.up_proj", "model.layers.83.mlp.experts.60.up_proj", "model.layers.83.mlp.experts.61.up_proj", "model.layers.83.mlp.experts.62.up_proj", "model.layers.83.mlp.experts.63.up_proj", "model.layers.83.mlp.experts.64.up_proj", "model.layers.83.mlp.experts.65.up_proj", "model.layers.83.mlp.experts.66.up_proj", "model.layers.83.mlp.experts.67.up_proj", "model.layers.83.mlp.experts.68.up_proj", "model.layers.83.mlp.experts.69.up_proj", "model.layers.83.mlp.experts.70.up_proj", "model.layers.83.mlp.experts.71.up_proj", "model.layers.83.mlp.experts.72.up_proj", "model.layers.83.mlp.experts.73.up_proj", "model.layers.83.mlp.experts.74.up_proj", "model.layers.83.mlp.experts.75.up_proj", "model.layers.83.mlp.experts.76.up_proj", "model.layers.83.mlp.experts.77.up_proj", "model.layers.83.mlp.experts.78.up_proj", "model.layers.83.mlp.experts.79.up_proj", "model.layers.83.mlp.experts.80.up_proj", "model.layers.83.mlp.experts.81.up_proj", "model.layers.83.mlp.experts.82.up_proj", "model.layers.83.mlp.experts.83.up_proj", "model.layers.83.mlp.experts.84.up_proj", "model.layers.83.mlp.experts.85.up_proj", "model.layers.83.mlp.experts.86.up_proj", "model.layers.83.mlp.experts.87.up_proj", "model.layers.83.mlp.experts.88.up_proj", "model.layers.83.mlp.experts.89.up_proj", "model.layers.83.mlp.experts.90.up_proj", "model.layers.83.mlp.experts.91.up_proj", "model.layers.83.mlp.experts.92.up_proj", "model.layers.83.mlp.experts.93.up_proj", "model.layers.83.mlp.experts.94.up_proj", "model.layers.83.mlp.experts.95.up_proj", "model.layers.83.mlp.experts.96.up_proj", "model.layers.83.mlp.experts.97.up_proj", "model.layers.83.mlp.experts.98.up_proj", "model.layers.83.mlp.experts.99.up_proj", "model.layers.83.mlp.experts.100.up_proj", "model.layers.83.mlp.experts.101.up_proj", "model.layers.83.mlp.experts.102.up_proj", "model.layers.83.mlp.experts.103.up_proj", "model.layers.83.mlp.experts.104.up_proj", "model.layers.83.mlp.experts.105.up_proj", "model.layers.83.mlp.experts.106.up_proj", "model.layers.83.mlp.experts.107.up_proj", "model.layers.83.mlp.experts.108.up_proj", "model.layers.83.mlp.experts.109.up_proj", "model.layers.83.mlp.experts.110.up_proj", "model.layers.83.mlp.experts.111.up_proj", "model.layers.83.mlp.experts.112.up_proj", "model.layers.83.mlp.experts.113.up_proj", "model.layers.83.mlp.experts.114.up_proj", "model.layers.83.mlp.experts.115.up_proj", "model.layers.83.mlp.experts.116.up_proj", "model.layers.83.mlp.experts.117.up_proj", "model.layers.83.mlp.experts.118.up_proj", "model.layers.83.mlp.experts.119.up_proj", "model.layers.83.mlp.experts.120.up_proj", "model.layers.83.mlp.experts.121.up_proj", "model.layers.83.mlp.experts.122.up_proj", "model.layers.83.mlp.experts.123.up_proj", "model.layers.83.mlp.experts.124.up_proj", "model.layers.83.mlp.experts.125.up_proj", "model.layers.83.mlp.experts.126.up_proj", "model.layers.83.mlp.experts.127.up_proj", "model.layers.83.mlp.experts.128.up_proj", "model.layers.83.mlp.experts.129.up_proj", "model.layers.83.mlp.experts.130.up_proj", "model.layers.83.mlp.experts.131.up_proj", "model.layers.83.mlp.experts.132.up_proj", "model.layers.83.mlp.experts.133.up_proj", "model.layers.83.mlp.experts.134.up_proj", "model.layers.83.mlp.experts.135.up_proj", "model.layers.83.mlp.experts.136.up_proj", "model.layers.83.mlp.experts.137.up_proj", "model.layers.83.mlp.experts.138.up_proj", "model.layers.83.mlp.experts.139.up_proj", "model.layers.83.mlp.experts.140.up_proj", "model.layers.83.mlp.experts.141.up_proj", "model.layers.83.mlp.experts.142.up_proj", "model.layers.83.mlp.experts.143.up_proj", "model.layers.83.mlp.experts.144.up_proj", "model.layers.83.mlp.experts.145.up_proj", "model.layers.83.mlp.experts.146.up_proj", "model.layers.83.mlp.experts.147.up_proj", "model.layers.83.mlp.experts.148.up_proj", "model.layers.83.mlp.experts.149.up_proj", "model.layers.83.mlp.experts.150.up_proj", "model.layers.83.mlp.experts.151.up_proj", "model.layers.83.mlp.experts.152.up_proj", "model.layers.83.mlp.experts.153.up_proj", "model.layers.83.mlp.experts.154.up_proj", "model.layers.83.mlp.experts.155.up_proj", "model.layers.83.mlp.experts.156.up_proj", "model.layers.83.mlp.experts.157.up_proj", "model.layers.83.mlp.experts.158.up_proj", "model.layers.83.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00066345036029819, "dbits": 2516582400 } ] }, { "idx": 500, "layers": [ "model.layers.83.mlp.experts.0.down_proj", "model.layers.83.mlp.experts.1.down_proj", "model.layers.83.mlp.experts.2.down_proj", "model.layers.83.mlp.experts.3.down_proj", "model.layers.83.mlp.experts.4.down_proj", "model.layers.83.mlp.experts.5.down_proj", "model.layers.83.mlp.experts.6.down_proj", "model.layers.83.mlp.experts.7.down_proj", "model.layers.83.mlp.experts.8.down_proj", "model.layers.83.mlp.experts.9.down_proj", "model.layers.83.mlp.experts.10.down_proj", "model.layers.83.mlp.experts.11.down_proj", "model.layers.83.mlp.experts.12.down_proj", "model.layers.83.mlp.experts.13.down_proj", "model.layers.83.mlp.experts.14.down_proj", "model.layers.83.mlp.experts.15.down_proj", "model.layers.83.mlp.experts.16.down_proj", "model.layers.83.mlp.experts.17.down_proj", "model.layers.83.mlp.experts.18.down_proj", "model.layers.83.mlp.experts.19.down_proj", "model.layers.83.mlp.experts.20.down_proj", "model.layers.83.mlp.experts.21.down_proj", "model.layers.83.mlp.experts.22.down_proj", "model.layers.83.mlp.experts.23.down_proj", "model.layers.83.mlp.experts.24.down_proj", "model.layers.83.mlp.experts.25.down_proj", "model.layers.83.mlp.experts.26.down_proj", "model.layers.83.mlp.experts.27.down_proj", "model.layers.83.mlp.experts.28.down_proj", "model.layers.83.mlp.experts.29.down_proj", "model.layers.83.mlp.experts.30.down_proj", "model.layers.83.mlp.experts.31.down_proj", "model.layers.83.mlp.experts.32.down_proj", "model.layers.83.mlp.experts.33.down_proj", "model.layers.83.mlp.experts.34.down_proj", "model.layers.83.mlp.experts.35.down_proj", "model.layers.83.mlp.experts.36.down_proj", "model.layers.83.mlp.experts.37.down_proj", "model.layers.83.mlp.experts.38.down_proj", "model.layers.83.mlp.experts.39.down_proj", "model.layers.83.mlp.experts.40.down_proj", "model.layers.83.mlp.experts.41.down_proj", "model.layers.83.mlp.experts.42.down_proj", "model.layers.83.mlp.experts.43.down_proj", "model.layers.83.mlp.experts.44.down_proj", "model.layers.83.mlp.experts.45.down_proj", "model.layers.83.mlp.experts.46.down_proj", "model.layers.83.mlp.experts.47.down_proj", "model.layers.83.mlp.experts.48.down_proj", "model.layers.83.mlp.experts.49.down_proj", "model.layers.83.mlp.experts.50.down_proj", "model.layers.83.mlp.experts.51.down_proj", "model.layers.83.mlp.experts.52.down_proj", "model.layers.83.mlp.experts.53.down_proj", "model.layers.83.mlp.experts.54.down_proj", "model.layers.83.mlp.experts.55.down_proj", "model.layers.83.mlp.experts.56.down_proj", "model.layers.83.mlp.experts.57.down_proj", "model.layers.83.mlp.experts.58.down_proj", "model.layers.83.mlp.experts.59.down_proj", "model.layers.83.mlp.experts.60.down_proj", "model.layers.83.mlp.experts.61.down_proj", "model.layers.83.mlp.experts.62.down_proj", "model.layers.83.mlp.experts.63.down_proj", "model.layers.83.mlp.experts.64.down_proj", "model.layers.83.mlp.experts.65.down_proj", "model.layers.83.mlp.experts.66.down_proj", "model.layers.83.mlp.experts.67.down_proj", "model.layers.83.mlp.experts.68.down_proj", "model.layers.83.mlp.experts.69.down_proj", "model.layers.83.mlp.experts.70.down_proj", "model.layers.83.mlp.experts.71.down_proj", "model.layers.83.mlp.experts.72.down_proj", "model.layers.83.mlp.experts.73.down_proj", "model.layers.83.mlp.experts.74.down_proj", "model.layers.83.mlp.experts.75.down_proj", "model.layers.83.mlp.experts.76.down_proj", "model.layers.83.mlp.experts.77.down_proj", "model.layers.83.mlp.experts.78.down_proj", "model.layers.83.mlp.experts.79.down_proj", "model.layers.83.mlp.experts.80.down_proj", "model.layers.83.mlp.experts.81.down_proj", "model.layers.83.mlp.experts.82.down_proj", "model.layers.83.mlp.experts.83.down_proj", "model.layers.83.mlp.experts.84.down_proj", "model.layers.83.mlp.experts.85.down_proj", "model.layers.83.mlp.experts.86.down_proj", "model.layers.83.mlp.experts.87.down_proj", "model.layers.83.mlp.experts.88.down_proj", "model.layers.83.mlp.experts.89.down_proj", "model.layers.83.mlp.experts.90.down_proj", "model.layers.83.mlp.experts.91.down_proj", "model.layers.83.mlp.experts.92.down_proj", "model.layers.83.mlp.experts.93.down_proj", "model.layers.83.mlp.experts.94.down_proj", "model.layers.83.mlp.experts.95.down_proj", "model.layers.83.mlp.experts.96.down_proj", "model.layers.83.mlp.experts.97.down_proj", "model.layers.83.mlp.experts.98.down_proj", "model.layers.83.mlp.experts.99.down_proj", "model.layers.83.mlp.experts.100.down_proj", "model.layers.83.mlp.experts.101.down_proj", "model.layers.83.mlp.experts.102.down_proj", "model.layers.83.mlp.experts.103.down_proj", "model.layers.83.mlp.experts.104.down_proj", "model.layers.83.mlp.experts.105.down_proj", "model.layers.83.mlp.experts.106.down_proj", "model.layers.83.mlp.experts.107.down_proj", "model.layers.83.mlp.experts.108.down_proj", "model.layers.83.mlp.experts.109.down_proj", "model.layers.83.mlp.experts.110.down_proj", "model.layers.83.mlp.experts.111.down_proj", "model.layers.83.mlp.experts.112.down_proj", "model.layers.83.mlp.experts.113.down_proj", "model.layers.83.mlp.experts.114.down_proj", "model.layers.83.mlp.experts.115.down_proj", "model.layers.83.mlp.experts.116.down_proj", "model.layers.83.mlp.experts.117.down_proj", "model.layers.83.mlp.experts.118.down_proj", "model.layers.83.mlp.experts.119.down_proj", "model.layers.83.mlp.experts.120.down_proj", "model.layers.83.mlp.experts.121.down_proj", "model.layers.83.mlp.experts.122.down_proj", "model.layers.83.mlp.experts.123.down_proj", "model.layers.83.mlp.experts.124.down_proj", "model.layers.83.mlp.experts.125.down_proj", "model.layers.83.mlp.experts.126.down_proj", "model.layers.83.mlp.experts.127.down_proj", "model.layers.83.mlp.experts.128.down_proj", "model.layers.83.mlp.experts.129.down_proj", "model.layers.83.mlp.experts.130.down_proj", "model.layers.83.mlp.experts.131.down_proj", "model.layers.83.mlp.experts.132.down_proj", "model.layers.83.mlp.experts.133.down_proj", "model.layers.83.mlp.experts.134.down_proj", "model.layers.83.mlp.experts.135.down_proj", "model.layers.83.mlp.experts.136.down_proj", "model.layers.83.mlp.experts.137.down_proj", "model.layers.83.mlp.experts.138.down_proj", "model.layers.83.mlp.experts.139.down_proj", "model.layers.83.mlp.experts.140.down_proj", "model.layers.83.mlp.experts.141.down_proj", "model.layers.83.mlp.experts.142.down_proj", "model.layers.83.mlp.experts.143.down_proj", "model.layers.83.mlp.experts.144.down_proj", "model.layers.83.mlp.experts.145.down_proj", "model.layers.83.mlp.experts.146.down_proj", "model.layers.83.mlp.experts.147.down_proj", "model.layers.83.mlp.experts.148.down_proj", "model.layers.83.mlp.experts.149.down_proj", "model.layers.83.mlp.experts.150.down_proj", "model.layers.83.mlp.experts.151.down_proj", "model.layers.83.mlp.experts.152.down_proj", "model.layers.83.mlp.experts.153.down_proj", "model.layers.83.mlp.experts.154.down_proj", "model.layers.83.mlp.experts.155.down_proj", "model.layers.83.mlp.experts.156.down_proj", "model.layers.83.mlp.experts.157.down_proj", "model.layers.83.mlp.experts.158.down_proj", "model.layers.83.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00025053098797800377, "dbits": 1258291200 } ] }, { "idx": 501, "layers": [ "model.layers.84.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0008461847901344299, "dbits": 62914560 } ] }, { "idx": 502, "layers": [ "model.layers.84.self_attn.k_proj", "model.layers.84.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0017003633081912883, "dbits": 10485760 } ] }, { "idx": 503, "layers": [ "model.layers.84.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0020670831203460915, "dbits": 62914560 } ] }, { "idx": 504, "layers": [ "model.layers.84.mlp.shared_experts.gate_proj", "model.layers.84.mlp.shared_experts.up_proj", "model.layers.84.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.000791458040475812, "dbits": 23592960 } ] }, { "idx": 505, "layers": [ "model.layers.84.mlp.experts.0.gate_proj", "model.layers.84.mlp.experts.1.gate_proj", "model.layers.84.mlp.experts.2.gate_proj", "model.layers.84.mlp.experts.3.gate_proj", "model.layers.84.mlp.experts.4.gate_proj", "model.layers.84.mlp.experts.5.gate_proj", "model.layers.84.mlp.experts.6.gate_proj", "model.layers.84.mlp.experts.7.gate_proj", "model.layers.84.mlp.experts.8.gate_proj", "model.layers.84.mlp.experts.9.gate_proj", "model.layers.84.mlp.experts.10.gate_proj", "model.layers.84.mlp.experts.11.gate_proj", "model.layers.84.mlp.experts.12.gate_proj", "model.layers.84.mlp.experts.13.gate_proj", "model.layers.84.mlp.experts.14.gate_proj", "model.layers.84.mlp.experts.15.gate_proj", "model.layers.84.mlp.experts.16.gate_proj", "model.layers.84.mlp.experts.17.gate_proj", "model.layers.84.mlp.experts.18.gate_proj", "model.layers.84.mlp.experts.19.gate_proj", "model.layers.84.mlp.experts.20.gate_proj", "model.layers.84.mlp.experts.21.gate_proj", "model.layers.84.mlp.experts.22.gate_proj", "model.layers.84.mlp.experts.23.gate_proj", "model.layers.84.mlp.experts.24.gate_proj", "model.layers.84.mlp.experts.25.gate_proj", "model.layers.84.mlp.experts.26.gate_proj", "model.layers.84.mlp.experts.27.gate_proj", "model.layers.84.mlp.experts.28.gate_proj", "model.layers.84.mlp.experts.29.gate_proj", "model.layers.84.mlp.experts.30.gate_proj", "model.layers.84.mlp.experts.31.gate_proj", "model.layers.84.mlp.experts.32.gate_proj", "model.layers.84.mlp.experts.33.gate_proj", "model.layers.84.mlp.experts.34.gate_proj", "model.layers.84.mlp.experts.35.gate_proj", "model.layers.84.mlp.experts.36.gate_proj", "model.layers.84.mlp.experts.37.gate_proj", "model.layers.84.mlp.experts.38.gate_proj", "model.layers.84.mlp.experts.39.gate_proj", "model.layers.84.mlp.experts.40.gate_proj", "model.layers.84.mlp.experts.41.gate_proj", "model.layers.84.mlp.experts.42.gate_proj", "model.layers.84.mlp.experts.43.gate_proj", "model.layers.84.mlp.experts.44.gate_proj", "model.layers.84.mlp.experts.45.gate_proj", "model.layers.84.mlp.experts.46.gate_proj", "model.layers.84.mlp.experts.47.gate_proj", "model.layers.84.mlp.experts.48.gate_proj", "model.layers.84.mlp.experts.49.gate_proj", "model.layers.84.mlp.experts.50.gate_proj", "model.layers.84.mlp.experts.51.gate_proj", "model.layers.84.mlp.experts.52.gate_proj", "model.layers.84.mlp.experts.53.gate_proj", "model.layers.84.mlp.experts.54.gate_proj", "model.layers.84.mlp.experts.55.gate_proj", "model.layers.84.mlp.experts.56.gate_proj", "model.layers.84.mlp.experts.57.gate_proj", "model.layers.84.mlp.experts.58.gate_proj", "model.layers.84.mlp.experts.59.gate_proj", "model.layers.84.mlp.experts.60.gate_proj", "model.layers.84.mlp.experts.61.gate_proj", "model.layers.84.mlp.experts.62.gate_proj", "model.layers.84.mlp.experts.63.gate_proj", "model.layers.84.mlp.experts.64.gate_proj", "model.layers.84.mlp.experts.65.gate_proj", "model.layers.84.mlp.experts.66.gate_proj", "model.layers.84.mlp.experts.67.gate_proj", "model.layers.84.mlp.experts.68.gate_proj", "model.layers.84.mlp.experts.69.gate_proj", "model.layers.84.mlp.experts.70.gate_proj", "model.layers.84.mlp.experts.71.gate_proj", "model.layers.84.mlp.experts.72.gate_proj", "model.layers.84.mlp.experts.73.gate_proj", "model.layers.84.mlp.experts.74.gate_proj", "model.layers.84.mlp.experts.75.gate_proj", "model.layers.84.mlp.experts.76.gate_proj", "model.layers.84.mlp.experts.77.gate_proj", "model.layers.84.mlp.experts.78.gate_proj", "model.layers.84.mlp.experts.79.gate_proj", "model.layers.84.mlp.experts.80.gate_proj", "model.layers.84.mlp.experts.81.gate_proj", "model.layers.84.mlp.experts.82.gate_proj", "model.layers.84.mlp.experts.83.gate_proj", "model.layers.84.mlp.experts.84.gate_proj", "model.layers.84.mlp.experts.85.gate_proj", "model.layers.84.mlp.experts.86.gate_proj", "model.layers.84.mlp.experts.87.gate_proj", "model.layers.84.mlp.experts.88.gate_proj", "model.layers.84.mlp.experts.89.gate_proj", "model.layers.84.mlp.experts.90.gate_proj", "model.layers.84.mlp.experts.91.gate_proj", "model.layers.84.mlp.experts.92.gate_proj", "model.layers.84.mlp.experts.93.gate_proj", "model.layers.84.mlp.experts.94.gate_proj", "model.layers.84.mlp.experts.95.gate_proj", "model.layers.84.mlp.experts.96.gate_proj", "model.layers.84.mlp.experts.97.gate_proj", "model.layers.84.mlp.experts.98.gate_proj", "model.layers.84.mlp.experts.99.gate_proj", "model.layers.84.mlp.experts.100.gate_proj", "model.layers.84.mlp.experts.101.gate_proj", "model.layers.84.mlp.experts.102.gate_proj", "model.layers.84.mlp.experts.103.gate_proj", "model.layers.84.mlp.experts.104.gate_proj", "model.layers.84.mlp.experts.105.gate_proj", "model.layers.84.mlp.experts.106.gate_proj", "model.layers.84.mlp.experts.107.gate_proj", "model.layers.84.mlp.experts.108.gate_proj", "model.layers.84.mlp.experts.109.gate_proj", "model.layers.84.mlp.experts.110.gate_proj", "model.layers.84.mlp.experts.111.gate_proj", "model.layers.84.mlp.experts.112.gate_proj", "model.layers.84.mlp.experts.113.gate_proj", "model.layers.84.mlp.experts.114.gate_proj", "model.layers.84.mlp.experts.115.gate_proj", "model.layers.84.mlp.experts.116.gate_proj", "model.layers.84.mlp.experts.117.gate_proj", "model.layers.84.mlp.experts.118.gate_proj", "model.layers.84.mlp.experts.119.gate_proj", "model.layers.84.mlp.experts.120.gate_proj", "model.layers.84.mlp.experts.121.gate_proj", "model.layers.84.mlp.experts.122.gate_proj", "model.layers.84.mlp.experts.123.gate_proj", "model.layers.84.mlp.experts.124.gate_proj", "model.layers.84.mlp.experts.125.gate_proj", "model.layers.84.mlp.experts.126.gate_proj", "model.layers.84.mlp.experts.127.gate_proj", "model.layers.84.mlp.experts.128.gate_proj", "model.layers.84.mlp.experts.129.gate_proj", "model.layers.84.mlp.experts.130.gate_proj", "model.layers.84.mlp.experts.131.gate_proj", "model.layers.84.mlp.experts.132.gate_proj", "model.layers.84.mlp.experts.133.gate_proj", "model.layers.84.mlp.experts.134.gate_proj", "model.layers.84.mlp.experts.135.gate_proj", "model.layers.84.mlp.experts.136.gate_proj", "model.layers.84.mlp.experts.137.gate_proj", "model.layers.84.mlp.experts.138.gate_proj", "model.layers.84.mlp.experts.139.gate_proj", "model.layers.84.mlp.experts.140.gate_proj", "model.layers.84.mlp.experts.141.gate_proj", "model.layers.84.mlp.experts.142.gate_proj", "model.layers.84.mlp.experts.143.gate_proj", "model.layers.84.mlp.experts.144.gate_proj", "model.layers.84.mlp.experts.145.gate_proj", "model.layers.84.mlp.experts.146.gate_proj", "model.layers.84.mlp.experts.147.gate_proj", "model.layers.84.mlp.experts.148.gate_proj", "model.layers.84.mlp.experts.149.gate_proj", "model.layers.84.mlp.experts.150.gate_proj", "model.layers.84.mlp.experts.151.gate_proj", "model.layers.84.mlp.experts.152.gate_proj", "model.layers.84.mlp.experts.153.gate_proj", "model.layers.84.mlp.experts.154.gate_proj", "model.layers.84.mlp.experts.155.gate_proj", "model.layers.84.mlp.experts.156.gate_proj", "model.layers.84.mlp.experts.157.gate_proj", "model.layers.84.mlp.experts.158.gate_proj", "model.layers.84.mlp.experts.159.gate_proj", "model.layers.84.mlp.experts.0.up_proj", "model.layers.84.mlp.experts.1.up_proj", "model.layers.84.mlp.experts.2.up_proj", "model.layers.84.mlp.experts.3.up_proj", "model.layers.84.mlp.experts.4.up_proj", "model.layers.84.mlp.experts.5.up_proj", "model.layers.84.mlp.experts.6.up_proj", "model.layers.84.mlp.experts.7.up_proj", "model.layers.84.mlp.experts.8.up_proj", "model.layers.84.mlp.experts.9.up_proj", "model.layers.84.mlp.experts.10.up_proj", "model.layers.84.mlp.experts.11.up_proj", "model.layers.84.mlp.experts.12.up_proj", "model.layers.84.mlp.experts.13.up_proj", "model.layers.84.mlp.experts.14.up_proj", "model.layers.84.mlp.experts.15.up_proj", "model.layers.84.mlp.experts.16.up_proj", "model.layers.84.mlp.experts.17.up_proj", "model.layers.84.mlp.experts.18.up_proj", "model.layers.84.mlp.experts.19.up_proj", "model.layers.84.mlp.experts.20.up_proj", "model.layers.84.mlp.experts.21.up_proj", "model.layers.84.mlp.experts.22.up_proj", "model.layers.84.mlp.experts.23.up_proj", "model.layers.84.mlp.experts.24.up_proj", "model.layers.84.mlp.experts.25.up_proj", "model.layers.84.mlp.experts.26.up_proj", "model.layers.84.mlp.experts.27.up_proj", "model.layers.84.mlp.experts.28.up_proj", "model.layers.84.mlp.experts.29.up_proj", "model.layers.84.mlp.experts.30.up_proj", "model.layers.84.mlp.experts.31.up_proj", "model.layers.84.mlp.experts.32.up_proj", "model.layers.84.mlp.experts.33.up_proj", "model.layers.84.mlp.experts.34.up_proj", "model.layers.84.mlp.experts.35.up_proj", "model.layers.84.mlp.experts.36.up_proj", "model.layers.84.mlp.experts.37.up_proj", "model.layers.84.mlp.experts.38.up_proj", "model.layers.84.mlp.experts.39.up_proj", "model.layers.84.mlp.experts.40.up_proj", "model.layers.84.mlp.experts.41.up_proj", "model.layers.84.mlp.experts.42.up_proj", "model.layers.84.mlp.experts.43.up_proj", "model.layers.84.mlp.experts.44.up_proj", "model.layers.84.mlp.experts.45.up_proj", "model.layers.84.mlp.experts.46.up_proj", "model.layers.84.mlp.experts.47.up_proj", "model.layers.84.mlp.experts.48.up_proj", "model.layers.84.mlp.experts.49.up_proj", "model.layers.84.mlp.experts.50.up_proj", "model.layers.84.mlp.experts.51.up_proj", "model.layers.84.mlp.experts.52.up_proj", "model.layers.84.mlp.experts.53.up_proj", "model.layers.84.mlp.experts.54.up_proj", "model.layers.84.mlp.experts.55.up_proj", "model.layers.84.mlp.experts.56.up_proj", "model.layers.84.mlp.experts.57.up_proj", "model.layers.84.mlp.experts.58.up_proj", "model.layers.84.mlp.experts.59.up_proj", "model.layers.84.mlp.experts.60.up_proj", "model.layers.84.mlp.experts.61.up_proj", "model.layers.84.mlp.experts.62.up_proj", "model.layers.84.mlp.experts.63.up_proj", "model.layers.84.mlp.experts.64.up_proj", "model.layers.84.mlp.experts.65.up_proj", "model.layers.84.mlp.experts.66.up_proj", "model.layers.84.mlp.experts.67.up_proj", "model.layers.84.mlp.experts.68.up_proj", "model.layers.84.mlp.experts.69.up_proj", "model.layers.84.mlp.experts.70.up_proj", "model.layers.84.mlp.experts.71.up_proj", "model.layers.84.mlp.experts.72.up_proj", "model.layers.84.mlp.experts.73.up_proj", "model.layers.84.mlp.experts.74.up_proj", "model.layers.84.mlp.experts.75.up_proj", "model.layers.84.mlp.experts.76.up_proj", "model.layers.84.mlp.experts.77.up_proj", "model.layers.84.mlp.experts.78.up_proj", "model.layers.84.mlp.experts.79.up_proj", "model.layers.84.mlp.experts.80.up_proj", "model.layers.84.mlp.experts.81.up_proj", "model.layers.84.mlp.experts.82.up_proj", "model.layers.84.mlp.experts.83.up_proj", "model.layers.84.mlp.experts.84.up_proj", "model.layers.84.mlp.experts.85.up_proj", "model.layers.84.mlp.experts.86.up_proj", "model.layers.84.mlp.experts.87.up_proj", "model.layers.84.mlp.experts.88.up_proj", "model.layers.84.mlp.experts.89.up_proj", "model.layers.84.mlp.experts.90.up_proj", "model.layers.84.mlp.experts.91.up_proj", "model.layers.84.mlp.experts.92.up_proj", "model.layers.84.mlp.experts.93.up_proj", "model.layers.84.mlp.experts.94.up_proj", "model.layers.84.mlp.experts.95.up_proj", "model.layers.84.mlp.experts.96.up_proj", "model.layers.84.mlp.experts.97.up_proj", "model.layers.84.mlp.experts.98.up_proj", "model.layers.84.mlp.experts.99.up_proj", "model.layers.84.mlp.experts.100.up_proj", "model.layers.84.mlp.experts.101.up_proj", "model.layers.84.mlp.experts.102.up_proj", "model.layers.84.mlp.experts.103.up_proj", "model.layers.84.mlp.experts.104.up_proj", "model.layers.84.mlp.experts.105.up_proj", "model.layers.84.mlp.experts.106.up_proj", "model.layers.84.mlp.experts.107.up_proj", "model.layers.84.mlp.experts.108.up_proj", "model.layers.84.mlp.experts.109.up_proj", "model.layers.84.mlp.experts.110.up_proj", "model.layers.84.mlp.experts.111.up_proj", "model.layers.84.mlp.experts.112.up_proj", "model.layers.84.mlp.experts.113.up_proj", "model.layers.84.mlp.experts.114.up_proj", "model.layers.84.mlp.experts.115.up_proj", "model.layers.84.mlp.experts.116.up_proj", "model.layers.84.mlp.experts.117.up_proj", "model.layers.84.mlp.experts.118.up_proj", "model.layers.84.mlp.experts.119.up_proj", "model.layers.84.mlp.experts.120.up_proj", "model.layers.84.mlp.experts.121.up_proj", "model.layers.84.mlp.experts.122.up_proj", "model.layers.84.mlp.experts.123.up_proj", "model.layers.84.mlp.experts.124.up_proj", "model.layers.84.mlp.experts.125.up_proj", "model.layers.84.mlp.experts.126.up_proj", "model.layers.84.mlp.experts.127.up_proj", "model.layers.84.mlp.experts.128.up_proj", "model.layers.84.mlp.experts.129.up_proj", "model.layers.84.mlp.experts.130.up_proj", "model.layers.84.mlp.experts.131.up_proj", "model.layers.84.mlp.experts.132.up_proj", "model.layers.84.mlp.experts.133.up_proj", "model.layers.84.mlp.experts.134.up_proj", "model.layers.84.mlp.experts.135.up_proj", "model.layers.84.mlp.experts.136.up_proj", "model.layers.84.mlp.experts.137.up_proj", "model.layers.84.mlp.experts.138.up_proj", "model.layers.84.mlp.experts.139.up_proj", "model.layers.84.mlp.experts.140.up_proj", "model.layers.84.mlp.experts.141.up_proj", "model.layers.84.mlp.experts.142.up_proj", "model.layers.84.mlp.experts.143.up_proj", "model.layers.84.mlp.experts.144.up_proj", "model.layers.84.mlp.experts.145.up_proj", "model.layers.84.mlp.experts.146.up_proj", "model.layers.84.mlp.experts.147.up_proj", "model.layers.84.mlp.experts.148.up_proj", "model.layers.84.mlp.experts.149.up_proj", "model.layers.84.mlp.experts.150.up_proj", "model.layers.84.mlp.experts.151.up_proj", "model.layers.84.mlp.experts.152.up_proj", "model.layers.84.mlp.experts.153.up_proj", "model.layers.84.mlp.experts.154.up_proj", "model.layers.84.mlp.experts.155.up_proj", "model.layers.84.mlp.experts.156.up_proj", "model.layers.84.mlp.experts.157.up_proj", "model.layers.84.mlp.experts.158.up_proj", "model.layers.84.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0003945678472518921, "dbits": 2516582400 } ] }, { "idx": 506, "layers": [ "model.layers.84.mlp.experts.0.down_proj", "model.layers.84.mlp.experts.1.down_proj", "model.layers.84.mlp.experts.2.down_proj", "model.layers.84.mlp.experts.3.down_proj", "model.layers.84.mlp.experts.4.down_proj", "model.layers.84.mlp.experts.5.down_proj", "model.layers.84.mlp.experts.6.down_proj", "model.layers.84.mlp.experts.7.down_proj", "model.layers.84.mlp.experts.8.down_proj", "model.layers.84.mlp.experts.9.down_proj", "model.layers.84.mlp.experts.10.down_proj", "model.layers.84.mlp.experts.11.down_proj", "model.layers.84.mlp.experts.12.down_proj", "model.layers.84.mlp.experts.13.down_proj", "model.layers.84.mlp.experts.14.down_proj", "model.layers.84.mlp.experts.15.down_proj", "model.layers.84.mlp.experts.16.down_proj", "model.layers.84.mlp.experts.17.down_proj", "model.layers.84.mlp.experts.18.down_proj", "model.layers.84.mlp.experts.19.down_proj", "model.layers.84.mlp.experts.20.down_proj", "model.layers.84.mlp.experts.21.down_proj", "model.layers.84.mlp.experts.22.down_proj", "model.layers.84.mlp.experts.23.down_proj", "model.layers.84.mlp.experts.24.down_proj", "model.layers.84.mlp.experts.25.down_proj", "model.layers.84.mlp.experts.26.down_proj", "model.layers.84.mlp.experts.27.down_proj", "model.layers.84.mlp.experts.28.down_proj", "model.layers.84.mlp.experts.29.down_proj", "model.layers.84.mlp.experts.30.down_proj", "model.layers.84.mlp.experts.31.down_proj", "model.layers.84.mlp.experts.32.down_proj", "model.layers.84.mlp.experts.33.down_proj", "model.layers.84.mlp.experts.34.down_proj", "model.layers.84.mlp.experts.35.down_proj", "model.layers.84.mlp.experts.36.down_proj", "model.layers.84.mlp.experts.37.down_proj", "model.layers.84.mlp.experts.38.down_proj", "model.layers.84.mlp.experts.39.down_proj", "model.layers.84.mlp.experts.40.down_proj", "model.layers.84.mlp.experts.41.down_proj", "model.layers.84.mlp.experts.42.down_proj", "model.layers.84.mlp.experts.43.down_proj", "model.layers.84.mlp.experts.44.down_proj", "model.layers.84.mlp.experts.45.down_proj", "model.layers.84.mlp.experts.46.down_proj", "model.layers.84.mlp.experts.47.down_proj", "model.layers.84.mlp.experts.48.down_proj", "model.layers.84.mlp.experts.49.down_proj", "model.layers.84.mlp.experts.50.down_proj", "model.layers.84.mlp.experts.51.down_proj", "model.layers.84.mlp.experts.52.down_proj", "model.layers.84.mlp.experts.53.down_proj", "model.layers.84.mlp.experts.54.down_proj", "model.layers.84.mlp.experts.55.down_proj", "model.layers.84.mlp.experts.56.down_proj", "model.layers.84.mlp.experts.57.down_proj", "model.layers.84.mlp.experts.58.down_proj", "model.layers.84.mlp.experts.59.down_proj", "model.layers.84.mlp.experts.60.down_proj", "model.layers.84.mlp.experts.61.down_proj", "model.layers.84.mlp.experts.62.down_proj", "model.layers.84.mlp.experts.63.down_proj", "model.layers.84.mlp.experts.64.down_proj", "model.layers.84.mlp.experts.65.down_proj", "model.layers.84.mlp.experts.66.down_proj", "model.layers.84.mlp.experts.67.down_proj", "model.layers.84.mlp.experts.68.down_proj", "model.layers.84.mlp.experts.69.down_proj", "model.layers.84.mlp.experts.70.down_proj", "model.layers.84.mlp.experts.71.down_proj", "model.layers.84.mlp.experts.72.down_proj", "model.layers.84.mlp.experts.73.down_proj", "model.layers.84.mlp.experts.74.down_proj", "model.layers.84.mlp.experts.75.down_proj", "model.layers.84.mlp.experts.76.down_proj", "model.layers.84.mlp.experts.77.down_proj", "model.layers.84.mlp.experts.78.down_proj", "model.layers.84.mlp.experts.79.down_proj", "model.layers.84.mlp.experts.80.down_proj", "model.layers.84.mlp.experts.81.down_proj", "model.layers.84.mlp.experts.82.down_proj", "model.layers.84.mlp.experts.83.down_proj", "model.layers.84.mlp.experts.84.down_proj", "model.layers.84.mlp.experts.85.down_proj", "model.layers.84.mlp.experts.86.down_proj", "model.layers.84.mlp.experts.87.down_proj", "model.layers.84.mlp.experts.88.down_proj", "model.layers.84.mlp.experts.89.down_proj", "model.layers.84.mlp.experts.90.down_proj", "model.layers.84.mlp.experts.91.down_proj", "model.layers.84.mlp.experts.92.down_proj", "model.layers.84.mlp.experts.93.down_proj", "model.layers.84.mlp.experts.94.down_proj", "model.layers.84.mlp.experts.95.down_proj", "model.layers.84.mlp.experts.96.down_proj", "model.layers.84.mlp.experts.97.down_proj", "model.layers.84.mlp.experts.98.down_proj", "model.layers.84.mlp.experts.99.down_proj", "model.layers.84.mlp.experts.100.down_proj", "model.layers.84.mlp.experts.101.down_proj", "model.layers.84.mlp.experts.102.down_proj", "model.layers.84.mlp.experts.103.down_proj", "model.layers.84.mlp.experts.104.down_proj", "model.layers.84.mlp.experts.105.down_proj", "model.layers.84.mlp.experts.106.down_proj", "model.layers.84.mlp.experts.107.down_proj", "model.layers.84.mlp.experts.108.down_proj", "model.layers.84.mlp.experts.109.down_proj", "model.layers.84.mlp.experts.110.down_proj", "model.layers.84.mlp.experts.111.down_proj", "model.layers.84.mlp.experts.112.down_proj", "model.layers.84.mlp.experts.113.down_proj", "model.layers.84.mlp.experts.114.down_proj", "model.layers.84.mlp.experts.115.down_proj", "model.layers.84.mlp.experts.116.down_proj", "model.layers.84.mlp.experts.117.down_proj", "model.layers.84.mlp.experts.118.down_proj", "model.layers.84.mlp.experts.119.down_proj", "model.layers.84.mlp.experts.120.down_proj", "model.layers.84.mlp.experts.121.down_proj", "model.layers.84.mlp.experts.122.down_proj", "model.layers.84.mlp.experts.123.down_proj", "model.layers.84.mlp.experts.124.down_proj", "model.layers.84.mlp.experts.125.down_proj", "model.layers.84.mlp.experts.126.down_proj", "model.layers.84.mlp.experts.127.down_proj", "model.layers.84.mlp.experts.128.down_proj", "model.layers.84.mlp.experts.129.down_proj", "model.layers.84.mlp.experts.130.down_proj", "model.layers.84.mlp.experts.131.down_proj", "model.layers.84.mlp.experts.132.down_proj", "model.layers.84.mlp.experts.133.down_proj", "model.layers.84.mlp.experts.134.down_proj", "model.layers.84.mlp.experts.135.down_proj", "model.layers.84.mlp.experts.136.down_proj", "model.layers.84.mlp.experts.137.down_proj", "model.layers.84.mlp.experts.138.down_proj", "model.layers.84.mlp.experts.139.down_proj", "model.layers.84.mlp.experts.140.down_proj", "model.layers.84.mlp.experts.141.down_proj", "model.layers.84.mlp.experts.142.down_proj", "model.layers.84.mlp.experts.143.down_proj", "model.layers.84.mlp.experts.144.down_proj", "model.layers.84.mlp.experts.145.down_proj", "model.layers.84.mlp.experts.146.down_proj", "model.layers.84.mlp.experts.147.down_proj", "model.layers.84.mlp.experts.148.down_proj", "model.layers.84.mlp.experts.149.down_proj", "model.layers.84.mlp.experts.150.down_proj", "model.layers.84.mlp.experts.151.down_proj", "model.layers.84.mlp.experts.152.down_proj", "model.layers.84.mlp.experts.153.down_proj", "model.layers.84.mlp.experts.154.down_proj", "model.layers.84.mlp.experts.155.down_proj", "model.layers.84.mlp.experts.156.down_proj", "model.layers.84.mlp.experts.157.down_proj", "model.layers.84.mlp.experts.158.down_proj", "model.layers.84.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00016806051135065392, "dbits": 1258291200 } ] }, { "idx": 507, "layers": [ "model.layers.85.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0029085822403430717, "dbits": 62914560 } ] }, { "idx": 508, "layers": [ "model.layers.85.self_attn.k_proj", "model.layers.85.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0008400313556194416, "dbits": 10485760 } ] }, { "idx": 509, "layers": [ "model.layers.85.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0003927066922187472, "dbits": 62914560 } ] }, { "idx": 510, "layers": [ "model.layers.85.mlp.shared_experts.gate_proj", "model.layers.85.mlp.shared_experts.up_proj", "model.layers.85.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0013185098767280468, "dbits": 23592960 } ] }, { "idx": 511, "layers": [ "model.layers.85.mlp.experts.0.gate_proj", "model.layers.85.mlp.experts.1.gate_proj", "model.layers.85.mlp.experts.2.gate_proj", "model.layers.85.mlp.experts.3.gate_proj", "model.layers.85.mlp.experts.4.gate_proj", "model.layers.85.mlp.experts.5.gate_proj", "model.layers.85.mlp.experts.6.gate_proj", "model.layers.85.mlp.experts.7.gate_proj", "model.layers.85.mlp.experts.8.gate_proj", "model.layers.85.mlp.experts.9.gate_proj", "model.layers.85.mlp.experts.10.gate_proj", "model.layers.85.mlp.experts.11.gate_proj", "model.layers.85.mlp.experts.12.gate_proj", "model.layers.85.mlp.experts.13.gate_proj", "model.layers.85.mlp.experts.14.gate_proj", "model.layers.85.mlp.experts.15.gate_proj", "model.layers.85.mlp.experts.16.gate_proj", "model.layers.85.mlp.experts.17.gate_proj", "model.layers.85.mlp.experts.18.gate_proj", "model.layers.85.mlp.experts.19.gate_proj", "model.layers.85.mlp.experts.20.gate_proj", "model.layers.85.mlp.experts.21.gate_proj", "model.layers.85.mlp.experts.22.gate_proj", "model.layers.85.mlp.experts.23.gate_proj", "model.layers.85.mlp.experts.24.gate_proj", "model.layers.85.mlp.experts.25.gate_proj", "model.layers.85.mlp.experts.26.gate_proj", "model.layers.85.mlp.experts.27.gate_proj", "model.layers.85.mlp.experts.28.gate_proj", "model.layers.85.mlp.experts.29.gate_proj", "model.layers.85.mlp.experts.30.gate_proj", "model.layers.85.mlp.experts.31.gate_proj", "model.layers.85.mlp.experts.32.gate_proj", "model.layers.85.mlp.experts.33.gate_proj", "model.layers.85.mlp.experts.34.gate_proj", "model.layers.85.mlp.experts.35.gate_proj", "model.layers.85.mlp.experts.36.gate_proj", "model.layers.85.mlp.experts.37.gate_proj", "model.layers.85.mlp.experts.38.gate_proj", "model.layers.85.mlp.experts.39.gate_proj", "model.layers.85.mlp.experts.40.gate_proj", "model.layers.85.mlp.experts.41.gate_proj", "model.layers.85.mlp.experts.42.gate_proj", "model.layers.85.mlp.experts.43.gate_proj", "model.layers.85.mlp.experts.44.gate_proj", "model.layers.85.mlp.experts.45.gate_proj", "model.layers.85.mlp.experts.46.gate_proj", "model.layers.85.mlp.experts.47.gate_proj", "model.layers.85.mlp.experts.48.gate_proj", "model.layers.85.mlp.experts.49.gate_proj", "model.layers.85.mlp.experts.50.gate_proj", "model.layers.85.mlp.experts.51.gate_proj", "model.layers.85.mlp.experts.52.gate_proj", "model.layers.85.mlp.experts.53.gate_proj", "model.layers.85.mlp.experts.54.gate_proj", "model.layers.85.mlp.experts.55.gate_proj", "model.layers.85.mlp.experts.56.gate_proj", "model.layers.85.mlp.experts.57.gate_proj", "model.layers.85.mlp.experts.58.gate_proj", "model.layers.85.mlp.experts.59.gate_proj", "model.layers.85.mlp.experts.60.gate_proj", "model.layers.85.mlp.experts.61.gate_proj", "model.layers.85.mlp.experts.62.gate_proj", "model.layers.85.mlp.experts.63.gate_proj", "model.layers.85.mlp.experts.64.gate_proj", "model.layers.85.mlp.experts.65.gate_proj", "model.layers.85.mlp.experts.66.gate_proj", "model.layers.85.mlp.experts.67.gate_proj", "model.layers.85.mlp.experts.68.gate_proj", "model.layers.85.mlp.experts.69.gate_proj", "model.layers.85.mlp.experts.70.gate_proj", "model.layers.85.mlp.experts.71.gate_proj", "model.layers.85.mlp.experts.72.gate_proj", "model.layers.85.mlp.experts.73.gate_proj", "model.layers.85.mlp.experts.74.gate_proj", "model.layers.85.mlp.experts.75.gate_proj", "model.layers.85.mlp.experts.76.gate_proj", "model.layers.85.mlp.experts.77.gate_proj", "model.layers.85.mlp.experts.78.gate_proj", "model.layers.85.mlp.experts.79.gate_proj", "model.layers.85.mlp.experts.80.gate_proj", "model.layers.85.mlp.experts.81.gate_proj", "model.layers.85.mlp.experts.82.gate_proj", "model.layers.85.mlp.experts.83.gate_proj", "model.layers.85.mlp.experts.84.gate_proj", "model.layers.85.mlp.experts.85.gate_proj", "model.layers.85.mlp.experts.86.gate_proj", "model.layers.85.mlp.experts.87.gate_proj", "model.layers.85.mlp.experts.88.gate_proj", "model.layers.85.mlp.experts.89.gate_proj", "model.layers.85.mlp.experts.90.gate_proj", "model.layers.85.mlp.experts.91.gate_proj", "model.layers.85.mlp.experts.92.gate_proj", "model.layers.85.mlp.experts.93.gate_proj", "model.layers.85.mlp.experts.94.gate_proj", "model.layers.85.mlp.experts.95.gate_proj", "model.layers.85.mlp.experts.96.gate_proj", "model.layers.85.mlp.experts.97.gate_proj", "model.layers.85.mlp.experts.98.gate_proj", "model.layers.85.mlp.experts.99.gate_proj", "model.layers.85.mlp.experts.100.gate_proj", "model.layers.85.mlp.experts.101.gate_proj", "model.layers.85.mlp.experts.102.gate_proj", "model.layers.85.mlp.experts.103.gate_proj", "model.layers.85.mlp.experts.104.gate_proj", "model.layers.85.mlp.experts.105.gate_proj", "model.layers.85.mlp.experts.106.gate_proj", "model.layers.85.mlp.experts.107.gate_proj", "model.layers.85.mlp.experts.108.gate_proj", "model.layers.85.mlp.experts.109.gate_proj", "model.layers.85.mlp.experts.110.gate_proj", "model.layers.85.mlp.experts.111.gate_proj", "model.layers.85.mlp.experts.112.gate_proj", "model.layers.85.mlp.experts.113.gate_proj", "model.layers.85.mlp.experts.114.gate_proj", "model.layers.85.mlp.experts.115.gate_proj", "model.layers.85.mlp.experts.116.gate_proj", "model.layers.85.mlp.experts.117.gate_proj", "model.layers.85.mlp.experts.118.gate_proj", "model.layers.85.mlp.experts.119.gate_proj", "model.layers.85.mlp.experts.120.gate_proj", "model.layers.85.mlp.experts.121.gate_proj", "model.layers.85.mlp.experts.122.gate_proj", "model.layers.85.mlp.experts.123.gate_proj", "model.layers.85.mlp.experts.124.gate_proj", "model.layers.85.mlp.experts.125.gate_proj", "model.layers.85.mlp.experts.126.gate_proj", "model.layers.85.mlp.experts.127.gate_proj", "model.layers.85.mlp.experts.128.gate_proj", "model.layers.85.mlp.experts.129.gate_proj", "model.layers.85.mlp.experts.130.gate_proj", "model.layers.85.mlp.experts.131.gate_proj", "model.layers.85.mlp.experts.132.gate_proj", "model.layers.85.mlp.experts.133.gate_proj", "model.layers.85.mlp.experts.134.gate_proj", "model.layers.85.mlp.experts.135.gate_proj", "model.layers.85.mlp.experts.136.gate_proj", "model.layers.85.mlp.experts.137.gate_proj", "model.layers.85.mlp.experts.138.gate_proj", "model.layers.85.mlp.experts.139.gate_proj", "model.layers.85.mlp.experts.140.gate_proj", "model.layers.85.mlp.experts.141.gate_proj", "model.layers.85.mlp.experts.142.gate_proj", "model.layers.85.mlp.experts.143.gate_proj", "model.layers.85.mlp.experts.144.gate_proj", "model.layers.85.mlp.experts.145.gate_proj", "model.layers.85.mlp.experts.146.gate_proj", "model.layers.85.mlp.experts.147.gate_proj", "model.layers.85.mlp.experts.148.gate_proj", "model.layers.85.mlp.experts.149.gate_proj", "model.layers.85.mlp.experts.150.gate_proj", "model.layers.85.mlp.experts.151.gate_proj", "model.layers.85.mlp.experts.152.gate_proj", "model.layers.85.mlp.experts.153.gate_proj", "model.layers.85.mlp.experts.154.gate_proj", "model.layers.85.mlp.experts.155.gate_proj", "model.layers.85.mlp.experts.156.gate_proj", "model.layers.85.mlp.experts.157.gate_proj", "model.layers.85.mlp.experts.158.gate_proj", "model.layers.85.mlp.experts.159.gate_proj", "model.layers.85.mlp.experts.0.up_proj", "model.layers.85.mlp.experts.1.up_proj", "model.layers.85.mlp.experts.2.up_proj", "model.layers.85.mlp.experts.3.up_proj", "model.layers.85.mlp.experts.4.up_proj", "model.layers.85.mlp.experts.5.up_proj", "model.layers.85.mlp.experts.6.up_proj", "model.layers.85.mlp.experts.7.up_proj", "model.layers.85.mlp.experts.8.up_proj", "model.layers.85.mlp.experts.9.up_proj", "model.layers.85.mlp.experts.10.up_proj", "model.layers.85.mlp.experts.11.up_proj", "model.layers.85.mlp.experts.12.up_proj", "model.layers.85.mlp.experts.13.up_proj", "model.layers.85.mlp.experts.14.up_proj", "model.layers.85.mlp.experts.15.up_proj", "model.layers.85.mlp.experts.16.up_proj", "model.layers.85.mlp.experts.17.up_proj", "model.layers.85.mlp.experts.18.up_proj", "model.layers.85.mlp.experts.19.up_proj", "model.layers.85.mlp.experts.20.up_proj", "model.layers.85.mlp.experts.21.up_proj", "model.layers.85.mlp.experts.22.up_proj", "model.layers.85.mlp.experts.23.up_proj", "model.layers.85.mlp.experts.24.up_proj", "model.layers.85.mlp.experts.25.up_proj", "model.layers.85.mlp.experts.26.up_proj", "model.layers.85.mlp.experts.27.up_proj", "model.layers.85.mlp.experts.28.up_proj", "model.layers.85.mlp.experts.29.up_proj", "model.layers.85.mlp.experts.30.up_proj", "model.layers.85.mlp.experts.31.up_proj", "model.layers.85.mlp.experts.32.up_proj", "model.layers.85.mlp.experts.33.up_proj", "model.layers.85.mlp.experts.34.up_proj", "model.layers.85.mlp.experts.35.up_proj", "model.layers.85.mlp.experts.36.up_proj", "model.layers.85.mlp.experts.37.up_proj", "model.layers.85.mlp.experts.38.up_proj", "model.layers.85.mlp.experts.39.up_proj", "model.layers.85.mlp.experts.40.up_proj", "model.layers.85.mlp.experts.41.up_proj", "model.layers.85.mlp.experts.42.up_proj", "model.layers.85.mlp.experts.43.up_proj", "model.layers.85.mlp.experts.44.up_proj", "model.layers.85.mlp.experts.45.up_proj", "model.layers.85.mlp.experts.46.up_proj", "model.layers.85.mlp.experts.47.up_proj", "model.layers.85.mlp.experts.48.up_proj", "model.layers.85.mlp.experts.49.up_proj", "model.layers.85.mlp.experts.50.up_proj", "model.layers.85.mlp.experts.51.up_proj", "model.layers.85.mlp.experts.52.up_proj", "model.layers.85.mlp.experts.53.up_proj", "model.layers.85.mlp.experts.54.up_proj", "model.layers.85.mlp.experts.55.up_proj", "model.layers.85.mlp.experts.56.up_proj", "model.layers.85.mlp.experts.57.up_proj", "model.layers.85.mlp.experts.58.up_proj", "model.layers.85.mlp.experts.59.up_proj", "model.layers.85.mlp.experts.60.up_proj", "model.layers.85.mlp.experts.61.up_proj", "model.layers.85.mlp.experts.62.up_proj", "model.layers.85.mlp.experts.63.up_proj", "model.layers.85.mlp.experts.64.up_proj", "model.layers.85.mlp.experts.65.up_proj", "model.layers.85.mlp.experts.66.up_proj", "model.layers.85.mlp.experts.67.up_proj", "model.layers.85.mlp.experts.68.up_proj", "model.layers.85.mlp.experts.69.up_proj", "model.layers.85.mlp.experts.70.up_proj", "model.layers.85.mlp.experts.71.up_proj", "model.layers.85.mlp.experts.72.up_proj", "model.layers.85.mlp.experts.73.up_proj", "model.layers.85.mlp.experts.74.up_proj", "model.layers.85.mlp.experts.75.up_proj", "model.layers.85.mlp.experts.76.up_proj", "model.layers.85.mlp.experts.77.up_proj", "model.layers.85.mlp.experts.78.up_proj", "model.layers.85.mlp.experts.79.up_proj", "model.layers.85.mlp.experts.80.up_proj", "model.layers.85.mlp.experts.81.up_proj", "model.layers.85.mlp.experts.82.up_proj", "model.layers.85.mlp.experts.83.up_proj", "model.layers.85.mlp.experts.84.up_proj", "model.layers.85.mlp.experts.85.up_proj", "model.layers.85.mlp.experts.86.up_proj", "model.layers.85.mlp.experts.87.up_proj", "model.layers.85.mlp.experts.88.up_proj", "model.layers.85.mlp.experts.89.up_proj", "model.layers.85.mlp.experts.90.up_proj", "model.layers.85.mlp.experts.91.up_proj", "model.layers.85.mlp.experts.92.up_proj", "model.layers.85.mlp.experts.93.up_proj", "model.layers.85.mlp.experts.94.up_proj", "model.layers.85.mlp.experts.95.up_proj", "model.layers.85.mlp.experts.96.up_proj", "model.layers.85.mlp.experts.97.up_proj", "model.layers.85.mlp.experts.98.up_proj", "model.layers.85.mlp.experts.99.up_proj", "model.layers.85.mlp.experts.100.up_proj", "model.layers.85.mlp.experts.101.up_proj", "model.layers.85.mlp.experts.102.up_proj", "model.layers.85.mlp.experts.103.up_proj", "model.layers.85.mlp.experts.104.up_proj", "model.layers.85.mlp.experts.105.up_proj", "model.layers.85.mlp.experts.106.up_proj", "model.layers.85.mlp.experts.107.up_proj", "model.layers.85.mlp.experts.108.up_proj", "model.layers.85.mlp.experts.109.up_proj", "model.layers.85.mlp.experts.110.up_proj", "model.layers.85.mlp.experts.111.up_proj", "model.layers.85.mlp.experts.112.up_proj", "model.layers.85.mlp.experts.113.up_proj", "model.layers.85.mlp.experts.114.up_proj", "model.layers.85.mlp.experts.115.up_proj", "model.layers.85.mlp.experts.116.up_proj", "model.layers.85.mlp.experts.117.up_proj", "model.layers.85.mlp.experts.118.up_proj", "model.layers.85.mlp.experts.119.up_proj", "model.layers.85.mlp.experts.120.up_proj", "model.layers.85.mlp.experts.121.up_proj", "model.layers.85.mlp.experts.122.up_proj", "model.layers.85.mlp.experts.123.up_proj", "model.layers.85.mlp.experts.124.up_proj", "model.layers.85.mlp.experts.125.up_proj", "model.layers.85.mlp.experts.126.up_proj", "model.layers.85.mlp.experts.127.up_proj", "model.layers.85.mlp.experts.128.up_proj", "model.layers.85.mlp.experts.129.up_proj", "model.layers.85.mlp.experts.130.up_proj", "model.layers.85.mlp.experts.131.up_proj", "model.layers.85.mlp.experts.132.up_proj", "model.layers.85.mlp.experts.133.up_proj", "model.layers.85.mlp.experts.134.up_proj", "model.layers.85.mlp.experts.135.up_proj", "model.layers.85.mlp.experts.136.up_proj", "model.layers.85.mlp.experts.137.up_proj", "model.layers.85.mlp.experts.138.up_proj", "model.layers.85.mlp.experts.139.up_proj", "model.layers.85.mlp.experts.140.up_proj", "model.layers.85.mlp.experts.141.up_proj", "model.layers.85.mlp.experts.142.up_proj", "model.layers.85.mlp.experts.143.up_proj", "model.layers.85.mlp.experts.144.up_proj", "model.layers.85.mlp.experts.145.up_proj", "model.layers.85.mlp.experts.146.up_proj", "model.layers.85.mlp.experts.147.up_proj", "model.layers.85.mlp.experts.148.up_proj", "model.layers.85.mlp.experts.149.up_proj", "model.layers.85.mlp.experts.150.up_proj", "model.layers.85.mlp.experts.151.up_proj", "model.layers.85.mlp.experts.152.up_proj", "model.layers.85.mlp.experts.153.up_proj", "model.layers.85.mlp.experts.154.up_proj", "model.layers.85.mlp.experts.155.up_proj", "model.layers.85.mlp.experts.156.up_proj", "model.layers.85.mlp.experts.157.up_proj", "model.layers.85.mlp.experts.158.up_proj", "model.layers.85.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00040474608540536083, "dbits": 2516582400 } ] }, { "idx": 512, "layers": [ "model.layers.85.mlp.experts.0.down_proj", "model.layers.85.mlp.experts.1.down_proj", "model.layers.85.mlp.experts.2.down_proj", "model.layers.85.mlp.experts.3.down_proj", "model.layers.85.mlp.experts.4.down_proj", "model.layers.85.mlp.experts.5.down_proj", "model.layers.85.mlp.experts.6.down_proj", "model.layers.85.mlp.experts.7.down_proj", "model.layers.85.mlp.experts.8.down_proj", "model.layers.85.mlp.experts.9.down_proj", "model.layers.85.mlp.experts.10.down_proj", "model.layers.85.mlp.experts.11.down_proj", "model.layers.85.mlp.experts.12.down_proj", "model.layers.85.mlp.experts.13.down_proj", "model.layers.85.mlp.experts.14.down_proj", "model.layers.85.mlp.experts.15.down_proj", "model.layers.85.mlp.experts.16.down_proj", "model.layers.85.mlp.experts.17.down_proj", "model.layers.85.mlp.experts.18.down_proj", "model.layers.85.mlp.experts.19.down_proj", "model.layers.85.mlp.experts.20.down_proj", "model.layers.85.mlp.experts.21.down_proj", "model.layers.85.mlp.experts.22.down_proj", "model.layers.85.mlp.experts.23.down_proj", "model.layers.85.mlp.experts.24.down_proj", "model.layers.85.mlp.experts.25.down_proj", "model.layers.85.mlp.experts.26.down_proj", "model.layers.85.mlp.experts.27.down_proj", "model.layers.85.mlp.experts.28.down_proj", "model.layers.85.mlp.experts.29.down_proj", "model.layers.85.mlp.experts.30.down_proj", "model.layers.85.mlp.experts.31.down_proj", "model.layers.85.mlp.experts.32.down_proj", "model.layers.85.mlp.experts.33.down_proj", "model.layers.85.mlp.experts.34.down_proj", "model.layers.85.mlp.experts.35.down_proj", "model.layers.85.mlp.experts.36.down_proj", "model.layers.85.mlp.experts.37.down_proj", "model.layers.85.mlp.experts.38.down_proj", "model.layers.85.mlp.experts.39.down_proj", "model.layers.85.mlp.experts.40.down_proj", "model.layers.85.mlp.experts.41.down_proj", "model.layers.85.mlp.experts.42.down_proj", "model.layers.85.mlp.experts.43.down_proj", "model.layers.85.mlp.experts.44.down_proj", "model.layers.85.mlp.experts.45.down_proj", "model.layers.85.mlp.experts.46.down_proj", "model.layers.85.mlp.experts.47.down_proj", "model.layers.85.mlp.experts.48.down_proj", "model.layers.85.mlp.experts.49.down_proj", "model.layers.85.mlp.experts.50.down_proj", "model.layers.85.mlp.experts.51.down_proj", "model.layers.85.mlp.experts.52.down_proj", "model.layers.85.mlp.experts.53.down_proj", "model.layers.85.mlp.experts.54.down_proj", "model.layers.85.mlp.experts.55.down_proj", "model.layers.85.mlp.experts.56.down_proj", "model.layers.85.mlp.experts.57.down_proj", "model.layers.85.mlp.experts.58.down_proj", "model.layers.85.mlp.experts.59.down_proj", "model.layers.85.mlp.experts.60.down_proj", "model.layers.85.mlp.experts.61.down_proj", "model.layers.85.mlp.experts.62.down_proj", "model.layers.85.mlp.experts.63.down_proj", "model.layers.85.mlp.experts.64.down_proj", "model.layers.85.mlp.experts.65.down_proj", "model.layers.85.mlp.experts.66.down_proj", "model.layers.85.mlp.experts.67.down_proj", "model.layers.85.mlp.experts.68.down_proj", "model.layers.85.mlp.experts.69.down_proj", "model.layers.85.mlp.experts.70.down_proj", "model.layers.85.mlp.experts.71.down_proj", "model.layers.85.mlp.experts.72.down_proj", "model.layers.85.mlp.experts.73.down_proj", "model.layers.85.mlp.experts.74.down_proj", "model.layers.85.mlp.experts.75.down_proj", "model.layers.85.mlp.experts.76.down_proj", "model.layers.85.mlp.experts.77.down_proj", "model.layers.85.mlp.experts.78.down_proj", "model.layers.85.mlp.experts.79.down_proj", "model.layers.85.mlp.experts.80.down_proj", "model.layers.85.mlp.experts.81.down_proj", "model.layers.85.mlp.experts.82.down_proj", "model.layers.85.mlp.experts.83.down_proj", "model.layers.85.mlp.experts.84.down_proj", "model.layers.85.mlp.experts.85.down_proj", "model.layers.85.mlp.experts.86.down_proj", "model.layers.85.mlp.experts.87.down_proj", "model.layers.85.mlp.experts.88.down_proj", "model.layers.85.mlp.experts.89.down_proj", "model.layers.85.mlp.experts.90.down_proj", "model.layers.85.mlp.experts.91.down_proj", "model.layers.85.mlp.experts.92.down_proj", "model.layers.85.mlp.experts.93.down_proj", "model.layers.85.mlp.experts.94.down_proj", "model.layers.85.mlp.experts.95.down_proj", "model.layers.85.mlp.experts.96.down_proj", "model.layers.85.mlp.experts.97.down_proj", "model.layers.85.mlp.experts.98.down_proj", "model.layers.85.mlp.experts.99.down_proj", "model.layers.85.mlp.experts.100.down_proj", "model.layers.85.mlp.experts.101.down_proj", "model.layers.85.mlp.experts.102.down_proj", "model.layers.85.mlp.experts.103.down_proj", "model.layers.85.mlp.experts.104.down_proj", "model.layers.85.mlp.experts.105.down_proj", "model.layers.85.mlp.experts.106.down_proj", "model.layers.85.mlp.experts.107.down_proj", "model.layers.85.mlp.experts.108.down_proj", "model.layers.85.mlp.experts.109.down_proj", "model.layers.85.mlp.experts.110.down_proj", "model.layers.85.mlp.experts.111.down_proj", "model.layers.85.mlp.experts.112.down_proj", "model.layers.85.mlp.experts.113.down_proj", "model.layers.85.mlp.experts.114.down_proj", "model.layers.85.mlp.experts.115.down_proj", "model.layers.85.mlp.experts.116.down_proj", "model.layers.85.mlp.experts.117.down_proj", "model.layers.85.mlp.experts.118.down_proj", "model.layers.85.mlp.experts.119.down_proj", "model.layers.85.mlp.experts.120.down_proj", "model.layers.85.mlp.experts.121.down_proj", "model.layers.85.mlp.experts.122.down_proj", "model.layers.85.mlp.experts.123.down_proj", "model.layers.85.mlp.experts.124.down_proj", "model.layers.85.mlp.experts.125.down_proj", "model.layers.85.mlp.experts.126.down_proj", "model.layers.85.mlp.experts.127.down_proj", "model.layers.85.mlp.experts.128.down_proj", "model.layers.85.mlp.experts.129.down_proj", "model.layers.85.mlp.experts.130.down_proj", "model.layers.85.mlp.experts.131.down_proj", "model.layers.85.mlp.experts.132.down_proj", "model.layers.85.mlp.experts.133.down_proj", "model.layers.85.mlp.experts.134.down_proj", "model.layers.85.mlp.experts.135.down_proj", "model.layers.85.mlp.experts.136.down_proj", "model.layers.85.mlp.experts.137.down_proj", "model.layers.85.mlp.experts.138.down_proj", "model.layers.85.mlp.experts.139.down_proj", "model.layers.85.mlp.experts.140.down_proj", "model.layers.85.mlp.experts.141.down_proj", "model.layers.85.mlp.experts.142.down_proj", "model.layers.85.mlp.experts.143.down_proj", "model.layers.85.mlp.experts.144.down_proj", "model.layers.85.mlp.experts.145.down_proj", "model.layers.85.mlp.experts.146.down_proj", "model.layers.85.mlp.experts.147.down_proj", "model.layers.85.mlp.experts.148.down_proj", "model.layers.85.mlp.experts.149.down_proj", "model.layers.85.mlp.experts.150.down_proj", "model.layers.85.mlp.experts.151.down_proj", "model.layers.85.mlp.experts.152.down_proj", "model.layers.85.mlp.experts.153.down_proj", "model.layers.85.mlp.experts.154.down_proj", "model.layers.85.mlp.experts.155.down_proj", "model.layers.85.mlp.experts.156.down_proj", "model.layers.85.mlp.experts.157.down_proj", "model.layers.85.mlp.experts.158.down_proj", "model.layers.85.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00029068514704705395, "dbits": 1258291200 } ] }, { "idx": 513, "layers": [ "model.layers.86.self_attn.q_proj" ], "candidates": [ { "dkld": 0.0003734499216079823, "dbits": 62914560 } ] }, { "idx": 514, "layers": [ "model.layers.86.self_attn.k_proj", "model.layers.86.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0006127230823039787, "dbits": 10485760 } ] }, { "idx": 515, "layers": [ "model.layers.86.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0006724983453750277, "dbits": 62914560 } ] }, { "idx": 516, "layers": [ "model.layers.86.mlp.shared_experts.gate_proj", "model.layers.86.mlp.shared_experts.up_proj", "model.layers.86.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0015548311173915752, "dbits": 23592960 } ] }, { "idx": 517, "layers": [ "model.layers.86.mlp.experts.0.gate_proj", "model.layers.86.mlp.experts.1.gate_proj", "model.layers.86.mlp.experts.2.gate_proj", "model.layers.86.mlp.experts.3.gate_proj", "model.layers.86.mlp.experts.4.gate_proj", "model.layers.86.mlp.experts.5.gate_proj", "model.layers.86.mlp.experts.6.gate_proj", "model.layers.86.mlp.experts.7.gate_proj", "model.layers.86.mlp.experts.8.gate_proj", "model.layers.86.mlp.experts.9.gate_proj", "model.layers.86.mlp.experts.10.gate_proj", "model.layers.86.mlp.experts.11.gate_proj", "model.layers.86.mlp.experts.12.gate_proj", "model.layers.86.mlp.experts.13.gate_proj", "model.layers.86.mlp.experts.14.gate_proj", "model.layers.86.mlp.experts.15.gate_proj", "model.layers.86.mlp.experts.16.gate_proj", "model.layers.86.mlp.experts.17.gate_proj", "model.layers.86.mlp.experts.18.gate_proj", "model.layers.86.mlp.experts.19.gate_proj", "model.layers.86.mlp.experts.20.gate_proj", "model.layers.86.mlp.experts.21.gate_proj", "model.layers.86.mlp.experts.22.gate_proj", "model.layers.86.mlp.experts.23.gate_proj", "model.layers.86.mlp.experts.24.gate_proj", "model.layers.86.mlp.experts.25.gate_proj", "model.layers.86.mlp.experts.26.gate_proj", "model.layers.86.mlp.experts.27.gate_proj", "model.layers.86.mlp.experts.28.gate_proj", "model.layers.86.mlp.experts.29.gate_proj", "model.layers.86.mlp.experts.30.gate_proj", "model.layers.86.mlp.experts.31.gate_proj", "model.layers.86.mlp.experts.32.gate_proj", "model.layers.86.mlp.experts.33.gate_proj", "model.layers.86.mlp.experts.34.gate_proj", "model.layers.86.mlp.experts.35.gate_proj", "model.layers.86.mlp.experts.36.gate_proj", "model.layers.86.mlp.experts.37.gate_proj", "model.layers.86.mlp.experts.38.gate_proj", "model.layers.86.mlp.experts.39.gate_proj", "model.layers.86.mlp.experts.40.gate_proj", "model.layers.86.mlp.experts.41.gate_proj", "model.layers.86.mlp.experts.42.gate_proj", "model.layers.86.mlp.experts.43.gate_proj", "model.layers.86.mlp.experts.44.gate_proj", "model.layers.86.mlp.experts.45.gate_proj", "model.layers.86.mlp.experts.46.gate_proj", "model.layers.86.mlp.experts.47.gate_proj", "model.layers.86.mlp.experts.48.gate_proj", "model.layers.86.mlp.experts.49.gate_proj", "model.layers.86.mlp.experts.50.gate_proj", "model.layers.86.mlp.experts.51.gate_proj", "model.layers.86.mlp.experts.52.gate_proj", "model.layers.86.mlp.experts.53.gate_proj", "model.layers.86.mlp.experts.54.gate_proj", "model.layers.86.mlp.experts.55.gate_proj", "model.layers.86.mlp.experts.56.gate_proj", "model.layers.86.mlp.experts.57.gate_proj", "model.layers.86.mlp.experts.58.gate_proj", "model.layers.86.mlp.experts.59.gate_proj", "model.layers.86.mlp.experts.60.gate_proj", "model.layers.86.mlp.experts.61.gate_proj", "model.layers.86.mlp.experts.62.gate_proj", "model.layers.86.mlp.experts.63.gate_proj", "model.layers.86.mlp.experts.64.gate_proj", "model.layers.86.mlp.experts.65.gate_proj", "model.layers.86.mlp.experts.66.gate_proj", "model.layers.86.mlp.experts.67.gate_proj", "model.layers.86.mlp.experts.68.gate_proj", "model.layers.86.mlp.experts.69.gate_proj", "model.layers.86.mlp.experts.70.gate_proj", "model.layers.86.mlp.experts.71.gate_proj", "model.layers.86.mlp.experts.72.gate_proj", "model.layers.86.mlp.experts.73.gate_proj", "model.layers.86.mlp.experts.74.gate_proj", "model.layers.86.mlp.experts.75.gate_proj", "model.layers.86.mlp.experts.76.gate_proj", "model.layers.86.mlp.experts.77.gate_proj", "model.layers.86.mlp.experts.78.gate_proj", "model.layers.86.mlp.experts.79.gate_proj", "model.layers.86.mlp.experts.80.gate_proj", "model.layers.86.mlp.experts.81.gate_proj", "model.layers.86.mlp.experts.82.gate_proj", "model.layers.86.mlp.experts.83.gate_proj", "model.layers.86.mlp.experts.84.gate_proj", "model.layers.86.mlp.experts.85.gate_proj", "model.layers.86.mlp.experts.86.gate_proj", "model.layers.86.mlp.experts.87.gate_proj", "model.layers.86.mlp.experts.88.gate_proj", "model.layers.86.mlp.experts.89.gate_proj", "model.layers.86.mlp.experts.90.gate_proj", "model.layers.86.mlp.experts.91.gate_proj", "model.layers.86.mlp.experts.92.gate_proj", "model.layers.86.mlp.experts.93.gate_proj", "model.layers.86.mlp.experts.94.gate_proj", "model.layers.86.mlp.experts.95.gate_proj", "model.layers.86.mlp.experts.96.gate_proj", "model.layers.86.mlp.experts.97.gate_proj", "model.layers.86.mlp.experts.98.gate_proj", "model.layers.86.mlp.experts.99.gate_proj", "model.layers.86.mlp.experts.100.gate_proj", "model.layers.86.mlp.experts.101.gate_proj", "model.layers.86.mlp.experts.102.gate_proj", "model.layers.86.mlp.experts.103.gate_proj", "model.layers.86.mlp.experts.104.gate_proj", "model.layers.86.mlp.experts.105.gate_proj", "model.layers.86.mlp.experts.106.gate_proj", "model.layers.86.mlp.experts.107.gate_proj", "model.layers.86.mlp.experts.108.gate_proj", "model.layers.86.mlp.experts.109.gate_proj", "model.layers.86.mlp.experts.110.gate_proj", "model.layers.86.mlp.experts.111.gate_proj", "model.layers.86.mlp.experts.112.gate_proj", "model.layers.86.mlp.experts.113.gate_proj", "model.layers.86.mlp.experts.114.gate_proj", "model.layers.86.mlp.experts.115.gate_proj", "model.layers.86.mlp.experts.116.gate_proj", "model.layers.86.mlp.experts.117.gate_proj", "model.layers.86.mlp.experts.118.gate_proj", "model.layers.86.mlp.experts.119.gate_proj", "model.layers.86.mlp.experts.120.gate_proj", "model.layers.86.mlp.experts.121.gate_proj", "model.layers.86.mlp.experts.122.gate_proj", "model.layers.86.mlp.experts.123.gate_proj", "model.layers.86.mlp.experts.124.gate_proj", "model.layers.86.mlp.experts.125.gate_proj", "model.layers.86.mlp.experts.126.gate_proj", "model.layers.86.mlp.experts.127.gate_proj", "model.layers.86.mlp.experts.128.gate_proj", "model.layers.86.mlp.experts.129.gate_proj", "model.layers.86.mlp.experts.130.gate_proj", "model.layers.86.mlp.experts.131.gate_proj", "model.layers.86.mlp.experts.132.gate_proj", "model.layers.86.mlp.experts.133.gate_proj", "model.layers.86.mlp.experts.134.gate_proj", "model.layers.86.mlp.experts.135.gate_proj", "model.layers.86.mlp.experts.136.gate_proj", "model.layers.86.mlp.experts.137.gate_proj", "model.layers.86.mlp.experts.138.gate_proj", "model.layers.86.mlp.experts.139.gate_proj", "model.layers.86.mlp.experts.140.gate_proj", "model.layers.86.mlp.experts.141.gate_proj", "model.layers.86.mlp.experts.142.gate_proj", "model.layers.86.mlp.experts.143.gate_proj", "model.layers.86.mlp.experts.144.gate_proj", "model.layers.86.mlp.experts.145.gate_proj", "model.layers.86.mlp.experts.146.gate_proj", "model.layers.86.mlp.experts.147.gate_proj", "model.layers.86.mlp.experts.148.gate_proj", "model.layers.86.mlp.experts.149.gate_proj", "model.layers.86.mlp.experts.150.gate_proj", "model.layers.86.mlp.experts.151.gate_proj", "model.layers.86.mlp.experts.152.gate_proj", "model.layers.86.mlp.experts.153.gate_proj", "model.layers.86.mlp.experts.154.gate_proj", "model.layers.86.mlp.experts.155.gate_proj", "model.layers.86.mlp.experts.156.gate_proj", "model.layers.86.mlp.experts.157.gate_proj", "model.layers.86.mlp.experts.158.gate_proj", "model.layers.86.mlp.experts.159.gate_proj", "model.layers.86.mlp.experts.0.up_proj", "model.layers.86.mlp.experts.1.up_proj", "model.layers.86.mlp.experts.2.up_proj", "model.layers.86.mlp.experts.3.up_proj", "model.layers.86.mlp.experts.4.up_proj", "model.layers.86.mlp.experts.5.up_proj", "model.layers.86.mlp.experts.6.up_proj", "model.layers.86.mlp.experts.7.up_proj", "model.layers.86.mlp.experts.8.up_proj", "model.layers.86.mlp.experts.9.up_proj", "model.layers.86.mlp.experts.10.up_proj", "model.layers.86.mlp.experts.11.up_proj", "model.layers.86.mlp.experts.12.up_proj", "model.layers.86.mlp.experts.13.up_proj", "model.layers.86.mlp.experts.14.up_proj", "model.layers.86.mlp.experts.15.up_proj", "model.layers.86.mlp.experts.16.up_proj", "model.layers.86.mlp.experts.17.up_proj", "model.layers.86.mlp.experts.18.up_proj", "model.layers.86.mlp.experts.19.up_proj", "model.layers.86.mlp.experts.20.up_proj", "model.layers.86.mlp.experts.21.up_proj", "model.layers.86.mlp.experts.22.up_proj", "model.layers.86.mlp.experts.23.up_proj", "model.layers.86.mlp.experts.24.up_proj", "model.layers.86.mlp.experts.25.up_proj", "model.layers.86.mlp.experts.26.up_proj", "model.layers.86.mlp.experts.27.up_proj", "model.layers.86.mlp.experts.28.up_proj", "model.layers.86.mlp.experts.29.up_proj", "model.layers.86.mlp.experts.30.up_proj", "model.layers.86.mlp.experts.31.up_proj", "model.layers.86.mlp.experts.32.up_proj", "model.layers.86.mlp.experts.33.up_proj", "model.layers.86.mlp.experts.34.up_proj", "model.layers.86.mlp.experts.35.up_proj", "model.layers.86.mlp.experts.36.up_proj", "model.layers.86.mlp.experts.37.up_proj", "model.layers.86.mlp.experts.38.up_proj", "model.layers.86.mlp.experts.39.up_proj", "model.layers.86.mlp.experts.40.up_proj", "model.layers.86.mlp.experts.41.up_proj", "model.layers.86.mlp.experts.42.up_proj", "model.layers.86.mlp.experts.43.up_proj", "model.layers.86.mlp.experts.44.up_proj", "model.layers.86.mlp.experts.45.up_proj", "model.layers.86.mlp.experts.46.up_proj", "model.layers.86.mlp.experts.47.up_proj", "model.layers.86.mlp.experts.48.up_proj", "model.layers.86.mlp.experts.49.up_proj", "model.layers.86.mlp.experts.50.up_proj", "model.layers.86.mlp.experts.51.up_proj", "model.layers.86.mlp.experts.52.up_proj", "model.layers.86.mlp.experts.53.up_proj", "model.layers.86.mlp.experts.54.up_proj", "model.layers.86.mlp.experts.55.up_proj", "model.layers.86.mlp.experts.56.up_proj", "model.layers.86.mlp.experts.57.up_proj", "model.layers.86.mlp.experts.58.up_proj", "model.layers.86.mlp.experts.59.up_proj", "model.layers.86.mlp.experts.60.up_proj", "model.layers.86.mlp.experts.61.up_proj", "model.layers.86.mlp.experts.62.up_proj", "model.layers.86.mlp.experts.63.up_proj", "model.layers.86.mlp.experts.64.up_proj", "model.layers.86.mlp.experts.65.up_proj", "model.layers.86.mlp.experts.66.up_proj", "model.layers.86.mlp.experts.67.up_proj", "model.layers.86.mlp.experts.68.up_proj", "model.layers.86.mlp.experts.69.up_proj", "model.layers.86.mlp.experts.70.up_proj", "model.layers.86.mlp.experts.71.up_proj", "model.layers.86.mlp.experts.72.up_proj", "model.layers.86.mlp.experts.73.up_proj", "model.layers.86.mlp.experts.74.up_proj", "model.layers.86.mlp.experts.75.up_proj", "model.layers.86.mlp.experts.76.up_proj", "model.layers.86.mlp.experts.77.up_proj", "model.layers.86.mlp.experts.78.up_proj", "model.layers.86.mlp.experts.79.up_proj", "model.layers.86.mlp.experts.80.up_proj", "model.layers.86.mlp.experts.81.up_proj", "model.layers.86.mlp.experts.82.up_proj", "model.layers.86.mlp.experts.83.up_proj", "model.layers.86.mlp.experts.84.up_proj", "model.layers.86.mlp.experts.85.up_proj", "model.layers.86.mlp.experts.86.up_proj", "model.layers.86.mlp.experts.87.up_proj", "model.layers.86.mlp.experts.88.up_proj", "model.layers.86.mlp.experts.89.up_proj", "model.layers.86.mlp.experts.90.up_proj", "model.layers.86.mlp.experts.91.up_proj", "model.layers.86.mlp.experts.92.up_proj", "model.layers.86.mlp.experts.93.up_proj", "model.layers.86.mlp.experts.94.up_proj", "model.layers.86.mlp.experts.95.up_proj", "model.layers.86.mlp.experts.96.up_proj", "model.layers.86.mlp.experts.97.up_proj", "model.layers.86.mlp.experts.98.up_proj", "model.layers.86.mlp.experts.99.up_proj", "model.layers.86.mlp.experts.100.up_proj", "model.layers.86.mlp.experts.101.up_proj", "model.layers.86.mlp.experts.102.up_proj", "model.layers.86.mlp.experts.103.up_proj", "model.layers.86.mlp.experts.104.up_proj", "model.layers.86.mlp.experts.105.up_proj", "model.layers.86.mlp.experts.106.up_proj", "model.layers.86.mlp.experts.107.up_proj", "model.layers.86.mlp.experts.108.up_proj", "model.layers.86.mlp.experts.109.up_proj", "model.layers.86.mlp.experts.110.up_proj", "model.layers.86.mlp.experts.111.up_proj", "model.layers.86.mlp.experts.112.up_proj", "model.layers.86.mlp.experts.113.up_proj", "model.layers.86.mlp.experts.114.up_proj", "model.layers.86.mlp.experts.115.up_proj", "model.layers.86.mlp.experts.116.up_proj", "model.layers.86.mlp.experts.117.up_proj", "model.layers.86.mlp.experts.118.up_proj", "model.layers.86.mlp.experts.119.up_proj", "model.layers.86.mlp.experts.120.up_proj", "model.layers.86.mlp.experts.121.up_proj", "model.layers.86.mlp.experts.122.up_proj", "model.layers.86.mlp.experts.123.up_proj", "model.layers.86.mlp.experts.124.up_proj", "model.layers.86.mlp.experts.125.up_proj", "model.layers.86.mlp.experts.126.up_proj", "model.layers.86.mlp.experts.127.up_proj", "model.layers.86.mlp.experts.128.up_proj", "model.layers.86.mlp.experts.129.up_proj", "model.layers.86.mlp.experts.130.up_proj", "model.layers.86.mlp.experts.131.up_proj", "model.layers.86.mlp.experts.132.up_proj", "model.layers.86.mlp.experts.133.up_proj", "model.layers.86.mlp.experts.134.up_proj", "model.layers.86.mlp.experts.135.up_proj", "model.layers.86.mlp.experts.136.up_proj", "model.layers.86.mlp.experts.137.up_proj", "model.layers.86.mlp.experts.138.up_proj", "model.layers.86.mlp.experts.139.up_proj", "model.layers.86.mlp.experts.140.up_proj", "model.layers.86.mlp.experts.141.up_proj", "model.layers.86.mlp.experts.142.up_proj", "model.layers.86.mlp.experts.143.up_proj", "model.layers.86.mlp.experts.144.up_proj", "model.layers.86.mlp.experts.145.up_proj", "model.layers.86.mlp.experts.146.up_proj", "model.layers.86.mlp.experts.147.up_proj", "model.layers.86.mlp.experts.148.up_proj", "model.layers.86.mlp.experts.149.up_proj", "model.layers.86.mlp.experts.150.up_proj", "model.layers.86.mlp.experts.151.up_proj", "model.layers.86.mlp.experts.152.up_proj", "model.layers.86.mlp.experts.153.up_proj", "model.layers.86.mlp.experts.154.up_proj", "model.layers.86.mlp.experts.155.up_proj", "model.layers.86.mlp.experts.156.up_proj", "model.layers.86.mlp.experts.157.up_proj", "model.layers.86.mlp.experts.158.up_proj", "model.layers.86.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0002853907644748688, "dbits": 2516582400 } ] }, { "idx": 518, "layers": [ "model.layers.86.mlp.experts.0.down_proj", "model.layers.86.mlp.experts.1.down_proj", "model.layers.86.mlp.experts.2.down_proj", "model.layers.86.mlp.experts.3.down_proj", "model.layers.86.mlp.experts.4.down_proj", "model.layers.86.mlp.experts.5.down_proj", "model.layers.86.mlp.experts.6.down_proj", "model.layers.86.mlp.experts.7.down_proj", "model.layers.86.mlp.experts.8.down_proj", "model.layers.86.mlp.experts.9.down_proj", "model.layers.86.mlp.experts.10.down_proj", "model.layers.86.mlp.experts.11.down_proj", "model.layers.86.mlp.experts.12.down_proj", "model.layers.86.mlp.experts.13.down_proj", "model.layers.86.mlp.experts.14.down_proj", "model.layers.86.mlp.experts.15.down_proj", "model.layers.86.mlp.experts.16.down_proj", "model.layers.86.mlp.experts.17.down_proj", "model.layers.86.mlp.experts.18.down_proj", "model.layers.86.mlp.experts.19.down_proj", "model.layers.86.mlp.experts.20.down_proj", "model.layers.86.mlp.experts.21.down_proj", "model.layers.86.mlp.experts.22.down_proj", "model.layers.86.mlp.experts.23.down_proj", "model.layers.86.mlp.experts.24.down_proj", "model.layers.86.mlp.experts.25.down_proj", "model.layers.86.mlp.experts.26.down_proj", "model.layers.86.mlp.experts.27.down_proj", "model.layers.86.mlp.experts.28.down_proj", "model.layers.86.mlp.experts.29.down_proj", "model.layers.86.mlp.experts.30.down_proj", "model.layers.86.mlp.experts.31.down_proj", "model.layers.86.mlp.experts.32.down_proj", "model.layers.86.mlp.experts.33.down_proj", "model.layers.86.mlp.experts.34.down_proj", "model.layers.86.mlp.experts.35.down_proj", "model.layers.86.mlp.experts.36.down_proj", "model.layers.86.mlp.experts.37.down_proj", "model.layers.86.mlp.experts.38.down_proj", "model.layers.86.mlp.experts.39.down_proj", "model.layers.86.mlp.experts.40.down_proj", "model.layers.86.mlp.experts.41.down_proj", "model.layers.86.mlp.experts.42.down_proj", "model.layers.86.mlp.experts.43.down_proj", "model.layers.86.mlp.experts.44.down_proj", "model.layers.86.mlp.experts.45.down_proj", "model.layers.86.mlp.experts.46.down_proj", "model.layers.86.mlp.experts.47.down_proj", "model.layers.86.mlp.experts.48.down_proj", "model.layers.86.mlp.experts.49.down_proj", "model.layers.86.mlp.experts.50.down_proj", "model.layers.86.mlp.experts.51.down_proj", "model.layers.86.mlp.experts.52.down_proj", "model.layers.86.mlp.experts.53.down_proj", "model.layers.86.mlp.experts.54.down_proj", "model.layers.86.mlp.experts.55.down_proj", "model.layers.86.mlp.experts.56.down_proj", "model.layers.86.mlp.experts.57.down_proj", "model.layers.86.mlp.experts.58.down_proj", "model.layers.86.mlp.experts.59.down_proj", "model.layers.86.mlp.experts.60.down_proj", "model.layers.86.mlp.experts.61.down_proj", "model.layers.86.mlp.experts.62.down_proj", "model.layers.86.mlp.experts.63.down_proj", "model.layers.86.mlp.experts.64.down_proj", "model.layers.86.mlp.experts.65.down_proj", "model.layers.86.mlp.experts.66.down_proj", "model.layers.86.mlp.experts.67.down_proj", "model.layers.86.mlp.experts.68.down_proj", "model.layers.86.mlp.experts.69.down_proj", "model.layers.86.mlp.experts.70.down_proj", "model.layers.86.mlp.experts.71.down_proj", "model.layers.86.mlp.experts.72.down_proj", "model.layers.86.mlp.experts.73.down_proj", "model.layers.86.mlp.experts.74.down_proj", "model.layers.86.mlp.experts.75.down_proj", "model.layers.86.mlp.experts.76.down_proj", "model.layers.86.mlp.experts.77.down_proj", "model.layers.86.mlp.experts.78.down_proj", "model.layers.86.mlp.experts.79.down_proj", "model.layers.86.mlp.experts.80.down_proj", "model.layers.86.mlp.experts.81.down_proj", "model.layers.86.mlp.experts.82.down_proj", "model.layers.86.mlp.experts.83.down_proj", "model.layers.86.mlp.experts.84.down_proj", "model.layers.86.mlp.experts.85.down_proj", "model.layers.86.mlp.experts.86.down_proj", "model.layers.86.mlp.experts.87.down_proj", "model.layers.86.mlp.experts.88.down_proj", "model.layers.86.mlp.experts.89.down_proj", "model.layers.86.mlp.experts.90.down_proj", "model.layers.86.mlp.experts.91.down_proj", "model.layers.86.mlp.experts.92.down_proj", "model.layers.86.mlp.experts.93.down_proj", "model.layers.86.mlp.experts.94.down_proj", "model.layers.86.mlp.experts.95.down_proj", "model.layers.86.mlp.experts.96.down_proj", "model.layers.86.mlp.experts.97.down_proj", "model.layers.86.mlp.experts.98.down_proj", "model.layers.86.mlp.experts.99.down_proj", "model.layers.86.mlp.experts.100.down_proj", "model.layers.86.mlp.experts.101.down_proj", "model.layers.86.mlp.experts.102.down_proj", "model.layers.86.mlp.experts.103.down_proj", "model.layers.86.mlp.experts.104.down_proj", "model.layers.86.mlp.experts.105.down_proj", "model.layers.86.mlp.experts.106.down_proj", "model.layers.86.mlp.experts.107.down_proj", "model.layers.86.mlp.experts.108.down_proj", "model.layers.86.mlp.experts.109.down_proj", "model.layers.86.mlp.experts.110.down_proj", "model.layers.86.mlp.experts.111.down_proj", "model.layers.86.mlp.experts.112.down_proj", "model.layers.86.mlp.experts.113.down_proj", "model.layers.86.mlp.experts.114.down_proj", "model.layers.86.mlp.experts.115.down_proj", "model.layers.86.mlp.experts.116.down_proj", "model.layers.86.mlp.experts.117.down_proj", "model.layers.86.mlp.experts.118.down_proj", "model.layers.86.mlp.experts.119.down_proj", "model.layers.86.mlp.experts.120.down_proj", "model.layers.86.mlp.experts.121.down_proj", "model.layers.86.mlp.experts.122.down_proj", "model.layers.86.mlp.experts.123.down_proj", "model.layers.86.mlp.experts.124.down_proj", "model.layers.86.mlp.experts.125.down_proj", "model.layers.86.mlp.experts.126.down_proj", "model.layers.86.mlp.experts.127.down_proj", "model.layers.86.mlp.experts.128.down_proj", "model.layers.86.mlp.experts.129.down_proj", "model.layers.86.mlp.experts.130.down_proj", "model.layers.86.mlp.experts.131.down_proj", "model.layers.86.mlp.experts.132.down_proj", "model.layers.86.mlp.experts.133.down_proj", "model.layers.86.mlp.experts.134.down_proj", "model.layers.86.mlp.experts.135.down_proj", "model.layers.86.mlp.experts.136.down_proj", "model.layers.86.mlp.experts.137.down_proj", "model.layers.86.mlp.experts.138.down_proj", "model.layers.86.mlp.experts.139.down_proj", "model.layers.86.mlp.experts.140.down_proj", "model.layers.86.mlp.experts.141.down_proj", "model.layers.86.mlp.experts.142.down_proj", "model.layers.86.mlp.experts.143.down_proj", "model.layers.86.mlp.experts.144.down_proj", "model.layers.86.mlp.experts.145.down_proj", "model.layers.86.mlp.experts.146.down_proj", "model.layers.86.mlp.experts.147.down_proj", "model.layers.86.mlp.experts.148.down_proj", "model.layers.86.mlp.experts.149.down_proj", "model.layers.86.mlp.experts.150.down_proj", "model.layers.86.mlp.experts.151.down_proj", "model.layers.86.mlp.experts.152.down_proj", "model.layers.86.mlp.experts.153.down_proj", "model.layers.86.mlp.experts.154.down_proj", "model.layers.86.mlp.experts.155.down_proj", "model.layers.86.mlp.experts.156.down_proj", "model.layers.86.mlp.experts.157.down_proj", "model.layers.86.mlp.experts.158.down_proj", "model.layers.86.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00017153024673460804, "dbits": 1258291200 } ] }, { "idx": 519, "layers": [ "model.layers.87.self_attn.q_proj" ], "candidates": [ { "dkld": -0.001614925265312206, "dbits": 62914560 } ] }, { "idx": 520, "layers": [ "model.layers.87.self_attn.k_proj", "model.layers.87.self_attn.v_proj" ], "candidates": [ { "dkld": 0.0009690538048743869, "dbits": 10485760 } ] }, { "idx": 521, "layers": [ "model.layers.87.self_attn.o_proj" ], "candidates": [ { "dkld": -0.000865030288696278, "dbits": 62914560 } ] }, { "idx": 522, "layers": [ "model.layers.87.mlp.shared_experts.gate_proj", "model.layers.87.mlp.shared_experts.up_proj", "model.layers.87.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0015680618584156258, "dbits": 23592960 } ] }, { "idx": 523, "layers": [ "model.layers.87.mlp.experts.0.gate_proj", "model.layers.87.mlp.experts.1.gate_proj", "model.layers.87.mlp.experts.2.gate_proj", "model.layers.87.mlp.experts.3.gate_proj", "model.layers.87.mlp.experts.4.gate_proj", "model.layers.87.mlp.experts.5.gate_proj", "model.layers.87.mlp.experts.6.gate_proj", "model.layers.87.mlp.experts.7.gate_proj", "model.layers.87.mlp.experts.8.gate_proj", "model.layers.87.mlp.experts.9.gate_proj", "model.layers.87.mlp.experts.10.gate_proj", "model.layers.87.mlp.experts.11.gate_proj", "model.layers.87.mlp.experts.12.gate_proj", "model.layers.87.mlp.experts.13.gate_proj", "model.layers.87.mlp.experts.14.gate_proj", "model.layers.87.mlp.experts.15.gate_proj", "model.layers.87.mlp.experts.16.gate_proj", "model.layers.87.mlp.experts.17.gate_proj", "model.layers.87.mlp.experts.18.gate_proj", "model.layers.87.mlp.experts.19.gate_proj", "model.layers.87.mlp.experts.20.gate_proj", "model.layers.87.mlp.experts.21.gate_proj", "model.layers.87.mlp.experts.22.gate_proj", "model.layers.87.mlp.experts.23.gate_proj", "model.layers.87.mlp.experts.24.gate_proj", "model.layers.87.mlp.experts.25.gate_proj", "model.layers.87.mlp.experts.26.gate_proj", "model.layers.87.mlp.experts.27.gate_proj", "model.layers.87.mlp.experts.28.gate_proj", "model.layers.87.mlp.experts.29.gate_proj", "model.layers.87.mlp.experts.30.gate_proj", "model.layers.87.mlp.experts.31.gate_proj", "model.layers.87.mlp.experts.32.gate_proj", "model.layers.87.mlp.experts.33.gate_proj", "model.layers.87.mlp.experts.34.gate_proj", "model.layers.87.mlp.experts.35.gate_proj", "model.layers.87.mlp.experts.36.gate_proj", "model.layers.87.mlp.experts.37.gate_proj", "model.layers.87.mlp.experts.38.gate_proj", "model.layers.87.mlp.experts.39.gate_proj", "model.layers.87.mlp.experts.40.gate_proj", "model.layers.87.mlp.experts.41.gate_proj", "model.layers.87.mlp.experts.42.gate_proj", "model.layers.87.mlp.experts.43.gate_proj", "model.layers.87.mlp.experts.44.gate_proj", "model.layers.87.mlp.experts.45.gate_proj", "model.layers.87.mlp.experts.46.gate_proj", "model.layers.87.mlp.experts.47.gate_proj", "model.layers.87.mlp.experts.48.gate_proj", "model.layers.87.mlp.experts.49.gate_proj", "model.layers.87.mlp.experts.50.gate_proj", "model.layers.87.mlp.experts.51.gate_proj", "model.layers.87.mlp.experts.52.gate_proj", "model.layers.87.mlp.experts.53.gate_proj", "model.layers.87.mlp.experts.54.gate_proj", "model.layers.87.mlp.experts.55.gate_proj", "model.layers.87.mlp.experts.56.gate_proj", "model.layers.87.mlp.experts.57.gate_proj", "model.layers.87.mlp.experts.58.gate_proj", "model.layers.87.mlp.experts.59.gate_proj", "model.layers.87.mlp.experts.60.gate_proj", "model.layers.87.mlp.experts.61.gate_proj", "model.layers.87.mlp.experts.62.gate_proj", "model.layers.87.mlp.experts.63.gate_proj", "model.layers.87.mlp.experts.64.gate_proj", "model.layers.87.mlp.experts.65.gate_proj", "model.layers.87.mlp.experts.66.gate_proj", "model.layers.87.mlp.experts.67.gate_proj", "model.layers.87.mlp.experts.68.gate_proj", "model.layers.87.mlp.experts.69.gate_proj", "model.layers.87.mlp.experts.70.gate_proj", "model.layers.87.mlp.experts.71.gate_proj", "model.layers.87.mlp.experts.72.gate_proj", "model.layers.87.mlp.experts.73.gate_proj", "model.layers.87.mlp.experts.74.gate_proj", "model.layers.87.mlp.experts.75.gate_proj", "model.layers.87.mlp.experts.76.gate_proj", "model.layers.87.mlp.experts.77.gate_proj", "model.layers.87.mlp.experts.78.gate_proj", "model.layers.87.mlp.experts.79.gate_proj", "model.layers.87.mlp.experts.80.gate_proj", "model.layers.87.mlp.experts.81.gate_proj", "model.layers.87.mlp.experts.82.gate_proj", "model.layers.87.mlp.experts.83.gate_proj", "model.layers.87.mlp.experts.84.gate_proj", "model.layers.87.mlp.experts.85.gate_proj", "model.layers.87.mlp.experts.86.gate_proj", "model.layers.87.mlp.experts.87.gate_proj", "model.layers.87.mlp.experts.88.gate_proj", "model.layers.87.mlp.experts.89.gate_proj", "model.layers.87.mlp.experts.90.gate_proj", "model.layers.87.mlp.experts.91.gate_proj", "model.layers.87.mlp.experts.92.gate_proj", "model.layers.87.mlp.experts.93.gate_proj", "model.layers.87.mlp.experts.94.gate_proj", "model.layers.87.mlp.experts.95.gate_proj", "model.layers.87.mlp.experts.96.gate_proj", "model.layers.87.mlp.experts.97.gate_proj", "model.layers.87.mlp.experts.98.gate_proj", "model.layers.87.mlp.experts.99.gate_proj", "model.layers.87.mlp.experts.100.gate_proj", "model.layers.87.mlp.experts.101.gate_proj", "model.layers.87.mlp.experts.102.gate_proj", "model.layers.87.mlp.experts.103.gate_proj", "model.layers.87.mlp.experts.104.gate_proj", "model.layers.87.mlp.experts.105.gate_proj", "model.layers.87.mlp.experts.106.gate_proj", "model.layers.87.mlp.experts.107.gate_proj", "model.layers.87.mlp.experts.108.gate_proj", "model.layers.87.mlp.experts.109.gate_proj", "model.layers.87.mlp.experts.110.gate_proj", "model.layers.87.mlp.experts.111.gate_proj", "model.layers.87.mlp.experts.112.gate_proj", "model.layers.87.mlp.experts.113.gate_proj", "model.layers.87.mlp.experts.114.gate_proj", "model.layers.87.mlp.experts.115.gate_proj", "model.layers.87.mlp.experts.116.gate_proj", "model.layers.87.mlp.experts.117.gate_proj", "model.layers.87.mlp.experts.118.gate_proj", "model.layers.87.mlp.experts.119.gate_proj", "model.layers.87.mlp.experts.120.gate_proj", "model.layers.87.mlp.experts.121.gate_proj", "model.layers.87.mlp.experts.122.gate_proj", "model.layers.87.mlp.experts.123.gate_proj", "model.layers.87.mlp.experts.124.gate_proj", "model.layers.87.mlp.experts.125.gate_proj", "model.layers.87.mlp.experts.126.gate_proj", "model.layers.87.mlp.experts.127.gate_proj", "model.layers.87.mlp.experts.128.gate_proj", "model.layers.87.mlp.experts.129.gate_proj", "model.layers.87.mlp.experts.130.gate_proj", "model.layers.87.mlp.experts.131.gate_proj", "model.layers.87.mlp.experts.132.gate_proj", "model.layers.87.mlp.experts.133.gate_proj", "model.layers.87.mlp.experts.134.gate_proj", "model.layers.87.mlp.experts.135.gate_proj", "model.layers.87.mlp.experts.136.gate_proj", "model.layers.87.mlp.experts.137.gate_proj", "model.layers.87.mlp.experts.138.gate_proj", "model.layers.87.mlp.experts.139.gate_proj", "model.layers.87.mlp.experts.140.gate_proj", "model.layers.87.mlp.experts.141.gate_proj", "model.layers.87.mlp.experts.142.gate_proj", "model.layers.87.mlp.experts.143.gate_proj", "model.layers.87.mlp.experts.144.gate_proj", "model.layers.87.mlp.experts.145.gate_proj", "model.layers.87.mlp.experts.146.gate_proj", "model.layers.87.mlp.experts.147.gate_proj", "model.layers.87.mlp.experts.148.gate_proj", "model.layers.87.mlp.experts.149.gate_proj", "model.layers.87.mlp.experts.150.gate_proj", "model.layers.87.mlp.experts.151.gate_proj", "model.layers.87.mlp.experts.152.gate_proj", "model.layers.87.mlp.experts.153.gate_proj", "model.layers.87.mlp.experts.154.gate_proj", "model.layers.87.mlp.experts.155.gate_proj", "model.layers.87.mlp.experts.156.gate_proj", "model.layers.87.mlp.experts.157.gate_proj", "model.layers.87.mlp.experts.158.gate_proj", "model.layers.87.mlp.experts.159.gate_proj", "model.layers.87.mlp.experts.0.up_proj", "model.layers.87.mlp.experts.1.up_proj", "model.layers.87.mlp.experts.2.up_proj", "model.layers.87.mlp.experts.3.up_proj", "model.layers.87.mlp.experts.4.up_proj", "model.layers.87.mlp.experts.5.up_proj", "model.layers.87.mlp.experts.6.up_proj", "model.layers.87.mlp.experts.7.up_proj", "model.layers.87.mlp.experts.8.up_proj", "model.layers.87.mlp.experts.9.up_proj", "model.layers.87.mlp.experts.10.up_proj", "model.layers.87.mlp.experts.11.up_proj", "model.layers.87.mlp.experts.12.up_proj", "model.layers.87.mlp.experts.13.up_proj", "model.layers.87.mlp.experts.14.up_proj", "model.layers.87.mlp.experts.15.up_proj", "model.layers.87.mlp.experts.16.up_proj", "model.layers.87.mlp.experts.17.up_proj", "model.layers.87.mlp.experts.18.up_proj", "model.layers.87.mlp.experts.19.up_proj", "model.layers.87.mlp.experts.20.up_proj", "model.layers.87.mlp.experts.21.up_proj", "model.layers.87.mlp.experts.22.up_proj", "model.layers.87.mlp.experts.23.up_proj", "model.layers.87.mlp.experts.24.up_proj", "model.layers.87.mlp.experts.25.up_proj", "model.layers.87.mlp.experts.26.up_proj", "model.layers.87.mlp.experts.27.up_proj", "model.layers.87.mlp.experts.28.up_proj", "model.layers.87.mlp.experts.29.up_proj", "model.layers.87.mlp.experts.30.up_proj", "model.layers.87.mlp.experts.31.up_proj", "model.layers.87.mlp.experts.32.up_proj", "model.layers.87.mlp.experts.33.up_proj", "model.layers.87.mlp.experts.34.up_proj", "model.layers.87.mlp.experts.35.up_proj", "model.layers.87.mlp.experts.36.up_proj", "model.layers.87.mlp.experts.37.up_proj", "model.layers.87.mlp.experts.38.up_proj", "model.layers.87.mlp.experts.39.up_proj", "model.layers.87.mlp.experts.40.up_proj", "model.layers.87.mlp.experts.41.up_proj", "model.layers.87.mlp.experts.42.up_proj", "model.layers.87.mlp.experts.43.up_proj", "model.layers.87.mlp.experts.44.up_proj", "model.layers.87.mlp.experts.45.up_proj", "model.layers.87.mlp.experts.46.up_proj", "model.layers.87.mlp.experts.47.up_proj", "model.layers.87.mlp.experts.48.up_proj", "model.layers.87.mlp.experts.49.up_proj", "model.layers.87.mlp.experts.50.up_proj", "model.layers.87.mlp.experts.51.up_proj", "model.layers.87.mlp.experts.52.up_proj", "model.layers.87.mlp.experts.53.up_proj", "model.layers.87.mlp.experts.54.up_proj", "model.layers.87.mlp.experts.55.up_proj", "model.layers.87.mlp.experts.56.up_proj", "model.layers.87.mlp.experts.57.up_proj", "model.layers.87.mlp.experts.58.up_proj", "model.layers.87.mlp.experts.59.up_proj", "model.layers.87.mlp.experts.60.up_proj", "model.layers.87.mlp.experts.61.up_proj", "model.layers.87.mlp.experts.62.up_proj", "model.layers.87.mlp.experts.63.up_proj", "model.layers.87.mlp.experts.64.up_proj", "model.layers.87.mlp.experts.65.up_proj", "model.layers.87.mlp.experts.66.up_proj", "model.layers.87.mlp.experts.67.up_proj", "model.layers.87.mlp.experts.68.up_proj", "model.layers.87.mlp.experts.69.up_proj", "model.layers.87.mlp.experts.70.up_proj", "model.layers.87.mlp.experts.71.up_proj", "model.layers.87.mlp.experts.72.up_proj", "model.layers.87.mlp.experts.73.up_proj", "model.layers.87.mlp.experts.74.up_proj", "model.layers.87.mlp.experts.75.up_proj", "model.layers.87.mlp.experts.76.up_proj", "model.layers.87.mlp.experts.77.up_proj", "model.layers.87.mlp.experts.78.up_proj", "model.layers.87.mlp.experts.79.up_proj", "model.layers.87.mlp.experts.80.up_proj", "model.layers.87.mlp.experts.81.up_proj", "model.layers.87.mlp.experts.82.up_proj", "model.layers.87.mlp.experts.83.up_proj", "model.layers.87.mlp.experts.84.up_proj", "model.layers.87.mlp.experts.85.up_proj", "model.layers.87.mlp.experts.86.up_proj", "model.layers.87.mlp.experts.87.up_proj", "model.layers.87.mlp.experts.88.up_proj", "model.layers.87.mlp.experts.89.up_proj", "model.layers.87.mlp.experts.90.up_proj", "model.layers.87.mlp.experts.91.up_proj", "model.layers.87.mlp.experts.92.up_proj", "model.layers.87.mlp.experts.93.up_proj", "model.layers.87.mlp.experts.94.up_proj", "model.layers.87.mlp.experts.95.up_proj", "model.layers.87.mlp.experts.96.up_proj", "model.layers.87.mlp.experts.97.up_proj", "model.layers.87.mlp.experts.98.up_proj", "model.layers.87.mlp.experts.99.up_proj", "model.layers.87.mlp.experts.100.up_proj", "model.layers.87.mlp.experts.101.up_proj", "model.layers.87.mlp.experts.102.up_proj", "model.layers.87.mlp.experts.103.up_proj", "model.layers.87.mlp.experts.104.up_proj", "model.layers.87.mlp.experts.105.up_proj", "model.layers.87.mlp.experts.106.up_proj", "model.layers.87.mlp.experts.107.up_proj", "model.layers.87.mlp.experts.108.up_proj", "model.layers.87.mlp.experts.109.up_proj", "model.layers.87.mlp.experts.110.up_proj", "model.layers.87.mlp.experts.111.up_proj", "model.layers.87.mlp.experts.112.up_proj", "model.layers.87.mlp.experts.113.up_proj", "model.layers.87.mlp.experts.114.up_proj", "model.layers.87.mlp.experts.115.up_proj", "model.layers.87.mlp.experts.116.up_proj", "model.layers.87.mlp.experts.117.up_proj", "model.layers.87.mlp.experts.118.up_proj", "model.layers.87.mlp.experts.119.up_proj", "model.layers.87.mlp.experts.120.up_proj", "model.layers.87.mlp.experts.121.up_proj", "model.layers.87.mlp.experts.122.up_proj", "model.layers.87.mlp.experts.123.up_proj", "model.layers.87.mlp.experts.124.up_proj", "model.layers.87.mlp.experts.125.up_proj", "model.layers.87.mlp.experts.126.up_proj", "model.layers.87.mlp.experts.127.up_proj", "model.layers.87.mlp.experts.128.up_proj", "model.layers.87.mlp.experts.129.up_proj", "model.layers.87.mlp.experts.130.up_proj", "model.layers.87.mlp.experts.131.up_proj", "model.layers.87.mlp.experts.132.up_proj", "model.layers.87.mlp.experts.133.up_proj", "model.layers.87.mlp.experts.134.up_proj", "model.layers.87.mlp.experts.135.up_proj", "model.layers.87.mlp.experts.136.up_proj", "model.layers.87.mlp.experts.137.up_proj", "model.layers.87.mlp.experts.138.up_proj", "model.layers.87.mlp.experts.139.up_proj", "model.layers.87.mlp.experts.140.up_proj", "model.layers.87.mlp.experts.141.up_proj", "model.layers.87.mlp.experts.142.up_proj", "model.layers.87.mlp.experts.143.up_proj", "model.layers.87.mlp.experts.144.up_proj", "model.layers.87.mlp.experts.145.up_proj", "model.layers.87.mlp.experts.146.up_proj", "model.layers.87.mlp.experts.147.up_proj", "model.layers.87.mlp.experts.148.up_proj", "model.layers.87.mlp.experts.149.up_proj", "model.layers.87.mlp.experts.150.up_proj", "model.layers.87.mlp.experts.151.up_proj", "model.layers.87.mlp.experts.152.up_proj", "model.layers.87.mlp.experts.153.up_proj", "model.layers.87.mlp.experts.154.up_proj", "model.layers.87.mlp.experts.155.up_proj", "model.layers.87.mlp.experts.156.up_proj", "model.layers.87.mlp.experts.157.up_proj", "model.layers.87.mlp.experts.158.up_proj", "model.layers.87.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00046596601605414234, "dbits": 2516582400 } ] }, { "idx": 524, "layers": [ "model.layers.87.mlp.experts.0.down_proj", "model.layers.87.mlp.experts.1.down_proj", "model.layers.87.mlp.experts.2.down_proj", "model.layers.87.mlp.experts.3.down_proj", "model.layers.87.mlp.experts.4.down_proj", "model.layers.87.mlp.experts.5.down_proj", "model.layers.87.mlp.experts.6.down_proj", "model.layers.87.mlp.experts.7.down_proj", "model.layers.87.mlp.experts.8.down_proj", "model.layers.87.mlp.experts.9.down_proj", "model.layers.87.mlp.experts.10.down_proj", "model.layers.87.mlp.experts.11.down_proj", "model.layers.87.mlp.experts.12.down_proj", "model.layers.87.mlp.experts.13.down_proj", "model.layers.87.mlp.experts.14.down_proj", "model.layers.87.mlp.experts.15.down_proj", "model.layers.87.mlp.experts.16.down_proj", "model.layers.87.mlp.experts.17.down_proj", "model.layers.87.mlp.experts.18.down_proj", "model.layers.87.mlp.experts.19.down_proj", "model.layers.87.mlp.experts.20.down_proj", "model.layers.87.mlp.experts.21.down_proj", "model.layers.87.mlp.experts.22.down_proj", "model.layers.87.mlp.experts.23.down_proj", "model.layers.87.mlp.experts.24.down_proj", "model.layers.87.mlp.experts.25.down_proj", "model.layers.87.mlp.experts.26.down_proj", "model.layers.87.mlp.experts.27.down_proj", "model.layers.87.mlp.experts.28.down_proj", "model.layers.87.mlp.experts.29.down_proj", "model.layers.87.mlp.experts.30.down_proj", "model.layers.87.mlp.experts.31.down_proj", "model.layers.87.mlp.experts.32.down_proj", "model.layers.87.mlp.experts.33.down_proj", "model.layers.87.mlp.experts.34.down_proj", "model.layers.87.mlp.experts.35.down_proj", "model.layers.87.mlp.experts.36.down_proj", "model.layers.87.mlp.experts.37.down_proj", "model.layers.87.mlp.experts.38.down_proj", "model.layers.87.mlp.experts.39.down_proj", "model.layers.87.mlp.experts.40.down_proj", "model.layers.87.mlp.experts.41.down_proj", "model.layers.87.mlp.experts.42.down_proj", "model.layers.87.mlp.experts.43.down_proj", "model.layers.87.mlp.experts.44.down_proj", "model.layers.87.mlp.experts.45.down_proj", "model.layers.87.mlp.experts.46.down_proj", "model.layers.87.mlp.experts.47.down_proj", "model.layers.87.mlp.experts.48.down_proj", "model.layers.87.mlp.experts.49.down_proj", "model.layers.87.mlp.experts.50.down_proj", "model.layers.87.mlp.experts.51.down_proj", "model.layers.87.mlp.experts.52.down_proj", "model.layers.87.mlp.experts.53.down_proj", "model.layers.87.mlp.experts.54.down_proj", "model.layers.87.mlp.experts.55.down_proj", "model.layers.87.mlp.experts.56.down_proj", "model.layers.87.mlp.experts.57.down_proj", "model.layers.87.mlp.experts.58.down_proj", "model.layers.87.mlp.experts.59.down_proj", "model.layers.87.mlp.experts.60.down_proj", "model.layers.87.mlp.experts.61.down_proj", "model.layers.87.mlp.experts.62.down_proj", "model.layers.87.mlp.experts.63.down_proj", "model.layers.87.mlp.experts.64.down_proj", "model.layers.87.mlp.experts.65.down_proj", "model.layers.87.mlp.experts.66.down_proj", "model.layers.87.mlp.experts.67.down_proj", "model.layers.87.mlp.experts.68.down_proj", "model.layers.87.mlp.experts.69.down_proj", "model.layers.87.mlp.experts.70.down_proj", "model.layers.87.mlp.experts.71.down_proj", "model.layers.87.mlp.experts.72.down_proj", "model.layers.87.mlp.experts.73.down_proj", "model.layers.87.mlp.experts.74.down_proj", "model.layers.87.mlp.experts.75.down_proj", "model.layers.87.mlp.experts.76.down_proj", "model.layers.87.mlp.experts.77.down_proj", "model.layers.87.mlp.experts.78.down_proj", "model.layers.87.mlp.experts.79.down_proj", "model.layers.87.mlp.experts.80.down_proj", "model.layers.87.mlp.experts.81.down_proj", "model.layers.87.mlp.experts.82.down_proj", "model.layers.87.mlp.experts.83.down_proj", "model.layers.87.mlp.experts.84.down_proj", "model.layers.87.mlp.experts.85.down_proj", "model.layers.87.mlp.experts.86.down_proj", "model.layers.87.mlp.experts.87.down_proj", "model.layers.87.mlp.experts.88.down_proj", "model.layers.87.mlp.experts.89.down_proj", "model.layers.87.mlp.experts.90.down_proj", "model.layers.87.mlp.experts.91.down_proj", "model.layers.87.mlp.experts.92.down_proj", "model.layers.87.mlp.experts.93.down_proj", "model.layers.87.mlp.experts.94.down_proj", "model.layers.87.mlp.experts.95.down_proj", "model.layers.87.mlp.experts.96.down_proj", "model.layers.87.mlp.experts.97.down_proj", "model.layers.87.mlp.experts.98.down_proj", "model.layers.87.mlp.experts.99.down_proj", "model.layers.87.mlp.experts.100.down_proj", "model.layers.87.mlp.experts.101.down_proj", "model.layers.87.mlp.experts.102.down_proj", "model.layers.87.mlp.experts.103.down_proj", "model.layers.87.mlp.experts.104.down_proj", "model.layers.87.mlp.experts.105.down_proj", "model.layers.87.mlp.experts.106.down_proj", "model.layers.87.mlp.experts.107.down_proj", "model.layers.87.mlp.experts.108.down_proj", "model.layers.87.mlp.experts.109.down_proj", "model.layers.87.mlp.experts.110.down_proj", "model.layers.87.mlp.experts.111.down_proj", "model.layers.87.mlp.experts.112.down_proj", "model.layers.87.mlp.experts.113.down_proj", "model.layers.87.mlp.experts.114.down_proj", "model.layers.87.mlp.experts.115.down_proj", "model.layers.87.mlp.experts.116.down_proj", "model.layers.87.mlp.experts.117.down_proj", "model.layers.87.mlp.experts.118.down_proj", "model.layers.87.mlp.experts.119.down_proj", "model.layers.87.mlp.experts.120.down_proj", "model.layers.87.mlp.experts.121.down_proj", "model.layers.87.mlp.experts.122.down_proj", "model.layers.87.mlp.experts.123.down_proj", "model.layers.87.mlp.experts.124.down_proj", "model.layers.87.mlp.experts.125.down_proj", "model.layers.87.mlp.experts.126.down_proj", "model.layers.87.mlp.experts.127.down_proj", "model.layers.87.mlp.experts.128.down_proj", "model.layers.87.mlp.experts.129.down_proj", "model.layers.87.mlp.experts.130.down_proj", "model.layers.87.mlp.experts.131.down_proj", "model.layers.87.mlp.experts.132.down_proj", "model.layers.87.mlp.experts.133.down_proj", "model.layers.87.mlp.experts.134.down_proj", "model.layers.87.mlp.experts.135.down_proj", "model.layers.87.mlp.experts.136.down_proj", "model.layers.87.mlp.experts.137.down_proj", "model.layers.87.mlp.experts.138.down_proj", "model.layers.87.mlp.experts.139.down_proj", "model.layers.87.mlp.experts.140.down_proj", "model.layers.87.mlp.experts.141.down_proj", "model.layers.87.mlp.experts.142.down_proj", "model.layers.87.mlp.experts.143.down_proj", "model.layers.87.mlp.experts.144.down_proj", "model.layers.87.mlp.experts.145.down_proj", "model.layers.87.mlp.experts.146.down_proj", "model.layers.87.mlp.experts.147.down_proj", "model.layers.87.mlp.experts.148.down_proj", "model.layers.87.mlp.experts.149.down_proj", "model.layers.87.mlp.experts.150.down_proj", "model.layers.87.mlp.experts.151.down_proj", "model.layers.87.mlp.experts.152.down_proj", "model.layers.87.mlp.experts.153.down_proj", "model.layers.87.mlp.experts.154.down_proj", "model.layers.87.mlp.experts.155.down_proj", "model.layers.87.mlp.experts.156.down_proj", "model.layers.87.mlp.experts.157.down_proj", "model.layers.87.mlp.experts.158.down_proj", "model.layers.87.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00030545368790629857, "dbits": 1258291200 } ] }, { "idx": 525, "layers": [ "model.layers.88.self_attn.q_proj" ], "candidates": [ { "dkld": -3.467351198199742e-05, "dbits": 62914560 } ] }, { "idx": 526, "layers": [ "model.layers.88.self_attn.k_proj", "model.layers.88.self_attn.v_proj" ], "candidates": [ { "dkld": 0.00033947229385372646, "dbits": 10485760 } ] }, { "idx": 527, "layers": [ "model.layers.88.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0009750284254551045, "dbits": 62914560 } ] }, { "idx": 528, "layers": [ "model.layers.88.mlp.shared_experts.gate_proj", "model.layers.88.mlp.shared_experts.up_proj", "model.layers.88.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0022927813231944927, "dbits": 23592960 } ] }, { "idx": 529, "layers": [ "model.layers.88.mlp.experts.0.gate_proj", "model.layers.88.mlp.experts.1.gate_proj", "model.layers.88.mlp.experts.2.gate_proj", "model.layers.88.mlp.experts.3.gate_proj", "model.layers.88.mlp.experts.4.gate_proj", "model.layers.88.mlp.experts.5.gate_proj", "model.layers.88.mlp.experts.6.gate_proj", "model.layers.88.mlp.experts.7.gate_proj", "model.layers.88.mlp.experts.8.gate_proj", "model.layers.88.mlp.experts.9.gate_proj", "model.layers.88.mlp.experts.10.gate_proj", "model.layers.88.mlp.experts.11.gate_proj", "model.layers.88.mlp.experts.12.gate_proj", "model.layers.88.mlp.experts.13.gate_proj", "model.layers.88.mlp.experts.14.gate_proj", "model.layers.88.mlp.experts.15.gate_proj", "model.layers.88.mlp.experts.16.gate_proj", "model.layers.88.mlp.experts.17.gate_proj", "model.layers.88.mlp.experts.18.gate_proj", "model.layers.88.mlp.experts.19.gate_proj", "model.layers.88.mlp.experts.20.gate_proj", "model.layers.88.mlp.experts.21.gate_proj", "model.layers.88.mlp.experts.22.gate_proj", "model.layers.88.mlp.experts.23.gate_proj", "model.layers.88.mlp.experts.24.gate_proj", "model.layers.88.mlp.experts.25.gate_proj", "model.layers.88.mlp.experts.26.gate_proj", "model.layers.88.mlp.experts.27.gate_proj", "model.layers.88.mlp.experts.28.gate_proj", "model.layers.88.mlp.experts.29.gate_proj", "model.layers.88.mlp.experts.30.gate_proj", "model.layers.88.mlp.experts.31.gate_proj", "model.layers.88.mlp.experts.32.gate_proj", "model.layers.88.mlp.experts.33.gate_proj", "model.layers.88.mlp.experts.34.gate_proj", "model.layers.88.mlp.experts.35.gate_proj", "model.layers.88.mlp.experts.36.gate_proj", "model.layers.88.mlp.experts.37.gate_proj", "model.layers.88.mlp.experts.38.gate_proj", "model.layers.88.mlp.experts.39.gate_proj", "model.layers.88.mlp.experts.40.gate_proj", "model.layers.88.mlp.experts.41.gate_proj", "model.layers.88.mlp.experts.42.gate_proj", "model.layers.88.mlp.experts.43.gate_proj", "model.layers.88.mlp.experts.44.gate_proj", "model.layers.88.mlp.experts.45.gate_proj", "model.layers.88.mlp.experts.46.gate_proj", "model.layers.88.mlp.experts.47.gate_proj", "model.layers.88.mlp.experts.48.gate_proj", "model.layers.88.mlp.experts.49.gate_proj", "model.layers.88.mlp.experts.50.gate_proj", "model.layers.88.mlp.experts.51.gate_proj", "model.layers.88.mlp.experts.52.gate_proj", "model.layers.88.mlp.experts.53.gate_proj", "model.layers.88.mlp.experts.54.gate_proj", "model.layers.88.mlp.experts.55.gate_proj", "model.layers.88.mlp.experts.56.gate_proj", "model.layers.88.mlp.experts.57.gate_proj", "model.layers.88.mlp.experts.58.gate_proj", "model.layers.88.mlp.experts.59.gate_proj", "model.layers.88.mlp.experts.60.gate_proj", "model.layers.88.mlp.experts.61.gate_proj", "model.layers.88.mlp.experts.62.gate_proj", "model.layers.88.mlp.experts.63.gate_proj", "model.layers.88.mlp.experts.64.gate_proj", "model.layers.88.mlp.experts.65.gate_proj", "model.layers.88.mlp.experts.66.gate_proj", "model.layers.88.mlp.experts.67.gate_proj", "model.layers.88.mlp.experts.68.gate_proj", "model.layers.88.mlp.experts.69.gate_proj", "model.layers.88.mlp.experts.70.gate_proj", "model.layers.88.mlp.experts.71.gate_proj", "model.layers.88.mlp.experts.72.gate_proj", "model.layers.88.mlp.experts.73.gate_proj", "model.layers.88.mlp.experts.74.gate_proj", "model.layers.88.mlp.experts.75.gate_proj", "model.layers.88.mlp.experts.76.gate_proj", "model.layers.88.mlp.experts.77.gate_proj", "model.layers.88.mlp.experts.78.gate_proj", "model.layers.88.mlp.experts.79.gate_proj", "model.layers.88.mlp.experts.80.gate_proj", "model.layers.88.mlp.experts.81.gate_proj", "model.layers.88.mlp.experts.82.gate_proj", "model.layers.88.mlp.experts.83.gate_proj", "model.layers.88.mlp.experts.84.gate_proj", "model.layers.88.mlp.experts.85.gate_proj", "model.layers.88.mlp.experts.86.gate_proj", "model.layers.88.mlp.experts.87.gate_proj", "model.layers.88.mlp.experts.88.gate_proj", "model.layers.88.mlp.experts.89.gate_proj", "model.layers.88.mlp.experts.90.gate_proj", "model.layers.88.mlp.experts.91.gate_proj", "model.layers.88.mlp.experts.92.gate_proj", "model.layers.88.mlp.experts.93.gate_proj", "model.layers.88.mlp.experts.94.gate_proj", "model.layers.88.mlp.experts.95.gate_proj", "model.layers.88.mlp.experts.96.gate_proj", "model.layers.88.mlp.experts.97.gate_proj", "model.layers.88.mlp.experts.98.gate_proj", "model.layers.88.mlp.experts.99.gate_proj", "model.layers.88.mlp.experts.100.gate_proj", "model.layers.88.mlp.experts.101.gate_proj", "model.layers.88.mlp.experts.102.gate_proj", "model.layers.88.mlp.experts.103.gate_proj", "model.layers.88.mlp.experts.104.gate_proj", "model.layers.88.mlp.experts.105.gate_proj", "model.layers.88.mlp.experts.106.gate_proj", "model.layers.88.mlp.experts.107.gate_proj", "model.layers.88.mlp.experts.108.gate_proj", "model.layers.88.mlp.experts.109.gate_proj", "model.layers.88.mlp.experts.110.gate_proj", "model.layers.88.mlp.experts.111.gate_proj", "model.layers.88.mlp.experts.112.gate_proj", "model.layers.88.mlp.experts.113.gate_proj", "model.layers.88.mlp.experts.114.gate_proj", "model.layers.88.mlp.experts.115.gate_proj", "model.layers.88.mlp.experts.116.gate_proj", "model.layers.88.mlp.experts.117.gate_proj", "model.layers.88.mlp.experts.118.gate_proj", "model.layers.88.mlp.experts.119.gate_proj", "model.layers.88.mlp.experts.120.gate_proj", "model.layers.88.mlp.experts.121.gate_proj", "model.layers.88.mlp.experts.122.gate_proj", "model.layers.88.mlp.experts.123.gate_proj", "model.layers.88.mlp.experts.124.gate_proj", "model.layers.88.mlp.experts.125.gate_proj", "model.layers.88.mlp.experts.126.gate_proj", "model.layers.88.mlp.experts.127.gate_proj", "model.layers.88.mlp.experts.128.gate_proj", "model.layers.88.mlp.experts.129.gate_proj", "model.layers.88.mlp.experts.130.gate_proj", "model.layers.88.mlp.experts.131.gate_proj", "model.layers.88.mlp.experts.132.gate_proj", "model.layers.88.mlp.experts.133.gate_proj", "model.layers.88.mlp.experts.134.gate_proj", "model.layers.88.mlp.experts.135.gate_proj", "model.layers.88.mlp.experts.136.gate_proj", "model.layers.88.mlp.experts.137.gate_proj", "model.layers.88.mlp.experts.138.gate_proj", "model.layers.88.mlp.experts.139.gate_proj", "model.layers.88.mlp.experts.140.gate_proj", "model.layers.88.mlp.experts.141.gate_proj", "model.layers.88.mlp.experts.142.gate_proj", "model.layers.88.mlp.experts.143.gate_proj", "model.layers.88.mlp.experts.144.gate_proj", "model.layers.88.mlp.experts.145.gate_proj", "model.layers.88.mlp.experts.146.gate_proj", "model.layers.88.mlp.experts.147.gate_proj", "model.layers.88.mlp.experts.148.gate_proj", "model.layers.88.mlp.experts.149.gate_proj", "model.layers.88.mlp.experts.150.gate_proj", "model.layers.88.mlp.experts.151.gate_proj", "model.layers.88.mlp.experts.152.gate_proj", "model.layers.88.mlp.experts.153.gate_proj", "model.layers.88.mlp.experts.154.gate_proj", "model.layers.88.mlp.experts.155.gate_proj", "model.layers.88.mlp.experts.156.gate_proj", "model.layers.88.mlp.experts.157.gate_proj", "model.layers.88.mlp.experts.158.gate_proj", "model.layers.88.mlp.experts.159.gate_proj", "model.layers.88.mlp.experts.0.up_proj", "model.layers.88.mlp.experts.1.up_proj", "model.layers.88.mlp.experts.2.up_proj", "model.layers.88.mlp.experts.3.up_proj", "model.layers.88.mlp.experts.4.up_proj", "model.layers.88.mlp.experts.5.up_proj", "model.layers.88.mlp.experts.6.up_proj", "model.layers.88.mlp.experts.7.up_proj", "model.layers.88.mlp.experts.8.up_proj", "model.layers.88.mlp.experts.9.up_proj", "model.layers.88.mlp.experts.10.up_proj", "model.layers.88.mlp.experts.11.up_proj", "model.layers.88.mlp.experts.12.up_proj", "model.layers.88.mlp.experts.13.up_proj", "model.layers.88.mlp.experts.14.up_proj", "model.layers.88.mlp.experts.15.up_proj", "model.layers.88.mlp.experts.16.up_proj", "model.layers.88.mlp.experts.17.up_proj", "model.layers.88.mlp.experts.18.up_proj", "model.layers.88.mlp.experts.19.up_proj", "model.layers.88.mlp.experts.20.up_proj", "model.layers.88.mlp.experts.21.up_proj", "model.layers.88.mlp.experts.22.up_proj", "model.layers.88.mlp.experts.23.up_proj", "model.layers.88.mlp.experts.24.up_proj", "model.layers.88.mlp.experts.25.up_proj", "model.layers.88.mlp.experts.26.up_proj", "model.layers.88.mlp.experts.27.up_proj", "model.layers.88.mlp.experts.28.up_proj", "model.layers.88.mlp.experts.29.up_proj", "model.layers.88.mlp.experts.30.up_proj", "model.layers.88.mlp.experts.31.up_proj", "model.layers.88.mlp.experts.32.up_proj", "model.layers.88.mlp.experts.33.up_proj", "model.layers.88.mlp.experts.34.up_proj", "model.layers.88.mlp.experts.35.up_proj", "model.layers.88.mlp.experts.36.up_proj", "model.layers.88.mlp.experts.37.up_proj", "model.layers.88.mlp.experts.38.up_proj", "model.layers.88.mlp.experts.39.up_proj", "model.layers.88.mlp.experts.40.up_proj", "model.layers.88.mlp.experts.41.up_proj", "model.layers.88.mlp.experts.42.up_proj", "model.layers.88.mlp.experts.43.up_proj", "model.layers.88.mlp.experts.44.up_proj", "model.layers.88.mlp.experts.45.up_proj", "model.layers.88.mlp.experts.46.up_proj", "model.layers.88.mlp.experts.47.up_proj", "model.layers.88.mlp.experts.48.up_proj", "model.layers.88.mlp.experts.49.up_proj", "model.layers.88.mlp.experts.50.up_proj", "model.layers.88.mlp.experts.51.up_proj", "model.layers.88.mlp.experts.52.up_proj", "model.layers.88.mlp.experts.53.up_proj", "model.layers.88.mlp.experts.54.up_proj", "model.layers.88.mlp.experts.55.up_proj", "model.layers.88.mlp.experts.56.up_proj", "model.layers.88.mlp.experts.57.up_proj", "model.layers.88.mlp.experts.58.up_proj", "model.layers.88.mlp.experts.59.up_proj", "model.layers.88.mlp.experts.60.up_proj", "model.layers.88.mlp.experts.61.up_proj", "model.layers.88.mlp.experts.62.up_proj", "model.layers.88.mlp.experts.63.up_proj", "model.layers.88.mlp.experts.64.up_proj", "model.layers.88.mlp.experts.65.up_proj", "model.layers.88.mlp.experts.66.up_proj", "model.layers.88.mlp.experts.67.up_proj", "model.layers.88.mlp.experts.68.up_proj", "model.layers.88.mlp.experts.69.up_proj", "model.layers.88.mlp.experts.70.up_proj", "model.layers.88.mlp.experts.71.up_proj", "model.layers.88.mlp.experts.72.up_proj", "model.layers.88.mlp.experts.73.up_proj", "model.layers.88.mlp.experts.74.up_proj", "model.layers.88.mlp.experts.75.up_proj", "model.layers.88.mlp.experts.76.up_proj", "model.layers.88.mlp.experts.77.up_proj", "model.layers.88.mlp.experts.78.up_proj", "model.layers.88.mlp.experts.79.up_proj", "model.layers.88.mlp.experts.80.up_proj", "model.layers.88.mlp.experts.81.up_proj", "model.layers.88.mlp.experts.82.up_proj", "model.layers.88.mlp.experts.83.up_proj", "model.layers.88.mlp.experts.84.up_proj", "model.layers.88.mlp.experts.85.up_proj", "model.layers.88.mlp.experts.86.up_proj", "model.layers.88.mlp.experts.87.up_proj", "model.layers.88.mlp.experts.88.up_proj", "model.layers.88.mlp.experts.89.up_proj", "model.layers.88.mlp.experts.90.up_proj", "model.layers.88.mlp.experts.91.up_proj", "model.layers.88.mlp.experts.92.up_proj", "model.layers.88.mlp.experts.93.up_proj", "model.layers.88.mlp.experts.94.up_proj", "model.layers.88.mlp.experts.95.up_proj", "model.layers.88.mlp.experts.96.up_proj", "model.layers.88.mlp.experts.97.up_proj", "model.layers.88.mlp.experts.98.up_proj", "model.layers.88.mlp.experts.99.up_proj", "model.layers.88.mlp.experts.100.up_proj", "model.layers.88.mlp.experts.101.up_proj", "model.layers.88.mlp.experts.102.up_proj", "model.layers.88.mlp.experts.103.up_proj", "model.layers.88.mlp.experts.104.up_proj", "model.layers.88.mlp.experts.105.up_proj", "model.layers.88.mlp.experts.106.up_proj", "model.layers.88.mlp.experts.107.up_proj", "model.layers.88.mlp.experts.108.up_proj", "model.layers.88.mlp.experts.109.up_proj", "model.layers.88.mlp.experts.110.up_proj", "model.layers.88.mlp.experts.111.up_proj", "model.layers.88.mlp.experts.112.up_proj", "model.layers.88.mlp.experts.113.up_proj", "model.layers.88.mlp.experts.114.up_proj", "model.layers.88.mlp.experts.115.up_proj", "model.layers.88.mlp.experts.116.up_proj", "model.layers.88.mlp.experts.117.up_proj", "model.layers.88.mlp.experts.118.up_proj", "model.layers.88.mlp.experts.119.up_proj", "model.layers.88.mlp.experts.120.up_proj", "model.layers.88.mlp.experts.121.up_proj", "model.layers.88.mlp.experts.122.up_proj", "model.layers.88.mlp.experts.123.up_proj", "model.layers.88.mlp.experts.124.up_proj", "model.layers.88.mlp.experts.125.up_proj", "model.layers.88.mlp.experts.126.up_proj", "model.layers.88.mlp.experts.127.up_proj", "model.layers.88.mlp.experts.128.up_proj", "model.layers.88.mlp.experts.129.up_proj", "model.layers.88.mlp.experts.130.up_proj", "model.layers.88.mlp.experts.131.up_proj", "model.layers.88.mlp.experts.132.up_proj", "model.layers.88.mlp.experts.133.up_proj", "model.layers.88.mlp.experts.134.up_proj", "model.layers.88.mlp.experts.135.up_proj", "model.layers.88.mlp.experts.136.up_proj", "model.layers.88.mlp.experts.137.up_proj", "model.layers.88.mlp.experts.138.up_proj", "model.layers.88.mlp.experts.139.up_proj", "model.layers.88.mlp.experts.140.up_proj", "model.layers.88.mlp.experts.141.up_proj", "model.layers.88.mlp.experts.142.up_proj", "model.layers.88.mlp.experts.143.up_proj", "model.layers.88.mlp.experts.144.up_proj", "model.layers.88.mlp.experts.145.up_proj", "model.layers.88.mlp.experts.146.up_proj", "model.layers.88.mlp.experts.147.up_proj", "model.layers.88.mlp.experts.148.up_proj", "model.layers.88.mlp.experts.149.up_proj", "model.layers.88.mlp.experts.150.up_proj", "model.layers.88.mlp.experts.151.up_proj", "model.layers.88.mlp.experts.152.up_proj", "model.layers.88.mlp.experts.153.up_proj", "model.layers.88.mlp.experts.154.up_proj", "model.layers.88.mlp.experts.155.up_proj", "model.layers.88.mlp.experts.156.up_proj", "model.layers.88.mlp.experts.157.up_proj", "model.layers.88.mlp.experts.158.up_proj", "model.layers.88.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0003477126359939797, "dbits": 2516582400 } ] }, { "idx": 530, "layers": [ "model.layers.88.mlp.experts.0.down_proj", "model.layers.88.mlp.experts.1.down_proj", "model.layers.88.mlp.experts.2.down_proj", "model.layers.88.mlp.experts.3.down_proj", "model.layers.88.mlp.experts.4.down_proj", "model.layers.88.mlp.experts.5.down_proj", "model.layers.88.mlp.experts.6.down_proj", "model.layers.88.mlp.experts.7.down_proj", "model.layers.88.mlp.experts.8.down_proj", "model.layers.88.mlp.experts.9.down_proj", "model.layers.88.mlp.experts.10.down_proj", "model.layers.88.mlp.experts.11.down_proj", "model.layers.88.mlp.experts.12.down_proj", "model.layers.88.mlp.experts.13.down_proj", "model.layers.88.mlp.experts.14.down_proj", "model.layers.88.mlp.experts.15.down_proj", "model.layers.88.mlp.experts.16.down_proj", "model.layers.88.mlp.experts.17.down_proj", "model.layers.88.mlp.experts.18.down_proj", "model.layers.88.mlp.experts.19.down_proj", "model.layers.88.mlp.experts.20.down_proj", "model.layers.88.mlp.experts.21.down_proj", "model.layers.88.mlp.experts.22.down_proj", "model.layers.88.mlp.experts.23.down_proj", "model.layers.88.mlp.experts.24.down_proj", "model.layers.88.mlp.experts.25.down_proj", "model.layers.88.mlp.experts.26.down_proj", "model.layers.88.mlp.experts.27.down_proj", "model.layers.88.mlp.experts.28.down_proj", "model.layers.88.mlp.experts.29.down_proj", "model.layers.88.mlp.experts.30.down_proj", "model.layers.88.mlp.experts.31.down_proj", "model.layers.88.mlp.experts.32.down_proj", "model.layers.88.mlp.experts.33.down_proj", "model.layers.88.mlp.experts.34.down_proj", "model.layers.88.mlp.experts.35.down_proj", "model.layers.88.mlp.experts.36.down_proj", "model.layers.88.mlp.experts.37.down_proj", "model.layers.88.mlp.experts.38.down_proj", "model.layers.88.mlp.experts.39.down_proj", "model.layers.88.mlp.experts.40.down_proj", "model.layers.88.mlp.experts.41.down_proj", "model.layers.88.mlp.experts.42.down_proj", "model.layers.88.mlp.experts.43.down_proj", "model.layers.88.mlp.experts.44.down_proj", "model.layers.88.mlp.experts.45.down_proj", "model.layers.88.mlp.experts.46.down_proj", "model.layers.88.mlp.experts.47.down_proj", "model.layers.88.mlp.experts.48.down_proj", "model.layers.88.mlp.experts.49.down_proj", "model.layers.88.mlp.experts.50.down_proj", "model.layers.88.mlp.experts.51.down_proj", "model.layers.88.mlp.experts.52.down_proj", "model.layers.88.mlp.experts.53.down_proj", "model.layers.88.mlp.experts.54.down_proj", "model.layers.88.mlp.experts.55.down_proj", "model.layers.88.mlp.experts.56.down_proj", "model.layers.88.mlp.experts.57.down_proj", "model.layers.88.mlp.experts.58.down_proj", "model.layers.88.mlp.experts.59.down_proj", "model.layers.88.mlp.experts.60.down_proj", "model.layers.88.mlp.experts.61.down_proj", "model.layers.88.mlp.experts.62.down_proj", "model.layers.88.mlp.experts.63.down_proj", "model.layers.88.mlp.experts.64.down_proj", "model.layers.88.mlp.experts.65.down_proj", "model.layers.88.mlp.experts.66.down_proj", "model.layers.88.mlp.experts.67.down_proj", "model.layers.88.mlp.experts.68.down_proj", "model.layers.88.mlp.experts.69.down_proj", "model.layers.88.mlp.experts.70.down_proj", "model.layers.88.mlp.experts.71.down_proj", "model.layers.88.mlp.experts.72.down_proj", "model.layers.88.mlp.experts.73.down_proj", "model.layers.88.mlp.experts.74.down_proj", "model.layers.88.mlp.experts.75.down_proj", "model.layers.88.mlp.experts.76.down_proj", "model.layers.88.mlp.experts.77.down_proj", "model.layers.88.mlp.experts.78.down_proj", "model.layers.88.mlp.experts.79.down_proj", "model.layers.88.mlp.experts.80.down_proj", "model.layers.88.mlp.experts.81.down_proj", "model.layers.88.mlp.experts.82.down_proj", "model.layers.88.mlp.experts.83.down_proj", "model.layers.88.mlp.experts.84.down_proj", "model.layers.88.mlp.experts.85.down_proj", "model.layers.88.mlp.experts.86.down_proj", "model.layers.88.mlp.experts.87.down_proj", "model.layers.88.mlp.experts.88.down_proj", "model.layers.88.mlp.experts.89.down_proj", "model.layers.88.mlp.experts.90.down_proj", "model.layers.88.mlp.experts.91.down_proj", "model.layers.88.mlp.experts.92.down_proj", "model.layers.88.mlp.experts.93.down_proj", "model.layers.88.mlp.experts.94.down_proj", "model.layers.88.mlp.experts.95.down_proj", "model.layers.88.mlp.experts.96.down_proj", "model.layers.88.mlp.experts.97.down_proj", "model.layers.88.mlp.experts.98.down_proj", "model.layers.88.mlp.experts.99.down_proj", "model.layers.88.mlp.experts.100.down_proj", "model.layers.88.mlp.experts.101.down_proj", "model.layers.88.mlp.experts.102.down_proj", "model.layers.88.mlp.experts.103.down_proj", "model.layers.88.mlp.experts.104.down_proj", "model.layers.88.mlp.experts.105.down_proj", "model.layers.88.mlp.experts.106.down_proj", "model.layers.88.mlp.experts.107.down_proj", "model.layers.88.mlp.experts.108.down_proj", "model.layers.88.mlp.experts.109.down_proj", "model.layers.88.mlp.experts.110.down_proj", "model.layers.88.mlp.experts.111.down_proj", "model.layers.88.mlp.experts.112.down_proj", "model.layers.88.mlp.experts.113.down_proj", "model.layers.88.mlp.experts.114.down_proj", "model.layers.88.mlp.experts.115.down_proj", "model.layers.88.mlp.experts.116.down_proj", "model.layers.88.mlp.experts.117.down_proj", "model.layers.88.mlp.experts.118.down_proj", "model.layers.88.mlp.experts.119.down_proj", "model.layers.88.mlp.experts.120.down_proj", "model.layers.88.mlp.experts.121.down_proj", "model.layers.88.mlp.experts.122.down_proj", "model.layers.88.mlp.experts.123.down_proj", "model.layers.88.mlp.experts.124.down_proj", "model.layers.88.mlp.experts.125.down_proj", "model.layers.88.mlp.experts.126.down_proj", "model.layers.88.mlp.experts.127.down_proj", "model.layers.88.mlp.experts.128.down_proj", "model.layers.88.mlp.experts.129.down_proj", "model.layers.88.mlp.experts.130.down_proj", "model.layers.88.mlp.experts.131.down_proj", "model.layers.88.mlp.experts.132.down_proj", "model.layers.88.mlp.experts.133.down_proj", "model.layers.88.mlp.experts.134.down_proj", "model.layers.88.mlp.experts.135.down_proj", "model.layers.88.mlp.experts.136.down_proj", "model.layers.88.mlp.experts.137.down_proj", "model.layers.88.mlp.experts.138.down_proj", "model.layers.88.mlp.experts.139.down_proj", "model.layers.88.mlp.experts.140.down_proj", "model.layers.88.mlp.experts.141.down_proj", "model.layers.88.mlp.experts.142.down_proj", "model.layers.88.mlp.experts.143.down_proj", "model.layers.88.mlp.experts.144.down_proj", "model.layers.88.mlp.experts.145.down_proj", "model.layers.88.mlp.experts.146.down_proj", "model.layers.88.mlp.experts.147.down_proj", "model.layers.88.mlp.experts.148.down_proj", "model.layers.88.mlp.experts.149.down_proj", "model.layers.88.mlp.experts.150.down_proj", "model.layers.88.mlp.experts.151.down_proj", "model.layers.88.mlp.experts.152.down_proj", "model.layers.88.mlp.experts.153.down_proj", "model.layers.88.mlp.experts.154.down_proj", "model.layers.88.mlp.experts.155.down_proj", "model.layers.88.mlp.experts.156.down_proj", "model.layers.88.mlp.experts.157.down_proj", "model.layers.88.mlp.experts.158.down_proj", "model.layers.88.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0002633705735206604, "dbits": 1258291200 } ] }, { "idx": 531, "layers": [ "model.layers.89.self_attn.q_proj" ], "candidates": [ { "dkld": -0.0005554534494877084, "dbits": 62914560 } ] }, { "idx": 532, "layers": [ "model.layers.89.self_attn.k_proj", "model.layers.89.self_attn.v_proj" ], "candidates": [ { "dkld": -0.009088294208049796, "dbits": 10485760 } ] }, { "idx": 533, "layers": [ "model.layers.89.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0001439228653907776, "dbits": 62914560 } ] }, { "idx": 534, "layers": [ "model.layers.89.mlp.shared_experts.gate_proj", "model.layers.89.mlp.shared_experts.up_proj", "model.layers.89.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.003329025208950065, "dbits": 23592960 } ] }, { "idx": 535, "layers": [ "model.layers.89.mlp.experts.0.gate_proj", "model.layers.89.mlp.experts.1.gate_proj", "model.layers.89.mlp.experts.2.gate_proj", "model.layers.89.mlp.experts.3.gate_proj", "model.layers.89.mlp.experts.4.gate_proj", "model.layers.89.mlp.experts.5.gate_proj", "model.layers.89.mlp.experts.6.gate_proj", "model.layers.89.mlp.experts.7.gate_proj", "model.layers.89.mlp.experts.8.gate_proj", "model.layers.89.mlp.experts.9.gate_proj", "model.layers.89.mlp.experts.10.gate_proj", "model.layers.89.mlp.experts.11.gate_proj", "model.layers.89.mlp.experts.12.gate_proj", "model.layers.89.mlp.experts.13.gate_proj", "model.layers.89.mlp.experts.14.gate_proj", "model.layers.89.mlp.experts.15.gate_proj", "model.layers.89.mlp.experts.16.gate_proj", "model.layers.89.mlp.experts.17.gate_proj", "model.layers.89.mlp.experts.18.gate_proj", "model.layers.89.mlp.experts.19.gate_proj", "model.layers.89.mlp.experts.20.gate_proj", "model.layers.89.mlp.experts.21.gate_proj", "model.layers.89.mlp.experts.22.gate_proj", "model.layers.89.mlp.experts.23.gate_proj", "model.layers.89.mlp.experts.24.gate_proj", "model.layers.89.mlp.experts.25.gate_proj", "model.layers.89.mlp.experts.26.gate_proj", "model.layers.89.mlp.experts.27.gate_proj", "model.layers.89.mlp.experts.28.gate_proj", "model.layers.89.mlp.experts.29.gate_proj", "model.layers.89.mlp.experts.30.gate_proj", "model.layers.89.mlp.experts.31.gate_proj", "model.layers.89.mlp.experts.32.gate_proj", "model.layers.89.mlp.experts.33.gate_proj", "model.layers.89.mlp.experts.34.gate_proj", "model.layers.89.mlp.experts.35.gate_proj", "model.layers.89.mlp.experts.36.gate_proj", "model.layers.89.mlp.experts.37.gate_proj", "model.layers.89.mlp.experts.38.gate_proj", "model.layers.89.mlp.experts.39.gate_proj", "model.layers.89.mlp.experts.40.gate_proj", "model.layers.89.mlp.experts.41.gate_proj", "model.layers.89.mlp.experts.42.gate_proj", "model.layers.89.mlp.experts.43.gate_proj", "model.layers.89.mlp.experts.44.gate_proj", "model.layers.89.mlp.experts.45.gate_proj", "model.layers.89.mlp.experts.46.gate_proj", "model.layers.89.mlp.experts.47.gate_proj", "model.layers.89.mlp.experts.48.gate_proj", "model.layers.89.mlp.experts.49.gate_proj", "model.layers.89.mlp.experts.50.gate_proj", "model.layers.89.mlp.experts.51.gate_proj", "model.layers.89.mlp.experts.52.gate_proj", "model.layers.89.mlp.experts.53.gate_proj", "model.layers.89.mlp.experts.54.gate_proj", "model.layers.89.mlp.experts.55.gate_proj", "model.layers.89.mlp.experts.56.gate_proj", "model.layers.89.mlp.experts.57.gate_proj", "model.layers.89.mlp.experts.58.gate_proj", "model.layers.89.mlp.experts.59.gate_proj", "model.layers.89.mlp.experts.60.gate_proj", "model.layers.89.mlp.experts.61.gate_proj", "model.layers.89.mlp.experts.62.gate_proj", "model.layers.89.mlp.experts.63.gate_proj", "model.layers.89.mlp.experts.64.gate_proj", "model.layers.89.mlp.experts.65.gate_proj", "model.layers.89.mlp.experts.66.gate_proj", "model.layers.89.mlp.experts.67.gate_proj", "model.layers.89.mlp.experts.68.gate_proj", "model.layers.89.mlp.experts.69.gate_proj", "model.layers.89.mlp.experts.70.gate_proj", "model.layers.89.mlp.experts.71.gate_proj", "model.layers.89.mlp.experts.72.gate_proj", "model.layers.89.mlp.experts.73.gate_proj", "model.layers.89.mlp.experts.74.gate_proj", "model.layers.89.mlp.experts.75.gate_proj", "model.layers.89.mlp.experts.76.gate_proj", "model.layers.89.mlp.experts.77.gate_proj", "model.layers.89.mlp.experts.78.gate_proj", "model.layers.89.mlp.experts.79.gate_proj", "model.layers.89.mlp.experts.80.gate_proj", "model.layers.89.mlp.experts.81.gate_proj", "model.layers.89.mlp.experts.82.gate_proj", "model.layers.89.mlp.experts.83.gate_proj", "model.layers.89.mlp.experts.84.gate_proj", "model.layers.89.mlp.experts.85.gate_proj", "model.layers.89.mlp.experts.86.gate_proj", "model.layers.89.mlp.experts.87.gate_proj", "model.layers.89.mlp.experts.88.gate_proj", "model.layers.89.mlp.experts.89.gate_proj", "model.layers.89.mlp.experts.90.gate_proj", "model.layers.89.mlp.experts.91.gate_proj", "model.layers.89.mlp.experts.92.gate_proj", "model.layers.89.mlp.experts.93.gate_proj", "model.layers.89.mlp.experts.94.gate_proj", "model.layers.89.mlp.experts.95.gate_proj", "model.layers.89.mlp.experts.96.gate_proj", "model.layers.89.mlp.experts.97.gate_proj", "model.layers.89.mlp.experts.98.gate_proj", "model.layers.89.mlp.experts.99.gate_proj", "model.layers.89.mlp.experts.100.gate_proj", "model.layers.89.mlp.experts.101.gate_proj", "model.layers.89.mlp.experts.102.gate_proj", "model.layers.89.mlp.experts.103.gate_proj", "model.layers.89.mlp.experts.104.gate_proj", "model.layers.89.mlp.experts.105.gate_proj", "model.layers.89.mlp.experts.106.gate_proj", "model.layers.89.mlp.experts.107.gate_proj", "model.layers.89.mlp.experts.108.gate_proj", "model.layers.89.mlp.experts.109.gate_proj", "model.layers.89.mlp.experts.110.gate_proj", "model.layers.89.mlp.experts.111.gate_proj", "model.layers.89.mlp.experts.112.gate_proj", "model.layers.89.mlp.experts.113.gate_proj", "model.layers.89.mlp.experts.114.gate_proj", "model.layers.89.mlp.experts.115.gate_proj", "model.layers.89.mlp.experts.116.gate_proj", "model.layers.89.mlp.experts.117.gate_proj", "model.layers.89.mlp.experts.118.gate_proj", "model.layers.89.mlp.experts.119.gate_proj", "model.layers.89.mlp.experts.120.gate_proj", "model.layers.89.mlp.experts.121.gate_proj", "model.layers.89.mlp.experts.122.gate_proj", "model.layers.89.mlp.experts.123.gate_proj", "model.layers.89.mlp.experts.124.gate_proj", "model.layers.89.mlp.experts.125.gate_proj", "model.layers.89.mlp.experts.126.gate_proj", "model.layers.89.mlp.experts.127.gate_proj", "model.layers.89.mlp.experts.128.gate_proj", "model.layers.89.mlp.experts.129.gate_proj", "model.layers.89.mlp.experts.130.gate_proj", "model.layers.89.mlp.experts.131.gate_proj", "model.layers.89.mlp.experts.132.gate_proj", "model.layers.89.mlp.experts.133.gate_proj", "model.layers.89.mlp.experts.134.gate_proj", "model.layers.89.mlp.experts.135.gate_proj", "model.layers.89.mlp.experts.136.gate_proj", "model.layers.89.mlp.experts.137.gate_proj", "model.layers.89.mlp.experts.138.gate_proj", "model.layers.89.mlp.experts.139.gate_proj", "model.layers.89.mlp.experts.140.gate_proj", "model.layers.89.mlp.experts.141.gate_proj", "model.layers.89.mlp.experts.142.gate_proj", "model.layers.89.mlp.experts.143.gate_proj", "model.layers.89.mlp.experts.144.gate_proj", "model.layers.89.mlp.experts.145.gate_proj", "model.layers.89.mlp.experts.146.gate_proj", "model.layers.89.mlp.experts.147.gate_proj", "model.layers.89.mlp.experts.148.gate_proj", "model.layers.89.mlp.experts.149.gate_proj", "model.layers.89.mlp.experts.150.gate_proj", "model.layers.89.mlp.experts.151.gate_proj", "model.layers.89.mlp.experts.152.gate_proj", "model.layers.89.mlp.experts.153.gate_proj", "model.layers.89.mlp.experts.154.gate_proj", "model.layers.89.mlp.experts.155.gate_proj", "model.layers.89.mlp.experts.156.gate_proj", "model.layers.89.mlp.experts.157.gate_proj", "model.layers.89.mlp.experts.158.gate_proj", "model.layers.89.mlp.experts.159.gate_proj", "model.layers.89.mlp.experts.0.up_proj", "model.layers.89.mlp.experts.1.up_proj", "model.layers.89.mlp.experts.2.up_proj", "model.layers.89.mlp.experts.3.up_proj", "model.layers.89.mlp.experts.4.up_proj", "model.layers.89.mlp.experts.5.up_proj", "model.layers.89.mlp.experts.6.up_proj", "model.layers.89.mlp.experts.7.up_proj", "model.layers.89.mlp.experts.8.up_proj", "model.layers.89.mlp.experts.9.up_proj", "model.layers.89.mlp.experts.10.up_proj", "model.layers.89.mlp.experts.11.up_proj", "model.layers.89.mlp.experts.12.up_proj", "model.layers.89.mlp.experts.13.up_proj", "model.layers.89.mlp.experts.14.up_proj", "model.layers.89.mlp.experts.15.up_proj", "model.layers.89.mlp.experts.16.up_proj", "model.layers.89.mlp.experts.17.up_proj", "model.layers.89.mlp.experts.18.up_proj", "model.layers.89.mlp.experts.19.up_proj", "model.layers.89.mlp.experts.20.up_proj", "model.layers.89.mlp.experts.21.up_proj", "model.layers.89.mlp.experts.22.up_proj", "model.layers.89.mlp.experts.23.up_proj", "model.layers.89.mlp.experts.24.up_proj", "model.layers.89.mlp.experts.25.up_proj", "model.layers.89.mlp.experts.26.up_proj", "model.layers.89.mlp.experts.27.up_proj", "model.layers.89.mlp.experts.28.up_proj", "model.layers.89.mlp.experts.29.up_proj", "model.layers.89.mlp.experts.30.up_proj", "model.layers.89.mlp.experts.31.up_proj", "model.layers.89.mlp.experts.32.up_proj", "model.layers.89.mlp.experts.33.up_proj", "model.layers.89.mlp.experts.34.up_proj", "model.layers.89.mlp.experts.35.up_proj", "model.layers.89.mlp.experts.36.up_proj", "model.layers.89.mlp.experts.37.up_proj", "model.layers.89.mlp.experts.38.up_proj", "model.layers.89.mlp.experts.39.up_proj", "model.layers.89.mlp.experts.40.up_proj", "model.layers.89.mlp.experts.41.up_proj", "model.layers.89.mlp.experts.42.up_proj", "model.layers.89.mlp.experts.43.up_proj", "model.layers.89.mlp.experts.44.up_proj", "model.layers.89.mlp.experts.45.up_proj", "model.layers.89.mlp.experts.46.up_proj", "model.layers.89.mlp.experts.47.up_proj", "model.layers.89.mlp.experts.48.up_proj", "model.layers.89.mlp.experts.49.up_proj", "model.layers.89.mlp.experts.50.up_proj", "model.layers.89.mlp.experts.51.up_proj", "model.layers.89.mlp.experts.52.up_proj", "model.layers.89.mlp.experts.53.up_proj", "model.layers.89.mlp.experts.54.up_proj", "model.layers.89.mlp.experts.55.up_proj", "model.layers.89.mlp.experts.56.up_proj", "model.layers.89.mlp.experts.57.up_proj", "model.layers.89.mlp.experts.58.up_proj", "model.layers.89.mlp.experts.59.up_proj", "model.layers.89.mlp.experts.60.up_proj", "model.layers.89.mlp.experts.61.up_proj", "model.layers.89.mlp.experts.62.up_proj", "model.layers.89.mlp.experts.63.up_proj", "model.layers.89.mlp.experts.64.up_proj", "model.layers.89.mlp.experts.65.up_proj", "model.layers.89.mlp.experts.66.up_proj", "model.layers.89.mlp.experts.67.up_proj", "model.layers.89.mlp.experts.68.up_proj", "model.layers.89.mlp.experts.69.up_proj", "model.layers.89.mlp.experts.70.up_proj", "model.layers.89.mlp.experts.71.up_proj", "model.layers.89.mlp.experts.72.up_proj", "model.layers.89.mlp.experts.73.up_proj", "model.layers.89.mlp.experts.74.up_proj", "model.layers.89.mlp.experts.75.up_proj", "model.layers.89.mlp.experts.76.up_proj", "model.layers.89.mlp.experts.77.up_proj", "model.layers.89.mlp.experts.78.up_proj", "model.layers.89.mlp.experts.79.up_proj", "model.layers.89.mlp.experts.80.up_proj", "model.layers.89.mlp.experts.81.up_proj", "model.layers.89.mlp.experts.82.up_proj", "model.layers.89.mlp.experts.83.up_proj", "model.layers.89.mlp.experts.84.up_proj", "model.layers.89.mlp.experts.85.up_proj", "model.layers.89.mlp.experts.86.up_proj", "model.layers.89.mlp.experts.87.up_proj", "model.layers.89.mlp.experts.88.up_proj", "model.layers.89.mlp.experts.89.up_proj", "model.layers.89.mlp.experts.90.up_proj", "model.layers.89.mlp.experts.91.up_proj", "model.layers.89.mlp.experts.92.up_proj", "model.layers.89.mlp.experts.93.up_proj", "model.layers.89.mlp.experts.94.up_proj", "model.layers.89.mlp.experts.95.up_proj", "model.layers.89.mlp.experts.96.up_proj", "model.layers.89.mlp.experts.97.up_proj", "model.layers.89.mlp.experts.98.up_proj", "model.layers.89.mlp.experts.99.up_proj", "model.layers.89.mlp.experts.100.up_proj", "model.layers.89.mlp.experts.101.up_proj", "model.layers.89.mlp.experts.102.up_proj", "model.layers.89.mlp.experts.103.up_proj", "model.layers.89.mlp.experts.104.up_proj", "model.layers.89.mlp.experts.105.up_proj", "model.layers.89.mlp.experts.106.up_proj", "model.layers.89.mlp.experts.107.up_proj", "model.layers.89.mlp.experts.108.up_proj", "model.layers.89.mlp.experts.109.up_proj", "model.layers.89.mlp.experts.110.up_proj", "model.layers.89.mlp.experts.111.up_proj", "model.layers.89.mlp.experts.112.up_proj", "model.layers.89.mlp.experts.113.up_proj", "model.layers.89.mlp.experts.114.up_proj", "model.layers.89.mlp.experts.115.up_proj", "model.layers.89.mlp.experts.116.up_proj", "model.layers.89.mlp.experts.117.up_proj", "model.layers.89.mlp.experts.118.up_proj", "model.layers.89.mlp.experts.119.up_proj", "model.layers.89.mlp.experts.120.up_proj", "model.layers.89.mlp.experts.121.up_proj", "model.layers.89.mlp.experts.122.up_proj", "model.layers.89.mlp.experts.123.up_proj", "model.layers.89.mlp.experts.124.up_proj", "model.layers.89.mlp.experts.125.up_proj", "model.layers.89.mlp.experts.126.up_proj", "model.layers.89.mlp.experts.127.up_proj", "model.layers.89.mlp.experts.128.up_proj", "model.layers.89.mlp.experts.129.up_proj", "model.layers.89.mlp.experts.130.up_proj", "model.layers.89.mlp.experts.131.up_proj", "model.layers.89.mlp.experts.132.up_proj", "model.layers.89.mlp.experts.133.up_proj", "model.layers.89.mlp.experts.134.up_proj", "model.layers.89.mlp.experts.135.up_proj", "model.layers.89.mlp.experts.136.up_proj", "model.layers.89.mlp.experts.137.up_proj", "model.layers.89.mlp.experts.138.up_proj", "model.layers.89.mlp.experts.139.up_proj", "model.layers.89.mlp.experts.140.up_proj", "model.layers.89.mlp.experts.141.up_proj", "model.layers.89.mlp.experts.142.up_proj", "model.layers.89.mlp.experts.143.up_proj", "model.layers.89.mlp.experts.144.up_proj", "model.layers.89.mlp.experts.145.up_proj", "model.layers.89.mlp.experts.146.up_proj", "model.layers.89.mlp.experts.147.up_proj", "model.layers.89.mlp.experts.148.up_proj", "model.layers.89.mlp.experts.149.up_proj", "model.layers.89.mlp.experts.150.up_proj", "model.layers.89.mlp.experts.151.up_proj", "model.layers.89.mlp.experts.152.up_proj", "model.layers.89.mlp.experts.153.up_proj", "model.layers.89.mlp.experts.154.up_proj", "model.layers.89.mlp.experts.155.up_proj", "model.layers.89.mlp.experts.156.up_proj", "model.layers.89.mlp.experts.157.up_proj", "model.layers.89.mlp.experts.158.up_proj", "model.layers.89.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00028872787952421985, "dbits": 2516582400 } ] }, { "idx": 536, "layers": [ "model.layers.89.mlp.experts.0.down_proj", "model.layers.89.mlp.experts.1.down_proj", "model.layers.89.mlp.experts.2.down_proj", "model.layers.89.mlp.experts.3.down_proj", "model.layers.89.mlp.experts.4.down_proj", "model.layers.89.mlp.experts.5.down_proj", "model.layers.89.mlp.experts.6.down_proj", "model.layers.89.mlp.experts.7.down_proj", "model.layers.89.mlp.experts.8.down_proj", "model.layers.89.mlp.experts.9.down_proj", "model.layers.89.mlp.experts.10.down_proj", "model.layers.89.mlp.experts.11.down_proj", "model.layers.89.mlp.experts.12.down_proj", "model.layers.89.mlp.experts.13.down_proj", "model.layers.89.mlp.experts.14.down_proj", "model.layers.89.mlp.experts.15.down_proj", "model.layers.89.mlp.experts.16.down_proj", "model.layers.89.mlp.experts.17.down_proj", "model.layers.89.mlp.experts.18.down_proj", "model.layers.89.mlp.experts.19.down_proj", "model.layers.89.mlp.experts.20.down_proj", "model.layers.89.mlp.experts.21.down_proj", "model.layers.89.mlp.experts.22.down_proj", "model.layers.89.mlp.experts.23.down_proj", "model.layers.89.mlp.experts.24.down_proj", "model.layers.89.mlp.experts.25.down_proj", "model.layers.89.mlp.experts.26.down_proj", "model.layers.89.mlp.experts.27.down_proj", "model.layers.89.mlp.experts.28.down_proj", "model.layers.89.mlp.experts.29.down_proj", "model.layers.89.mlp.experts.30.down_proj", "model.layers.89.mlp.experts.31.down_proj", "model.layers.89.mlp.experts.32.down_proj", "model.layers.89.mlp.experts.33.down_proj", "model.layers.89.mlp.experts.34.down_proj", "model.layers.89.mlp.experts.35.down_proj", "model.layers.89.mlp.experts.36.down_proj", "model.layers.89.mlp.experts.37.down_proj", "model.layers.89.mlp.experts.38.down_proj", "model.layers.89.mlp.experts.39.down_proj", "model.layers.89.mlp.experts.40.down_proj", "model.layers.89.mlp.experts.41.down_proj", "model.layers.89.mlp.experts.42.down_proj", "model.layers.89.mlp.experts.43.down_proj", "model.layers.89.mlp.experts.44.down_proj", "model.layers.89.mlp.experts.45.down_proj", "model.layers.89.mlp.experts.46.down_proj", "model.layers.89.mlp.experts.47.down_proj", "model.layers.89.mlp.experts.48.down_proj", "model.layers.89.mlp.experts.49.down_proj", "model.layers.89.mlp.experts.50.down_proj", "model.layers.89.mlp.experts.51.down_proj", "model.layers.89.mlp.experts.52.down_proj", "model.layers.89.mlp.experts.53.down_proj", "model.layers.89.mlp.experts.54.down_proj", "model.layers.89.mlp.experts.55.down_proj", "model.layers.89.mlp.experts.56.down_proj", "model.layers.89.mlp.experts.57.down_proj", "model.layers.89.mlp.experts.58.down_proj", "model.layers.89.mlp.experts.59.down_proj", "model.layers.89.mlp.experts.60.down_proj", "model.layers.89.mlp.experts.61.down_proj", "model.layers.89.mlp.experts.62.down_proj", "model.layers.89.mlp.experts.63.down_proj", "model.layers.89.mlp.experts.64.down_proj", "model.layers.89.mlp.experts.65.down_proj", "model.layers.89.mlp.experts.66.down_proj", "model.layers.89.mlp.experts.67.down_proj", "model.layers.89.mlp.experts.68.down_proj", "model.layers.89.mlp.experts.69.down_proj", "model.layers.89.mlp.experts.70.down_proj", "model.layers.89.mlp.experts.71.down_proj", "model.layers.89.mlp.experts.72.down_proj", "model.layers.89.mlp.experts.73.down_proj", "model.layers.89.mlp.experts.74.down_proj", "model.layers.89.mlp.experts.75.down_proj", "model.layers.89.mlp.experts.76.down_proj", "model.layers.89.mlp.experts.77.down_proj", "model.layers.89.mlp.experts.78.down_proj", "model.layers.89.mlp.experts.79.down_proj", "model.layers.89.mlp.experts.80.down_proj", "model.layers.89.mlp.experts.81.down_proj", "model.layers.89.mlp.experts.82.down_proj", "model.layers.89.mlp.experts.83.down_proj", "model.layers.89.mlp.experts.84.down_proj", "model.layers.89.mlp.experts.85.down_proj", "model.layers.89.mlp.experts.86.down_proj", "model.layers.89.mlp.experts.87.down_proj", "model.layers.89.mlp.experts.88.down_proj", "model.layers.89.mlp.experts.89.down_proj", "model.layers.89.mlp.experts.90.down_proj", "model.layers.89.mlp.experts.91.down_proj", "model.layers.89.mlp.experts.92.down_proj", "model.layers.89.mlp.experts.93.down_proj", "model.layers.89.mlp.experts.94.down_proj", "model.layers.89.mlp.experts.95.down_proj", "model.layers.89.mlp.experts.96.down_proj", "model.layers.89.mlp.experts.97.down_proj", "model.layers.89.mlp.experts.98.down_proj", "model.layers.89.mlp.experts.99.down_proj", "model.layers.89.mlp.experts.100.down_proj", "model.layers.89.mlp.experts.101.down_proj", "model.layers.89.mlp.experts.102.down_proj", "model.layers.89.mlp.experts.103.down_proj", "model.layers.89.mlp.experts.104.down_proj", "model.layers.89.mlp.experts.105.down_proj", "model.layers.89.mlp.experts.106.down_proj", "model.layers.89.mlp.experts.107.down_proj", "model.layers.89.mlp.experts.108.down_proj", "model.layers.89.mlp.experts.109.down_proj", "model.layers.89.mlp.experts.110.down_proj", "model.layers.89.mlp.experts.111.down_proj", "model.layers.89.mlp.experts.112.down_proj", "model.layers.89.mlp.experts.113.down_proj", "model.layers.89.mlp.experts.114.down_proj", "model.layers.89.mlp.experts.115.down_proj", "model.layers.89.mlp.experts.116.down_proj", "model.layers.89.mlp.experts.117.down_proj", "model.layers.89.mlp.experts.118.down_proj", "model.layers.89.mlp.experts.119.down_proj", "model.layers.89.mlp.experts.120.down_proj", "model.layers.89.mlp.experts.121.down_proj", "model.layers.89.mlp.experts.122.down_proj", "model.layers.89.mlp.experts.123.down_proj", "model.layers.89.mlp.experts.124.down_proj", "model.layers.89.mlp.experts.125.down_proj", "model.layers.89.mlp.experts.126.down_proj", "model.layers.89.mlp.experts.127.down_proj", "model.layers.89.mlp.experts.128.down_proj", "model.layers.89.mlp.experts.129.down_proj", "model.layers.89.mlp.experts.130.down_proj", "model.layers.89.mlp.experts.131.down_proj", "model.layers.89.mlp.experts.132.down_proj", "model.layers.89.mlp.experts.133.down_proj", "model.layers.89.mlp.experts.134.down_proj", "model.layers.89.mlp.experts.135.down_proj", "model.layers.89.mlp.experts.136.down_proj", "model.layers.89.mlp.experts.137.down_proj", "model.layers.89.mlp.experts.138.down_proj", "model.layers.89.mlp.experts.139.down_proj", "model.layers.89.mlp.experts.140.down_proj", "model.layers.89.mlp.experts.141.down_proj", "model.layers.89.mlp.experts.142.down_proj", "model.layers.89.mlp.experts.143.down_proj", "model.layers.89.mlp.experts.144.down_proj", "model.layers.89.mlp.experts.145.down_proj", "model.layers.89.mlp.experts.146.down_proj", "model.layers.89.mlp.experts.147.down_proj", "model.layers.89.mlp.experts.148.down_proj", "model.layers.89.mlp.experts.149.down_proj", "model.layers.89.mlp.experts.150.down_proj", "model.layers.89.mlp.experts.151.down_proj", "model.layers.89.mlp.experts.152.down_proj", "model.layers.89.mlp.experts.153.down_proj", "model.layers.89.mlp.experts.154.down_proj", "model.layers.89.mlp.experts.155.down_proj", "model.layers.89.mlp.experts.156.down_proj", "model.layers.89.mlp.experts.157.down_proj", "model.layers.89.mlp.experts.158.down_proj", "model.layers.89.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.00031294673681259155, "dbits": 1258291200 } ] }, { "idx": 537, "layers": [ "model.layers.90.self_attn.q_proj" ], "candidates": [ { "dkld": -0.00046732649207115173, "dbits": 62914560 } ] }, { "idx": 538, "layers": [ "model.layers.90.self_attn.k_proj", "model.layers.90.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0009152919054031261, "dbits": 10485760 } ] }, { "idx": 539, "layers": [ "model.layers.90.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0004885509610176308, "dbits": 62914560 } ] }, { "idx": 540, "layers": [ "model.layers.90.mlp.shared_experts.gate_proj", "model.layers.90.mlp.shared_experts.up_proj", "model.layers.90.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.004040745645761512, "dbits": 23592960 } ] }, { "idx": 541, "layers": [ "model.layers.90.mlp.experts.0.gate_proj", "model.layers.90.mlp.experts.1.gate_proj", "model.layers.90.mlp.experts.2.gate_proj", "model.layers.90.mlp.experts.3.gate_proj", "model.layers.90.mlp.experts.4.gate_proj", "model.layers.90.mlp.experts.5.gate_proj", "model.layers.90.mlp.experts.6.gate_proj", "model.layers.90.mlp.experts.7.gate_proj", "model.layers.90.mlp.experts.8.gate_proj", "model.layers.90.mlp.experts.9.gate_proj", "model.layers.90.mlp.experts.10.gate_proj", "model.layers.90.mlp.experts.11.gate_proj", "model.layers.90.mlp.experts.12.gate_proj", "model.layers.90.mlp.experts.13.gate_proj", "model.layers.90.mlp.experts.14.gate_proj", "model.layers.90.mlp.experts.15.gate_proj", "model.layers.90.mlp.experts.16.gate_proj", "model.layers.90.mlp.experts.17.gate_proj", "model.layers.90.mlp.experts.18.gate_proj", "model.layers.90.mlp.experts.19.gate_proj", "model.layers.90.mlp.experts.20.gate_proj", "model.layers.90.mlp.experts.21.gate_proj", "model.layers.90.mlp.experts.22.gate_proj", "model.layers.90.mlp.experts.23.gate_proj", "model.layers.90.mlp.experts.24.gate_proj", "model.layers.90.mlp.experts.25.gate_proj", "model.layers.90.mlp.experts.26.gate_proj", "model.layers.90.mlp.experts.27.gate_proj", "model.layers.90.mlp.experts.28.gate_proj", "model.layers.90.mlp.experts.29.gate_proj", "model.layers.90.mlp.experts.30.gate_proj", "model.layers.90.mlp.experts.31.gate_proj", "model.layers.90.mlp.experts.32.gate_proj", "model.layers.90.mlp.experts.33.gate_proj", "model.layers.90.mlp.experts.34.gate_proj", "model.layers.90.mlp.experts.35.gate_proj", "model.layers.90.mlp.experts.36.gate_proj", "model.layers.90.mlp.experts.37.gate_proj", "model.layers.90.mlp.experts.38.gate_proj", "model.layers.90.mlp.experts.39.gate_proj", "model.layers.90.mlp.experts.40.gate_proj", "model.layers.90.mlp.experts.41.gate_proj", "model.layers.90.mlp.experts.42.gate_proj", "model.layers.90.mlp.experts.43.gate_proj", "model.layers.90.mlp.experts.44.gate_proj", "model.layers.90.mlp.experts.45.gate_proj", "model.layers.90.mlp.experts.46.gate_proj", "model.layers.90.mlp.experts.47.gate_proj", "model.layers.90.mlp.experts.48.gate_proj", "model.layers.90.mlp.experts.49.gate_proj", "model.layers.90.mlp.experts.50.gate_proj", "model.layers.90.mlp.experts.51.gate_proj", "model.layers.90.mlp.experts.52.gate_proj", "model.layers.90.mlp.experts.53.gate_proj", "model.layers.90.mlp.experts.54.gate_proj", "model.layers.90.mlp.experts.55.gate_proj", "model.layers.90.mlp.experts.56.gate_proj", "model.layers.90.mlp.experts.57.gate_proj", "model.layers.90.mlp.experts.58.gate_proj", "model.layers.90.mlp.experts.59.gate_proj", "model.layers.90.mlp.experts.60.gate_proj", "model.layers.90.mlp.experts.61.gate_proj", "model.layers.90.mlp.experts.62.gate_proj", "model.layers.90.mlp.experts.63.gate_proj", "model.layers.90.mlp.experts.64.gate_proj", "model.layers.90.mlp.experts.65.gate_proj", "model.layers.90.mlp.experts.66.gate_proj", "model.layers.90.mlp.experts.67.gate_proj", "model.layers.90.mlp.experts.68.gate_proj", "model.layers.90.mlp.experts.69.gate_proj", "model.layers.90.mlp.experts.70.gate_proj", "model.layers.90.mlp.experts.71.gate_proj", "model.layers.90.mlp.experts.72.gate_proj", "model.layers.90.mlp.experts.73.gate_proj", "model.layers.90.mlp.experts.74.gate_proj", "model.layers.90.mlp.experts.75.gate_proj", "model.layers.90.mlp.experts.76.gate_proj", "model.layers.90.mlp.experts.77.gate_proj", "model.layers.90.mlp.experts.78.gate_proj", "model.layers.90.mlp.experts.79.gate_proj", "model.layers.90.mlp.experts.80.gate_proj", "model.layers.90.mlp.experts.81.gate_proj", "model.layers.90.mlp.experts.82.gate_proj", "model.layers.90.mlp.experts.83.gate_proj", "model.layers.90.mlp.experts.84.gate_proj", "model.layers.90.mlp.experts.85.gate_proj", "model.layers.90.mlp.experts.86.gate_proj", "model.layers.90.mlp.experts.87.gate_proj", "model.layers.90.mlp.experts.88.gate_proj", "model.layers.90.mlp.experts.89.gate_proj", "model.layers.90.mlp.experts.90.gate_proj", "model.layers.90.mlp.experts.91.gate_proj", "model.layers.90.mlp.experts.92.gate_proj", "model.layers.90.mlp.experts.93.gate_proj", "model.layers.90.mlp.experts.94.gate_proj", "model.layers.90.mlp.experts.95.gate_proj", "model.layers.90.mlp.experts.96.gate_proj", "model.layers.90.mlp.experts.97.gate_proj", "model.layers.90.mlp.experts.98.gate_proj", "model.layers.90.mlp.experts.99.gate_proj", "model.layers.90.mlp.experts.100.gate_proj", "model.layers.90.mlp.experts.101.gate_proj", "model.layers.90.mlp.experts.102.gate_proj", "model.layers.90.mlp.experts.103.gate_proj", "model.layers.90.mlp.experts.104.gate_proj", "model.layers.90.mlp.experts.105.gate_proj", "model.layers.90.mlp.experts.106.gate_proj", "model.layers.90.mlp.experts.107.gate_proj", "model.layers.90.mlp.experts.108.gate_proj", "model.layers.90.mlp.experts.109.gate_proj", "model.layers.90.mlp.experts.110.gate_proj", "model.layers.90.mlp.experts.111.gate_proj", "model.layers.90.mlp.experts.112.gate_proj", "model.layers.90.mlp.experts.113.gate_proj", "model.layers.90.mlp.experts.114.gate_proj", "model.layers.90.mlp.experts.115.gate_proj", "model.layers.90.mlp.experts.116.gate_proj", "model.layers.90.mlp.experts.117.gate_proj", "model.layers.90.mlp.experts.118.gate_proj", "model.layers.90.mlp.experts.119.gate_proj", "model.layers.90.mlp.experts.120.gate_proj", "model.layers.90.mlp.experts.121.gate_proj", "model.layers.90.mlp.experts.122.gate_proj", "model.layers.90.mlp.experts.123.gate_proj", "model.layers.90.mlp.experts.124.gate_proj", "model.layers.90.mlp.experts.125.gate_proj", "model.layers.90.mlp.experts.126.gate_proj", "model.layers.90.mlp.experts.127.gate_proj", "model.layers.90.mlp.experts.128.gate_proj", "model.layers.90.mlp.experts.129.gate_proj", "model.layers.90.mlp.experts.130.gate_proj", "model.layers.90.mlp.experts.131.gate_proj", "model.layers.90.mlp.experts.132.gate_proj", "model.layers.90.mlp.experts.133.gate_proj", "model.layers.90.mlp.experts.134.gate_proj", "model.layers.90.mlp.experts.135.gate_proj", "model.layers.90.mlp.experts.136.gate_proj", "model.layers.90.mlp.experts.137.gate_proj", "model.layers.90.mlp.experts.138.gate_proj", "model.layers.90.mlp.experts.139.gate_proj", "model.layers.90.mlp.experts.140.gate_proj", "model.layers.90.mlp.experts.141.gate_proj", "model.layers.90.mlp.experts.142.gate_proj", "model.layers.90.mlp.experts.143.gate_proj", "model.layers.90.mlp.experts.144.gate_proj", "model.layers.90.mlp.experts.145.gate_proj", "model.layers.90.mlp.experts.146.gate_proj", "model.layers.90.mlp.experts.147.gate_proj", "model.layers.90.mlp.experts.148.gate_proj", "model.layers.90.mlp.experts.149.gate_proj", "model.layers.90.mlp.experts.150.gate_proj", "model.layers.90.mlp.experts.151.gate_proj", "model.layers.90.mlp.experts.152.gate_proj", "model.layers.90.mlp.experts.153.gate_proj", "model.layers.90.mlp.experts.154.gate_proj", "model.layers.90.mlp.experts.155.gate_proj", "model.layers.90.mlp.experts.156.gate_proj", "model.layers.90.mlp.experts.157.gate_proj", "model.layers.90.mlp.experts.158.gate_proj", "model.layers.90.mlp.experts.159.gate_proj", "model.layers.90.mlp.experts.0.up_proj", "model.layers.90.mlp.experts.1.up_proj", "model.layers.90.mlp.experts.2.up_proj", "model.layers.90.mlp.experts.3.up_proj", "model.layers.90.mlp.experts.4.up_proj", "model.layers.90.mlp.experts.5.up_proj", "model.layers.90.mlp.experts.6.up_proj", "model.layers.90.mlp.experts.7.up_proj", "model.layers.90.mlp.experts.8.up_proj", "model.layers.90.mlp.experts.9.up_proj", "model.layers.90.mlp.experts.10.up_proj", "model.layers.90.mlp.experts.11.up_proj", "model.layers.90.mlp.experts.12.up_proj", "model.layers.90.mlp.experts.13.up_proj", "model.layers.90.mlp.experts.14.up_proj", "model.layers.90.mlp.experts.15.up_proj", "model.layers.90.mlp.experts.16.up_proj", "model.layers.90.mlp.experts.17.up_proj", "model.layers.90.mlp.experts.18.up_proj", "model.layers.90.mlp.experts.19.up_proj", "model.layers.90.mlp.experts.20.up_proj", "model.layers.90.mlp.experts.21.up_proj", "model.layers.90.mlp.experts.22.up_proj", "model.layers.90.mlp.experts.23.up_proj", "model.layers.90.mlp.experts.24.up_proj", "model.layers.90.mlp.experts.25.up_proj", "model.layers.90.mlp.experts.26.up_proj", "model.layers.90.mlp.experts.27.up_proj", "model.layers.90.mlp.experts.28.up_proj", "model.layers.90.mlp.experts.29.up_proj", "model.layers.90.mlp.experts.30.up_proj", "model.layers.90.mlp.experts.31.up_proj", "model.layers.90.mlp.experts.32.up_proj", "model.layers.90.mlp.experts.33.up_proj", "model.layers.90.mlp.experts.34.up_proj", "model.layers.90.mlp.experts.35.up_proj", "model.layers.90.mlp.experts.36.up_proj", "model.layers.90.mlp.experts.37.up_proj", "model.layers.90.mlp.experts.38.up_proj", "model.layers.90.mlp.experts.39.up_proj", "model.layers.90.mlp.experts.40.up_proj", "model.layers.90.mlp.experts.41.up_proj", "model.layers.90.mlp.experts.42.up_proj", "model.layers.90.mlp.experts.43.up_proj", "model.layers.90.mlp.experts.44.up_proj", "model.layers.90.mlp.experts.45.up_proj", "model.layers.90.mlp.experts.46.up_proj", "model.layers.90.mlp.experts.47.up_proj", "model.layers.90.mlp.experts.48.up_proj", "model.layers.90.mlp.experts.49.up_proj", "model.layers.90.mlp.experts.50.up_proj", "model.layers.90.mlp.experts.51.up_proj", "model.layers.90.mlp.experts.52.up_proj", "model.layers.90.mlp.experts.53.up_proj", "model.layers.90.mlp.experts.54.up_proj", "model.layers.90.mlp.experts.55.up_proj", "model.layers.90.mlp.experts.56.up_proj", "model.layers.90.mlp.experts.57.up_proj", "model.layers.90.mlp.experts.58.up_proj", "model.layers.90.mlp.experts.59.up_proj", "model.layers.90.mlp.experts.60.up_proj", "model.layers.90.mlp.experts.61.up_proj", "model.layers.90.mlp.experts.62.up_proj", "model.layers.90.mlp.experts.63.up_proj", "model.layers.90.mlp.experts.64.up_proj", "model.layers.90.mlp.experts.65.up_proj", "model.layers.90.mlp.experts.66.up_proj", "model.layers.90.mlp.experts.67.up_proj", "model.layers.90.mlp.experts.68.up_proj", "model.layers.90.mlp.experts.69.up_proj", "model.layers.90.mlp.experts.70.up_proj", "model.layers.90.mlp.experts.71.up_proj", "model.layers.90.mlp.experts.72.up_proj", "model.layers.90.mlp.experts.73.up_proj", "model.layers.90.mlp.experts.74.up_proj", "model.layers.90.mlp.experts.75.up_proj", "model.layers.90.mlp.experts.76.up_proj", "model.layers.90.mlp.experts.77.up_proj", "model.layers.90.mlp.experts.78.up_proj", "model.layers.90.mlp.experts.79.up_proj", "model.layers.90.mlp.experts.80.up_proj", "model.layers.90.mlp.experts.81.up_proj", "model.layers.90.mlp.experts.82.up_proj", "model.layers.90.mlp.experts.83.up_proj", "model.layers.90.mlp.experts.84.up_proj", "model.layers.90.mlp.experts.85.up_proj", "model.layers.90.mlp.experts.86.up_proj", "model.layers.90.mlp.experts.87.up_proj", "model.layers.90.mlp.experts.88.up_proj", "model.layers.90.mlp.experts.89.up_proj", "model.layers.90.mlp.experts.90.up_proj", "model.layers.90.mlp.experts.91.up_proj", "model.layers.90.mlp.experts.92.up_proj", "model.layers.90.mlp.experts.93.up_proj", "model.layers.90.mlp.experts.94.up_proj", "model.layers.90.mlp.experts.95.up_proj", "model.layers.90.mlp.experts.96.up_proj", "model.layers.90.mlp.experts.97.up_proj", "model.layers.90.mlp.experts.98.up_proj", "model.layers.90.mlp.experts.99.up_proj", "model.layers.90.mlp.experts.100.up_proj", "model.layers.90.mlp.experts.101.up_proj", "model.layers.90.mlp.experts.102.up_proj", "model.layers.90.mlp.experts.103.up_proj", "model.layers.90.mlp.experts.104.up_proj", "model.layers.90.mlp.experts.105.up_proj", "model.layers.90.mlp.experts.106.up_proj", "model.layers.90.mlp.experts.107.up_proj", "model.layers.90.mlp.experts.108.up_proj", "model.layers.90.mlp.experts.109.up_proj", "model.layers.90.mlp.experts.110.up_proj", "model.layers.90.mlp.experts.111.up_proj", "model.layers.90.mlp.experts.112.up_proj", "model.layers.90.mlp.experts.113.up_proj", "model.layers.90.mlp.experts.114.up_proj", "model.layers.90.mlp.experts.115.up_proj", "model.layers.90.mlp.experts.116.up_proj", "model.layers.90.mlp.experts.117.up_proj", "model.layers.90.mlp.experts.118.up_proj", "model.layers.90.mlp.experts.119.up_proj", "model.layers.90.mlp.experts.120.up_proj", "model.layers.90.mlp.experts.121.up_proj", "model.layers.90.mlp.experts.122.up_proj", "model.layers.90.mlp.experts.123.up_proj", "model.layers.90.mlp.experts.124.up_proj", "model.layers.90.mlp.experts.125.up_proj", "model.layers.90.mlp.experts.126.up_proj", "model.layers.90.mlp.experts.127.up_proj", "model.layers.90.mlp.experts.128.up_proj", "model.layers.90.mlp.experts.129.up_proj", "model.layers.90.mlp.experts.130.up_proj", "model.layers.90.mlp.experts.131.up_proj", "model.layers.90.mlp.experts.132.up_proj", "model.layers.90.mlp.experts.133.up_proj", "model.layers.90.mlp.experts.134.up_proj", "model.layers.90.mlp.experts.135.up_proj", "model.layers.90.mlp.experts.136.up_proj", "model.layers.90.mlp.experts.137.up_proj", "model.layers.90.mlp.experts.138.up_proj", "model.layers.90.mlp.experts.139.up_proj", "model.layers.90.mlp.experts.140.up_proj", "model.layers.90.mlp.experts.141.up_proj", "model.layers.90.mlp.experts.142.up_proj", "model.layers.90.mlp.experts.143.up_proj", "model.layers.90.mlp.experts.144.up_proj", "model.layers.90.mlp.experts.145.up_proj", "model.layers.90.mlp.experts.146.up_proj", "model.layers.90.mlp.experts.147.up_proj", "model.layers.90.mlp.experts.148.up_proj", "model.layers.90.mlp.experts.149.up_proj", "model.layers.90.mlp.experts.150.up_proj", "model.layers.90.mlp.experts.151.up_proj", "model.layers.90.mlp.experts.152.up_proj", "model.layers.90.mlp.experts.153.up_proj", "model.layers.90.mlp.experts.154.up_proj", "model.layers.90.mlp.experts.155.up_proj", "model.layers.90.mlp.experts.156.up_proj", "model.layers.90.mlp.experts.157.up_proj", "model.layers.90.mlp.experts.158.up_proj", "model.layers.90.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.0005476392805576213, "dbits": 2516582400 } ] }, { "idx": 542, "layers": [ "model.layers.90.mlp.experts.0.down_proj", "model.layers.90.mlp.experts.1.down_proj", "model.layers.90.mlp.experts.2.down_proj", "model.layers.90.mlp.experts.3.down_proj", "model.layers.90.mlp.experts.4.down_proj", "model.layers.90.mlp.experts.5.down_proj", "model.layers.90.mlp.experts.6.down_proj", "model.layers.90.mlp.experts.7.down_proj", "model.layers.90.mlp.experts.8.down_proj", "model.layers.90.mlp.experts.9.down_proj", "model.layers.90.mlp.experts.10.down_proj", "model.layers.90.mlp.experts.11.down_proj", "model.layers.90.mlp.experts.12.down_proj", "model.layers.90.mlp.experts.13.down_proj", "model.layers.90.mlp.experts.14.down_proj", "model.layers.90.mlp.experts.15.down_proj", "model.layers.90.mlp.experts.16.down_proj", "model.layers.90.mlp.experts.17.down_proj", "model.layers.90.mlp.experts.18.down_proj", "model.layers.90.mlp.experts.19.down_proj", "model.layers.90.mlp.experts.20.down_proj", "model.layers.90.mlp.experts.21.down_proj", "model.layers.90.mlp.experts.22.down_proj", "model.layers.90.mlp.experts.23.down_proj", "model.layers.90.mlp.experts.24.down_proj", "model.layers.90.mlp.experts.25.down_proj", "model.layers.90.mlp.experts.26.down_proj", "model.layers.90.mlp.experts.27.down_proj", "model.layers.90.mlp.experts.28.down_proj", "model.layers.90.mlp.experts.29.down_proj", "model.layers.90.mlp.experts.30.down_proj", "model.layers.90.mlp.experts.31.down_proj", "model.layers.90.mlp.experts.32.down_proj", "model.layers.90.mlp.experts.33.down_proj", "model.layers.90.mlp.experts.34.down_proj", "model.layers.90.mlp.experts.35.down_proj", "model.layers.90.mlp.experts.36.down_proj", "model.layers.90.mlp.experts.37.down_proj", "model.layers.90.mlp.experts.38.down_proj", "model.layers.90.mlp.experts.39.down_proj", "model.layers.90.mlp.experts.40.down_proj", "model.layers.90.mlp.experts.41.down_proj", "model.layers.90.mlp.experts.42.down_proj", "model.layers.90.mlp.experts.43.down_proj", "model.layers.90.mlp.experts.44.down_proj", "model.layers.90.mlp.experts.45.down_proj", "model.layers.90.mlp.experts.46.down_proj", "model.layers.90.mlp.experts.47.down_proj", "model.layers.90.mlp.experts.48.down_proj", "model.layers.90.mlp.experts.49.down_proj", "model.layers.90.mlp.experts.50.down_proj", "model.layers.90.mlp.experts.51.down_proj", "model.layers.90.mlp.experts.52.down_proj", "model.layers.90.mlp.experts.53.down_proj", "model.layers.90.mlp.experts.54.down_proj", "model.layers.90.mlp.experts.55.down_proj", "model.layers.90.mlp.experts.56.down_proj", "model.layers.90.mlp.experts.57.down_proj", "model.layers.90.mlp.experts.58.down_proj", "model.layers.90.mlp.experts.59.down_proj", "model.layers.90.mlp.experts.60.down_proj", "model.layers.90.mlp.experts.61.down_proj", "model.layers.90.mlp.experts.62.down_proj", "model.layers.90.mlp.experts.63.down_proj", "model.layers.90.mlp.experts.64.down_proj", "model.layers.90.mlp.experts.65.down_proj", "model.layers.90.mlp.experts.66.down_proj", "model.layers.90.mlp.experts.67.down_proj", "model.layers.90.mlp.experts.68.down_proj", "model.layers.90.mlp.experts.69.down_proj", "model.layers.90.mlp.experts.70.down_proj", "model.layers.90.mlp.experts.71.down_proj", "model.layers.90.mlp.experts.72.down_proj", "model.layers.90.mlp.experts.73.down_proj", "model.layers.90.mlp.experts.74.down_proj", "model.layers.90.mlp.experts.75.down_proj", "model.layers.90.mlp.experts.76.down_proj", "model.layers.90.mlp.experts.77.down_proj", "model.layers.90.mlp.experts.78.down_proj", "model.layers.90.mlp.experts.79.down_proj", "model.layers.90.mlp.experts.80.down_proj", "model.layers.90.mlp.experts.81.down_proj", "model.layers.90.mlp.experts.82.down_proj", "model.layers.90.mlp.experts.83.down_proj", "model.layers.90.mlp.experts.84.down_proj", "model.layers.90.mlp.experts.85.down_proj", "model.layers.90.mlp.experts.86.down_proj", "model.layers.90.mlp.experts.87.down_proj", "model.layers.90.mlp.experts.88.down_proj", "model.layers.90.mlp.experts.89.down_proj", "model.layers.90.mlp.experts.90.down_proj", "model.layers.90.mlp.experts.91.down_proj", "model.layers.90.mlp.experts.92.down_proj", "model.layers.90.mlp.experts.93.down_proj", "model.layers.90.mlp.experts.94.down_proj", "model.layers.90.mlp.experts.95.down_proj", "model.layers.90.mlp.experts.96.down_proj", "model.layers.90.mlp.experts.97.down_proj", "model.layers.90.mlp.experts.98.down_proj", "model.layers.90.mlp.experts.99.down_proj", "model.layers.90.mlp.experts.100.down_proj", "model.layers.90.mlp.experts.101.down_proj", "model.layers.90.mlp.experts.102.down_proj", "model.layers.90.mlp.experts.103.down_proj", "model.layers.90.mlp.experts.104.down_proj", "model.layers.90.mlp.experts.105.down_proj", "model.layers.90.mlp.experts.106.down_proj", "model.layers.90.mlp.experts.107.down_proj", "model.layers.90.mlp.experts.108.down_proj", "model.layers.90.mlp.experts.109.down_proj", "model.layers.90.mlp.experts.110.down_proj", "model.layers.90.mlp.experts.111.down_proj", "model.layers.90.mlp.experts.112.down_proj", "model.layers.90.mlp.experts.113.down_proj", "model.layers.90.mlp.experts.114.down_proj", "model.layers.90.mlp.experts.115.down_proj", "model.layers.90.mlp.experts.116.down_proj", "model.layers.90.mlp.experts.117.down_proj", "model.layers.90.mlp.experts.118.down_proj", "model.layers.90.mlp.experts.119.down_proj", "model.layers.90.mlp.experts.120.down_proj", "model.layers.90.mlp.experts.121.down_proj", "model.layers.90.mlp.experts.122.down_proj", "model.layers.90.mlp.experts.123.down_proj", "model.layers.90.mlp.experts.124.down_proj", "model.layers.90.mlp.experts.125.down_proj", "model.layers.90.mlp.experts.126.down_proj", "model.layers.90.mlp.experts.127.down_proj", "model.layers.90.mlp.experts.128.down_proj", "model.layers.90.mlp.experts.129.down_proj", "model.layers.90.mlp.experts.130.down_proj", "model.layers.90.mlp.experts.131.down_proj", "model.layers.90.mlp.experts.132.down_proj", "model.layers.90.mlp.experts.133.down_proj", "model.layers.90.mlp.experts.134.down_proj", "model.layers.90.mlp.experts.135.down_proj", "model.layers.90.mlp.experts.136.down_proj", "model.layers.90.mlp.experts.137.down_proj", "model.layers.90.mlp.experts.138.down_proj", "model.layers.90.mlp.experts.139.down_proj", "model.layers.90.mlp.experts.140.down_proj", "model.layers.90.mlp.experts.141.down_proj", "model.layers.90.mlp.experts.142.down_proj", "model.layers.90.mlp.experts.143.down_proj", "model.layers.90.mlp.experts.144.down_proj", "model.layers.90.mlp.experts.145.down_proj", "model.layers.90.mlp.experts.146.down_proj", "model.layers.90.mlp.experts.147.down_proj", "model.layers.90.mlp.experts.148.down_proj", "model.layers.90.mlp.experts.149.down_proj", "model.layers.90.mlp.experts.150.down_proj", "model.layers.90.mlp.experts.151.down_proj", "model.layers.90.mlp.experts.152.down_proj", "model.layers.90.mlp.experts.153.down_proj", "model.layers.90.mlp.experts.154.down_proj", "model.layers.90.mlp.experts.155.down_proj", "model.layers.90.mlp.experts.156.down_proj", "model.layers.90.mlp.experts.157.down_proj", "model.layers.90.mlp.experts.158.down_proj", "model.layers.90.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -0.0003092177212238312, "dbits": 1258291200 } ] }, { "idx": 543, "layers": [ "model.layers.91.self_attn.q_proj" ], "candidates": [ { "dkld": 0.00044054239988328137, "dbits": 62914560 } ] }, { "idx": 544, "layers": [ "model.layers.91.self_attn.k_proj", "model.layers.91.self_attn.v_proj" ], "candidates": [ { "dkld": -0.0013594068586826324, "dbits": 10485760 } ] }, { "idx": 545, "layers": [ "model.layers.91.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0005806535482406616, "dbits": 62914560 } ] }, { "idx": 546, "layers": [ "model.layers.91.mlp.shared_experts.gate_proj", "model.layers.91.mlp.shared_experts.up_proj", "model.layers.91.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0012159086763858795, "dbits": 23592960 } ] }, { "idx": 547, "layers": [ "model.layers.91.mlp.experts.0.gate_proj", "model.layers.91.mlp.experts.1.gate_proj", "model.layers.91.mlp.experts.2.gate_proj", "model.layers.91.mlp.experts.3.gate_proj", "model.layers.91.mlp.experts.4.gate_proj", "model.layers.91.mlp.experts.5.gate_proj", "model.layers.91.mlp.experts.6.gate_proj", "model.layers.91.mlp.experts.7.gate_proj", "model.layers.91.mlp.experts.8.gate_proj", "model.layers.91.mlp.experts.9.gate_proj", "model.layers.91.mlp.experts.10.gate_proj", "model.layers.91.mlp.experts.11.gate_proj", "model.layers.91.mlp.experts.12.gate_proj", "model.layers.91.mlp.experts.13.gate_proj", "model.layers.91.mlp.experts.14.gate_proj", "model.layers.91.mlp.experts.15.gate_proj", "model.layers.91.mlp.experts.16.gate_proj", "model.layers.91.mlp.experts.17.gate_proj", "model.layers.91.mlp.experts.18.gate_proj", "model.layers.91.mlp.experts.19.gate_proj", "model.layers.91.mlp.experts.20.gate_proj", "model.layers.91.mlp.experts.21.gate_proj", "model.layers.91.mlp.experts.22.gate_proj", "model.layers.91.mlp.experts.23.gate_proj", "model.layers.91.mlp.experts.24.gate_proj", "model.layers.91.mlp.experts.25.gate_proj", "model.layers.91.mlp.experts.26.gate_proj", "model.layers.91.mlp.experts.27.gate_proj", "model.layers.91.mlp.experts.28.gate_proj", "model.layers.91.mlp.experts.29.gate_proj", "model.layers.91.mlp.experts.30.gate_proj", "model.layers.91.mlp.experts.31.gate_proj", "model.layers.91.mlp.experts.32.gate_proj", "model.layers.91.mlp.experts.33.gate_proj", "model.layers.91.mlp.experts.34.gate_proj", "model.layers.91.mlp.experts.35.gate_proj", "model.layers.91.mlp.experts.36.gate_proj", "model.layers.91.mlp.experts.37.gate_proj", "model.layers.91.mlp.experts.38.gate_proj", "model.layers.91.mlp.experts.39.gate_proj", "model.layers.91.mlp.experts.40.gate_proj", "model.layers.91.mlp.experts.41.gate_proj", "model.layers.91.mlp.experts.42.gate_proj", "model.layers.91.mlp.experts.43.gate_proj", "model.layers.91.mlp.experts.44.gate_proj", "model.layers.91.mlp.experts.45.gate_proj", "model.layers.91.mlp.experts.46.gate_proj", "model.layers.91.mlp.experts.47.gate_proj", "model.layers.91.mlp.experts.48.gate_proj", "model.layers.91.mlp.experts.49.gate_proj", "model.layers.91.mlp.experts.50.gate_proj", "model.layers.91.mlp.experts.51.gate_proj", "model.layers.91.mlp.experts.52.gate_proj", "model.layers.91.mlp.experts.53.gate_proj", "model.layers.91.mlp.experts.54.gate_proj", "model.layers.91.mlp.experts.55.gate_proj", "model.layers.91.mlp.experts.56.gate_proj", "model.layers.91.mlp.experts.57.gate_proj", "model.layers.91.mlp.experts.58.gate_proj", "model.layers.91.mlp.experts.59.gate_proj", "model.layers.91.mlp.experts.60.gate_proj", "model.layers.91.mlp.experts.61.gate_proj", "model.layers.91.mlp.experts.62.gate_proj", "model.layers.91.mlp.experts.63.gate_proj", "model.layers.91.mlp.experts.64.gate_proj", "model.layers.91.mlp.experts.65.gate_proj", "model.layers.91.mlp.experts.66.gate_proj", "model.layers.91.mlp.experts.67.gate_proj", "model.layers.91.mlp.experts.68.gate_proj", "model.layers.91.mlp.experts.69.gate_proj", "model.layers.91.mlp.experts.70.gate_proj", "model.layers.91.mlp.experts.71.gate_proj", "model.layers.91.mlp.experts.72.gate_proj", "model.layers.91.mlp.experts.73.gate_proj", "model.layers.91.mlp.experts.74.gate_proj", "model.layers.91.mlp.experts.75.gate_proj", "model.layers.91.mlp.experts.76.gate_proj", "model.layers.91.mlp.experts.77.gate_proj", "model.layers.91.mlp.experts.78.gate_proj", "model.layers.91.mlp.experts.79.gate_proj", "model.layers.91.mlp.experts.80.gate_proj", "model.layers.91.mlp.experts.81.gate_proj", "model.layers.91.mlp.experts.82.gate_proj", "model.layers.91.mlp.experts.83.gate_proj", "model.layers.91.mlp.experts.84.gate_proj", "model.layers.91.mlp.experts.85.gate_proj", "model.layers.91.mlp.experts.86.gate_proj", "model.layers.91.mlp.experts.87.gate_proj", "model.layers.91.mlp.experts.88.gate_proj", "model.layers.91.mlp.experts.89.gate_proj", "model.layers.91.mlp.experts.90.gate_proj", "model.layers.91.mlp.experts.91.gate_proj", "model.layers.91.mlp.experts.92.gate_proj", "model.layers.91.mlp.experts.93.gate_proj", "model.layers.91.mlp.experts.94.gate_proj", "model.layers.91.mlp.experts.95.gate_proj", "model.layers.91.mlp.experts.96.gate_proj", "model.layers.91.mlp.experts.97.gate_proj", "model.layers.91.mlp.experts.98.gate_proj", "model.layers.91.mlp.experts.99.gate_proj", "model.layers.91.mlp.experts.100.gate_proj", "model.layers.91.mlp.experts.101.gate_proj", "model.layers.91.mlp.experts.102.gate_proj", "model.layers.91.mlp.experts.103.gate_proj", "model.layers.91.mlp.experts.104.gate_proj", "model.layers.91.mlp.experts.105.gate_proj", "model.layers.91.mlp.experts.106.gate_proj", "model.layers.91.mlp.experts.107.gate_proj", "model.layers.91.mlp.experts.108.gate_proj", "model.layers.91.mlp.experts.109.gate_proj", "model.layers.91.mlp.experts.110.gate_proj", "model.layers.91.mlp.experts.111.gate_proj", "model.layers.91.mlp.experts.112.gate_proj", "model.layers.91.mlp.experts.113.gate_proj", "model.layers.91.mlp.experts.114.gate_proj", "model.layers.91.mlp.experts.115.gate_proj", "model.layers.91.mlp.experts.116.gate_proj", "model.layers.91.mlp.experts.117.gate_proj", "model.layers.91.mlp.experts.118.gate_proj", "model.layers.91.mlp.experts.119.gate_proj", "model.layers.91.mlp.experts.120.gate_proj", "model.layers.91.mlp.experts.121.gate_proj", "model.layers.91.mlp.experts.122.gate_proj", "model.layers.91.mlp.experts.123.gate_proj", "model.layers.91.mlp.experts.124.gate_proj", "model.layers.91.mlp.experts.125.gate_proj", "model.layers.91.mlp.experts.126.gate_proj", "model.layers.91.mlp.experts.127.gate_proj", "model.layers.91.mlp.experts.128.gate_proj", "model.layers.91.mlp.experts.129.gate_proj", "model.layers.91.mlp.experts.130.gate_proj", "model.layers.91.mlp.experts.131.gate_proj", "model.layers.91.mlp.experts.132.gate_proj", "model.layers.91.mlp.experts.133.gate_proj", "model.layers.91.mlp.experts.134.gate_proj", "model.layers.91.mlp.experts.135.gate_proj", "model.layers.91.mlp.experts.136.gate_proj", "model.layers.91.mlp.experts.137.gate_proj", "model.layers.91.mlp.experts.138.gate_proj", "model.layers.91.mlp.experts.139.gate_proj", "model.layers.91.mlp.experts.140.gate_proj", "model.layers.91.mlp.experts.141.gate_proj", "model.layers.91.mlp.experts.142.gate_proj", "model.layers.91.mlp.experts.143.gate_proj", "model.layers.91.mlp.experts.144.gate_proj", "model.layers.91.mlp.experts.145.gate_proj", "model.layers.91.mlp.experts.146.gate_proj", "model.layers.91.mlp.experts.147.gate_proj", "model.layers.91.mlp.experts.148.gate_proj", "model.layers.91.mlp.experts.149.gate_proj", "model.layers.91.mlp.experts.150.gate_proj", "model.layers.91.mlp.experts.151.gate_proj", "model.layers.91.mlp.experts.152.gate_proj", "model.layers.91.mlp.experts.153.gate_proj", "model.layers.91.mlp.experts.154.gate_proj", "model.layers.91.mlp.experts.155.gate_proj", "model.layers.91.mlp.experts.156.gate_proj", "model.layers.91.mlp.experts.157.gate_proj", "model.layers.91.mlp.experts.158.gate_proj", "model.layers.91.mlp.experts.159.gate_proj", "model.layers.91.mlp.experts.0.up_proj", "model.layers.91.mlp.experts.1.up_proj", "model.layers.91.mlp.experts.2.up_proj", "model.layers.91.mlp.experts.3.up_proj", "model.layers.91.mlp.experts.4.up_proj", "model.layers.91.mlp.experts.5.up_proj", "model.layers.91.mlp.experts.6.up_proj", "model.layers.91.mlp.experts.7.up_proj", "model.layers.91.mlp.experts.8.up_proj", "model.layers.91.mlp.experts.9.up_proj", "model.layers.91.mlp.experts.10.up_proj", "model.layers.91.mlp.experts.11.up_proj", "model.layers.91.mlp.experts.12.up_proj", "model.layers.91.mlp.experts.13.up_proj", "model.layers.91.mlp.experts.14.up_proj", "model.layers.91.mlp.experts.15.up_proj", "model.layers.91.mlp.experts.16.up_proj", "model.layers.91.mlp.experts.17.up_proj", "model.layers.91.mlp.experts.18.up_proj", "model.layers.91.mlp.experts.19.up_proj", "model.layers.91.mlp.experts.20.up_proj", "model.layers.91.mlp.experts.21.up_proj", "model.layers.91.mlp.experts.22.up_proj", "model.layers.91.mlp.experts.23.up_proj", "model.layers.91.mlp.experts.24.up_proj", "model.layers.91.mlp.experts.25.up_proj", "model.layers.91.mlp.experts.26.up_proj", "model.layers.91.mlp.experts.27.up_proj", "model.layers.91.mlp.experts.28.up_proj", "model.layers.91.mlp.experts.29.up_proj", "model.layers.91.mlp.experts.30.up_proj", "model.layers.91.mlp.experts.31.up_proj", "model.layers.91.mlp.experts.32.up_proj", "model.layers.91.mlp.experts.33.up_proj", "model.layers.91.mlp.experts.34.up_proj", "model.layers.91.mlp.experts.35.up_proj", "model.layers.91.mlp.experts.36.up_proj", "model.layers.91.mlp.experts.37.up_proj", "model.layers.91.mlp.experts.38.up_proj", "model.layers.91.mlp.experts.39.up_proj", "model.layers.91.mlp.experts.40.up_proj", "model.layers.91.mlp.experts.41.up_proj", "model.layers.91.mlp.experts.42.up_proj", "model.layers.91.mlp.experts.43.up_proj", "model.layers.91.mlp.experts.44.up_proj", "model.layers.91.mlp.experts.45.up_proj", "model.layers.91.mlp.experts.46.up_proj", "model.layers.91.mlp.experts.47.up_proj", "model.layers.91.mlp.experts.48.up_proj", "model.layers.91.mlp.experts.49.up_proj", "model.layers.91.mlp.experts.50.up_proj", "model.layers.91.mlp.experts.51.up_proj", "model.layers.91.mlp.experts.52.up_proj", "model.layers.91.mlp.experts.53.up_proj", "model.layers.91.mlp.experts.54.up_proj", "model.layers.91.mlp.experts.55.up_proj", "model.layers.91.mlp.experts.56.up_proj", "model.layers.91.mlp.experts.57.up_proj", "model.layers.91.mlp.experts.58.up_proj", "model.layers.91.mlp.experts.59.up_proj", "model.layers.91.mlp.experts.60.up_proj", "model.layers.91.mlp.experts.61.up_proj", "model.layers.91.mlp.experts.62.up_proj", "model.layers.91.mlp.experts.63.up_proj", "model.layers.91.mlp.experts.64.up_proj", "model.layers.91.mlp.experts.65.up_proj", "model.layers.91.mlp.experts.66.up_proj", "model.layers.91.mlp.experts.67.up_proj", "model.layers.91.mlp.experts.68.up_proj", "model.layers.91.mlp.experts.69.up_proj", "model.layers.91.mlp.experts.70.up_proj", "model.layers.91.mlp.experts.71.up_proj", "model.layers.91.mlp.experts.72.up_proj", "model.layers.91.mlp.experts.73.up_proj", "model.layers.91.mlp.experts.74.up_proj", "model.layers.91.mlp.experts.75.up_proj", "model.layers.91.mlp.experts.76.up_proj", "model.layers.91.mlp.experts.77.up_proj", "model.layers.91.mlp.experts.78.up_proj", "model.layers.91.mlp.experts.79.up_proj", "model.layers.91.mlp.experts.80.up_proj", "model.layers.91.mlp.experts.81.up_proj", "model.layers.91.mlp.experts.82.up_proj", "model.layers.91.mlp.experts.83.up_proj", "model.layers.91.mlp.experts.84.up_proj", "model.layers.91.mlp.experts.85.up_proj", "model.layers.91.mlp.experts.86.up_proj", "model.layers.91.mlp.experts.87.up_proj", "model.layers.91.mlp.experts.88.up_proj", "model.layers.91.mlp.experts.89.up_proj", "model.layers.91.mlp.experts.90.up_proj", "model.layers.91.mlp.experts.91.up_proj", "model.layers.91.mlp.experts.92.up_proj", "model.layers.91.mlp.experts.93.up_proj", "model.layers.91.mlp.experts.94.up_proj", "model.layers.91.mlp.experts.95.up_proj", "model.layers.91.mlp.experts.96.up_proj", "model.layers.91.mlp.experts.97.up_proj", "model.layers.91.mlp.experts.98.up_proj", "model.layers.91.mlp.experts.99.up_proj", "model.layers.91.mlp.experts.100.up_proj", "model.layers.91.mlp.experts.101.up_proj", "model.layers.91.mlp.experts.102.up_proj", "model.layers.91.mlp.experts.103.up_proj", "model.layers.91.mlp.experts.104.up_proj", "model.layers.91.mlp.experts.105.up_proj", "model.layers.91.mlp.experts.106.up_proj", "model.layers.91.mlp.experts.107.up_proj", "model.layers.91.mlp.experts.108.up_proj", "model.layers.91.mlp.experts.109.up_proj", "model.layers.91.mlp.experts.110.up_proj", "model.layers.91.mlp.experts.111.up_proj", "model.layers.91.mlp.experts.112.up_proj", "model.layers.91.mlp.experts.113.up_proj", "model.layers.91.mlp.experts.114.up_proj", "model.layers.91.mlp.experts.115.up_proj", "model.layers.91.mlp.experts.116.up_proj", "model.layers.91.mlp.experts.117.up_proj", "model.layers.91.mlp.experts.118.up_proj", "model.layers.91.mlp.experts.119.up_proj", "model.layers.91.mlp.experts.120.up_proj", "model.layers.91.mlp.experts.121.up_proj", "model.layers.91.mlp.experts.122.up_proj", "model.layers.91.mlp.experts.123.up_proj", "model.layers.91.mlp.experts.124.up_proj", "model.layers.91.mlp.experts.125.up_proj", "model.layers.91.mlp.experts.126.up_proj", "model.layers.91.mlp.experts.127.up_proj", "model.layers.91.mlp.experts.128.up_proj", "model.layers.91.mlp.experts.129.up_proj", "model.layers.91.mlp.experts.130.up_proj", "model.layers.91.mlp.experts.131.up_proj", "model.layers.91.mlp.experts.132.up_proj", "model.layers.91.mlp.experts.133.up_proj", "model.layers.91.mlp.experts.134.up_proj", "model.layers.91.mlp.experts.135.up_proj", "model.layers.91.mlp.experts.136.up_proj", "model.layers.91.mlp.experts.137.up_proj", "model.layers.91.mlp.experts.138.up_proj", "model.layers.91.mlp.experts.139.up_proj", "model.layers.91.mlp.experts.140.up_proj", "model.layers.91.mlp.experts.141.up_proj", "model.layers.91.mlp.experts.142.up_proj", "model.layers.91.mlp.experts.143.up_proj", "model.layers.91.mlp.experts.144.up_proj", "model.layers.91.mlp.experts.145.up_proj", "model.layers.91.mlp.experts.146.up_proj", "model.layers.91.mlp.experts.147.up_proj", "model.layers.91.mlp.experts.148.up_proj", "model.layers.91.mlp.experts.149.up_proj", "model.layers.91.mlp.experts.150.up_proj", "model.layers.91.mlp.experts.151.up_proj", "model.layers.91.mlp.experts.152.up_proj", "model.layers.91.mlp.experts.153.up_proj", "model.layers.91.mlp.experts.154.up_proj", "model.layers.91.mlp.experts.155.up_proj", "model.layers.91.mlp.experts.156.up_proj", "model.layers.91.mlp.experts.157.up_proj", "model.layers.91.mlp.experts.158.up_proj", "model.layers.91.mlp.experts.159.up_proj" ], "candidates": [ { "dkld": -0.00013649314641955845, "dbits": 2516582400 } ] }, { "idx": 548, "layers": [ "model.layers.91.mlp.experts.0.down_proj", "model.layers.91.mlp.experts.1.down_proj", "model.layers.91.mlp.experts.2.down_proj", "model.layers.91.mlp.experts.3.down_proj", "model.layers.91.mlp.experts.4.down_proj", "model.layers.91.mlp.experts.5.down_proj", "model.layers.91.mlp.experts.6.down_proj", "model.layers.91.mlp.experts.7.down_proj", "model.layers.91.mlp.experts.8.down_proj", "model.layers.91.mlp.experts.9.down_proj", "model.layers.91.mlp.experts.10.down_proj", "model.layers.91.mlp.experts.11.down_proj", "model.layers.91.mlp.experts.12.down_proj", "model.layers.91.mlp.experts.13.down_proj", "model.layers.91.mlp.experts.14.down_proj", "model.layers.91.mlp.experts.15.down_proj", "model.layers.91.mlp.experts.16.down_proj", "model.layers.91.mlp.experts.17.down_proj", "model.layers.91.mlp.experts.18.down_proj", "model.layers.91.mlp.experts.19.down_proj", "model.layers.91.mlp.experts.20.down_proj", "model.layers.91.mlp.experts.21.down_proj", "model.layers.91.mlp.experts.22.down_proj", "model.layers.91.mlp.experts.23.down_proj", "model.layers.91.mlp.experts.24.down_proj", "model.layers.91.mlp.experts.25.down_proj", "model.layers.91.mlp.experts.26.down_proj", "model.layers.91.mlp.experts.27.down_proj", "model.layers.91.mlp.experts.28.down_proj", "model.layers.91.mlp.experts.29.down_proj", "model.layers.91.mlp.experts.30.down_proj", "model.layers.91.mlp.experts.31.down_proj", "model.layers.91.mlp.experts.32.down_proj", "model.layers.91.mlp.experts.33.down_proj", "model.layers.91.mlp.experts.34.down_proj", "model.layers.91.mlp.experts.35.down_proj", "model.layers.91.mlp.experts.36.down_proj", "model.layers.91.mlp.experts.37.down_proj", "model.layers.91.mlp.experts.38.down_proj", "model.layers.91.mlp.experts.39.down_proj", "model.layers.91.mlp.experts.40.down_proj", "model.layers.91.mlp.experts.41.down_proj", "model.layers.91.mlp.experts.42.down_proj", "model.layers.91.mlp.experts.43.down_proj", "model.layers.91.mlp.experts.44.down_proj", "model.layers.91.mlp.experts.45.down_proj", "model.layers.91.mlp.experts.46.down_proj", "model.layers.91.mlp.experts.47.down_proj", "model.layers.91.mlp.experts.48.down_proj", "model.layers.91.mlp.experts.49.down_proj", "model.layers.91.mlp.experts.50.down_proj", "model.layers.91.mlp.experts.51.down_proj", "model.layers.91.mlp.experts.52.down_proj", "model.layers.91.mlp.experts.53.down_proj", "model.layers.91.mlp.experts.54.down_proj", "model.layers.91.mlp.experts.55.down_proj", "model.layers.91.mlp.experts.56.down_proj", "model.layers.91.mlp.experts.57.down_proj", "model.layers.91.mlp.experts.58.down_proj", "model.layers.91.mlp.experts.59.down_proj", "model.layers.91.mlp.experts.60.down_proj", "model.layers.91.mlp.experts.61.down_proj", "model.layers.91.mlp.experts.62.down_proj", "model.layers.91.mlp.experts.63.down_proj", "model.layers.91.mlp.experts.64.down_proj", "model.layers.91.mlp.experts.65.down_proj", "model.layers.91.mlp.experts.66.down_proj", "model.layers.91.mlp.experts.67.down_proj", "model.layers.91.mlp.experts.68.down_proj", "model.layers.91.mlp.experts.69.down_proj", "model.layers.91.mlp.experts.70.down_proj", "model.layers.91.mlp.experts.71.down_proj", "model.layers.91.mlp.experts.72.down_proj", "model.layers.91.mlp.experts.73.down_proj", "model.layers.91.mlp.experts.74.down_proj", "model.layers.91.mlp.experts.75.down_proj", "model.layers.91.mlp.experts.76.down_proj", "model.layers.91.mlp.experts.77.down_proj", "model.layers.91.mlp.experts.78.down_proj", "model.layers.91.mlp.experts.79.down_proj", "model.layers.91.mlp.experts.80.down_proj", "model.layers.91.mlp.experts.81.down_proj", "model.layers.91.mlp.experts.82.down_proj", "model.layers.91.mlp.experts.83.down_proj", "model.layers.91.mlp.experts.84.down_proj", "model.layers.91.mlp.experts.85.down_proj", "model.layers.91.mlp.experts.86.down_proj", "model.layers.91.mlp.experts.87.down_proj", "model.layers.91.mlp.experts.88.down_proj", "model.layers.91.mlp.experts.89.down_proj", "model.layers.91.mlp.experts.90.down_proj", "model.layers.91.mlp.experts.91.down_proj", "model.layers.91.mlp.experts.92.down_proj", "model.layers.91.mlp.experts.93.down_proj", "model.layers.91.mlp.experts.94.down_proj", "model.layers.91.mlp.experts.95.down_proj", "model.layers.91.mlp.experts.96.down_proj", "model.layers.91.mlp.experts.97.down_proj", "model.layers.91.mlp.experts.98.down_proj", "model.layers.91.mlp.experts.99.down_proj", "model.layers.91.mlp.experts.100.down_proj", "model.layers.91.mlp.experts.101.down_proj", "model.layers.91.mlp.experts.102.down_proj", "model.layers.91.mlp.experts.103.down_proj", "model.layers.91.mlp.experts.104.down_proj", "model.layers.91.mlp.experts.105.down_proj", "model.layers.91.mlp.experts.106.down_proj", "model.layers.91.mlp.experts.107.down_proj", "model.layers.91.mlp.experts.108.down_proj", "model.layers.91.mlp.experts.109.down_proj", "model.layers.91.mlp.experts.110.down_proj", "model.layers.91.mlp.experts.111.down_proj", "model.layers.91.mlp.experts.112.down_proj", "model.layers.91.mlp.experts.113.down_proj", "model.layers.91.mlp.experts.114.down_proj", "model.layers.91.mlp.experts.115.down_proj", "model.layers.91.mlp.experts.116.down_proj", "model.layers.91.mlp.experts.117.down_proj", "model.layers.91.mlp.experts.118.down_proj", "model.layers.91.mlp.experts.119.down_proj", "model.layers.91.mlp.experts.120.down_proj", "model.layers.91.mlp.experts.121.down_proj", "model.layers.91.mlp.experts.122.down_proj", "model.layers.91.mlp.experts.123.down_proj", "model.layers.91.mlp.experts.124.down_proj", "model.layers.91.mlp.experts.125.down_proj", "model.layers.91.mlp.experts.126.down_proj", "model.layers.91.mlp.experts.127.down_proj", "model.layers.91.mlp.experts.128.down_proj", "model.layers.91.mlp.experts.129.down_proj", "model.layers.91.mlp.experts.130.down_proj", "model.layers.91.mlp.experts.131.down_proj", "model.layers.91.mlp.experts.132.down_proj", "model.layers.91.mlp.experts.133.down_proj", "model.layers.91.mlp.experts.134.down_proj", "model.layers.91.mlp.experts.135.down_proj", "model.layers.91.mlp.experts.136.down_proj", "model.layers.91.mlp.experts.137.down_proj", "model.layers.91.mlp.experts.138.down_proj", "model.layers.91.mlp.experts.139.down_proj", "model.layers.91.mlp.experts.140.down_proj", "model.layers.91.mlp.experts.141.down_proj", "model.layers.91.mlp.experts.142.down_proj", "model.layers.91.mlp.experts.143.down_proj", "model.layers.91.mlp.experts.144.down_proj", "model.layers.91.mlp.experts.145.down_proj", "model.layers.91.mlp.experts.146.down_proj", "model.layers.91.mlp.experts.147.down_proj", "model.layers.91.mlp.experts.148.down_proj", "model.layers.91.mlp.experts.149.down_proj", "model.layers.91.mlp.experts.150.down_proj", "model.layers.91.mlp.experts.151.down_proj", "model.layers.91.mlp.experts.152.down_proj", "model.layers.91.mlp.experts.153.down_proj", "model.layers.91.mlp.experts.154.down_proj", "model.layers.91.mlp.experts.155.down_proj", "model.layers.91.mlp.experts.156.down_proj", "model.layers.91.mlp.experts.157.down_proj", "model.layers.91.mlp.experts.158.down_proj", "model.layers.91.mlp.experts.159.down_proj" ], "candidates": [ { "dkld": -7.344037294387817e-05, "dbits": 1258291200 } ] } ], "base_kld": 0.4867161110043526, "arch_string": "Glm4MoeForCausalLM" }