| { |
| "original_ppl": 5.709594249725342, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10567.6279296875 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10189.767578125 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10145.455078125 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10058.2685546875 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10255.1142578125 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10247.263671875 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10976.8837890625 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10739.5322265625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10739.3369140625 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8588.806640625 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 9590.7763671875 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 9879.37890625 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 9242.65234375 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10457.009765625 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10544.896484375 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10610.4453125 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 9306.521484375 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8802.19140625 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 9998.1513671875 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10256.3759765625 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10959.85546875 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10516.7548828125 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 11356.791015625 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 11073.08984375 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 11204.8544921875 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 12153.826171875 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 9231.9580078125 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10075.943359375 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10825.87890625 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 12785.171875 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10821.224609375 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 9910.1611328125 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10058.9404296875 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10436.1171875 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 11146.353515625 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 11659.0087890625 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 9418.8369140625 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10180.2294921875 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 9962.26953125 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 11479.43359375 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10776.0859375 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 11470.4169921875 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10557.3232421875 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10713.5888671875 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10879.771484375 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10029.302734375 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8872.9033203125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 9753.16015625 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10820.5224609375 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10781.5654296875 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 9849.642578125 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 9180.4384765625 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 12868.7216796875 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10868.7900390625 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10450.2705078125 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8471.5751953125 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10817.5302734375 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10570.3388671875 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 13298.19140625 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 9529.5908203125 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10790.4736328125 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10234.9873046875 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 9251.2587890625 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 11260.580078125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10129.4736328125 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8761.8564453125 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 9022.6669921875 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 11680.9775390625 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10368.7578125 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 9557.1943359375 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10703.908203125 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 12151.669921875 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 12307.1337890625 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10009.294921875 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 11075.8564453125 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 11463.69140625 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10337.84375 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10680.353515625 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10197.953125 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10211.8994140625 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8991.3408203125 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 9676.6142578125 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 12080.5859375 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10091.8974609375 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10554.6748046875 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10436.3271484375 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10313.8154296875 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 12580.716796875 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10480.9912109375 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10401.0927734375 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10003.1015625 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10767.0361328125 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10837.3564453125 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 11167.5810546875 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8934.3203125 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 11187.5576171875 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 9931.0703125 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 11020.18359375 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10910.08984375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 9984.7353515625 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 9652.2451171875 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8942.28125 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10678.275390625 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 11443.134765625 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 11020.7197265625 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10632.4658203125 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10628.0458984375 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 9440.8701171875 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 12414.9384765625 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10940.244140625 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 11481.5576171875 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 11989.66015625 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10691.451171875 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10677.4609375 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10757.7783203125 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 12009.7216796875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10884.337890625 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 11461.220703125 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10906.6787109375 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10778.2041015625 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 11037.0107421875 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 11151.72265625 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10253.0703125 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 11608.63671875 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10195.521484375 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 11130.271484375 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10519.5439453125 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10713.19140625 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10709.994140625 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10053.6171875 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10573.142578125 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 11842.8134765625 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 12637.943359375 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10699.1728515625 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10658.5576171875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10849.703125 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 11081.3291015625 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 11147.703125 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 11805.89453125 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10489.5810546875 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 11061.279296875 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10922.4375 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10153.4599609375 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 9357.04296875 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 11064.43359375 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 11501.986328125 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 11086.8779296875 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10847.095703125 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10781.2158203125 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10823.01953125 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10984.1513671875 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10976.4453125 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 11326.1162109375 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 11063.5048828125 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10700.029296875 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10697.7236328125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 11089.6279296875 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10051.4892578125 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10695.642578125 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 11200.6240234375 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10779.40625 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10751.193359375 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10875.310546875 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10518.2802734375 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10321.490234375 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10772.458984375 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 11081.3291015625 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10928.4697265625 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10713.181640625 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10939.4716796875 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10388.9990234375 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 11035.9169921875 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10970.7724609375 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 11015.2666015625 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 11003.580078125 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10400.6767578125 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 9993.7470703125 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10240.6015625 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10649.12890625 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10727.525390625 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10895.4716796875 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10772.5107421875 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10912.525390625 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 11053.232421875 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10848.7099609375 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10753.203125 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 11092.41015625 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 11366.5537109375 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 11235.0185546875 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10819.3564453125 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10522.3037109375 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10533.1962890625 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10387.294921875 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10760.91796875 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10947.3408203125 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10882.51171875 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 10507.8232421875 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 10348.2197265625 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 10611.326171875 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 11039.19140625 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 10514.5693359375 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 11334.728515625 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 10771.5341796875 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 10928.54296875 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 10939.095703125 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 10196.8447265625 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 10541.9189453125 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 9971.365234375 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 10726.451171875 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 10932.640625 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 10955.3828125 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 10858.2216796875 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 11159.27734375 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 10506.009765625 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 9995.9775390625 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 10862.572265625 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 11789.1767578125 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 10662.990234375 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 10548.818359375 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 9830.6953125 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 10317.2978515625 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8232.400390625 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 10862.7685546875 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 11919.38671875 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 34.57195281982422 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 34.52710723876953 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 33.759483337402344 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 32.43354415893555 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 32.05517578125 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 32.288536071777344 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 31.484949111938477 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 34.28741455078125 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 34.2617301940918 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 33.248924255371094 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 33.3477783203125 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 33.089942932128906 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 33.65793228149414 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 33.47496032714844 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 33.28439712524414 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 34.1829948425293 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 34.67659378051758 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 34.420021057128906 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 33.05862045288086 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 33.653533935546875 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 33.16836166381836 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 33.8890266418457 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 34.63047409057617 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 33.226661682128906 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 33.40636444091797 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 33.148216247558594 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 33.19480895996094 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 33.060516357421875 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 34.38041305541992 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 34.89942169189453 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 34.22050094604492 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 35.17528533935547 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 33.39213562011719 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 33.79841232299805 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 33.23938751220703 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 33.895816802978516 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 34.27817153930664 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 33.54795455932617 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 33.53366470336914 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 34.059391021728516 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 33.78882598876953 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 33.07892990112305 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 34.50130081176758 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 34.48667907714844 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 34.063663482666016 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 33.386802673339844 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 33.016632080078125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 33.38557434082031 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 32.862335205078125 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 35.20330047607422 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 34.48147201538086 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 34.12400817871094 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 33.47517776489258 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 34.02745056152344 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 34.83621597290039 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 34.40919876098633 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 34.31414794921875 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 34.19636917114258 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 34.39153289794922 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 34.5456657409668 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 34.0633430480957 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 34.094749450683594 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 33.85380172729492 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 34.03188705444336 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 34.119022369384766 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 34.74598693847656 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 34.483734130859375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 34.58083724975586 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 34.41065979003906 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 34.2165641784668 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 33.957088470458984 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 34.00929260253906 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 34.12889862060547 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 33.797481536865234 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 34.214481353759766 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 34.31638717651367 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 34.608943939208984 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 34.54572296142578 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 34.51774597167969 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 34.53952407836914 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 33.08501434326172 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 34.0992546081543 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 34.07937240600586 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 34.25077819824219 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 34.28379440307617 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 34.26295471191406 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 33.22490310668945 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 33.58615493774414 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 33.81552505493164 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 33.98564147949219 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 33.63060760498047 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 33.890804290771484 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 33.82011413574219 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 33.99924087524414 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 34.403194427490234 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 34.02565002441406 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 34.25650405883789 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 33.94101333618164 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 34.540428161621094 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 34.29387283325195 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 34.01964569091797 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 34.42142105102539 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 33.88022232055664 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 34.27000045776367 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 34.40312576293945 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 34.50580978393555 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 34.66890335083008 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 34.27513122558594 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 34.37164306640625 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 34.541316986083984 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 34.50294494628906 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 34.36033630371094 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 34.56711959838867 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 34.362464904785156 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 34.36616516113281 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 34.23607635498047 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 33.9645881652832 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 34.4565315246582 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 34.30780792236328 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 34.27949142456055 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 34.582603454589844 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 34.44756317138672 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 34.38725280761719 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 34.13260269165039 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 34.467872619628906 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 34.088993072509766 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 34.485233306884766 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 34.57403564453125 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 34.682552337646484 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 34.74006271362305 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 34.2639045715332 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 34.40919876098633 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 34.312774658203125 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 34.317691802978516 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 34.483306884765625 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 34.51898956298828 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 34.307220458984375 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 34.29369354248047 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 34.066001892089844 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 34.13399124145508 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 34.56247329711914 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 34.55335998535156 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 34.67139434814453 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 34.52983856201172 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 34.03361892700195 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 34.50508499145508 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 34.3360481262207 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 34.427818298339844 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 34.45600128173828 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 34.440826416015625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 34.39469528198242 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 34.30693435668945 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 34.37007141113281 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 34.36513900756836 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 34.605342864990234 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 34.379478454589844 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 34.61307144165039 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 34.54313278198242 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 34.373775482177734 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 34.57048416137695 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 34.41924285888672 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 34.54680252075195 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 34.607139587402344 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 34.717864990234375 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 34.61105728149414 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 34.29473114013672 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 34.4323616027832 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 34.353153228759766 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 34.35285186767578 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 34.476287841796875 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 34.416080474853516 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 34.39631271362305 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 34.278724670410156 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 34.69623565673828 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 34.41499710083008 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 34.44273376464844 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 34.565521240234375 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 34.46662902832031 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 34.48087692260742 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 34.32445526123047 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 34.53607177734375 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 34.318450927734375 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 34.54118728637695 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 34.53817367553711 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 34.60563278198242 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 34.45487976074219 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 34.445526123046875 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 34.558074951171875 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 34.528133392333984 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 34.438350677490234 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 34.52284240722656 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 34.60752868652344 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 34.66035842895508 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 34.51670837402344 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 34.5708122253418 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 34.3725700378418 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 34.559669494628906 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 34.59638977050781 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 34.5826416015625 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 34.5693473815918 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 34.57836151123047 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 34.65898513793945 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 34.550697326660156 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 34.395263671875 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 34.35078811645508 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 34.641231536865234 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 34.51400375366211 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 34.5910530090332 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 34.4173698425293 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 34.416351318359375 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 34.47446060180664 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 34.593849182128906 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 34.69132995605469 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 34.640846252441406 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 34.203651428222656 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 34.22004699707031 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 34.34844207763672 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 34.51666259765625 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 34.57122802734375 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 34.6813850402832 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 34.71495056152344 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 34.37432098388672 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 34.53300094604492 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 34.46146774291992 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.9461774826049805 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.947465419769287 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.901550769805908 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.8674397468566895 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.887763500213623 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.896790027618408 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.880622386932373 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.922502517700195 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.933195114135742 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.884892463684082 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.896417140960693 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.87729549407959 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.880502223968506 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.883674144744873 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.930445671081543 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.915066242218018 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.9209184646606445 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.9380011558532715 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.896726131439209 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.9177069664001465 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.91702938079834 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.929443359375 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.937050819396973 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.9357147216796875 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.929398059844971 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.89821720123291 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.9130635261535645 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.917545795440674 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.949504852294922 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.944932460784912 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.931194305419922 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.928679943084717 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.912296295166016 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.932130813598633 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.91456413269043 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.939064979553223 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.946615219116211 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.9263153076171875 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.929776191711426 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.914409637451172 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.9311299324035645 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.927009105682373 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.950355529785156 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.948017120361328 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.942692279815674 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.93778133392334 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.906951904296875 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.928264141082764 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.93275260925293 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.961323261260986 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.958410263061523 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.963601112365723 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.959017276763916 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.9284868240356445 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.931981563568115 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.926553726196289 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.94442081451416 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.946796894073486 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.941961288452148 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.941811561584473 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.931285381317139 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.934602737426758 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.92728853225708 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.944366455078125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.946315765380859 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.94504451751709 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.941877841949463 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.930385112762451 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.936042785644531 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.9395270347595215 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.938392639160156 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.937050819396973 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.937075614929199 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.931523323059082 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.928436279296875 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.9382429122924805 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.944479942321777 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.94779109954834 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.952007293701172 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.952056407928467 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.950871467590332 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.936583518981934 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.922083854675293 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.942915439605713 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.947587013244629 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.953937530517578 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.934799671173096 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.936850547790527 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.948805332183838 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.9409918785095215 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.935338497161865 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.936914920806885 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.937769889831543 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.939852237701416 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.9268083572387695 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.934116363525391 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.936207294464111 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.9416375160217285 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.944608688354492 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.9514079093933105 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.958968162536621 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.953381538391113 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.936254501342773 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.939049243927002 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.937917709350586 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.944266319274902 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.945455551147461 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.9394755363464355 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.9380784034729 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.938943386077881 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.937800407409668 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.939210414886475 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.943374156951904 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.945173263549805 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.939042091369629 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.937414169311523 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.935083389282227 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.938426494598389 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.940467357635498 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.939859390258789 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.940512657165527 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.943423271179199 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.939942836761475 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.930112838745117 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.938530921936035 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.935147762298584 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.954329490661621 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.948697566986084 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.942699432373047 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.939664363861084 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.929938793182373 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.941745281219482 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.937289237976074 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.941506862640381 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.941203594207764 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.939117431640625 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.938708782196045 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.928543567657471 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.928940773010254 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.932537078857422 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.948521137237549 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.949269771575928 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.94331693649292 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.943905830383301 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.927674293518066 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.946404457092285 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.933562278747559 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.94740104675293 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.945073127746582 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.94174337387085 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.943120002746582 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.93034553527832 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.941709041595459 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.934141159057617 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.943271636962891 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.9431047439575195 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.945976257324219 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.948796272277832 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.929816246032715 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.936256408691406 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.933286190032959 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.945980072021484 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.947525978088379 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.948060512542725 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.948775291442871 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.935506820678711 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.940329074859619 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.9392619132995605 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.945156097412109 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.94595193862915 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.946173191070557 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.944000720977783 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.935363292694092 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.936956405639648 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.934545516967773 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.946089744567871 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.94644832611084 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.944385051727295 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.94339656829834 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.931257247924805 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.932302474975586 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.938589572906494 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.945711135864258 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.946021556854248 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.9461750984191895 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.943582057952881 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.9326887130737305 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.938901424407959 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.942438125610352 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.945036888122559 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.9467267990112305 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.949273586273193 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.950535774230957 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.937090873718262 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.936392307281494 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.940166473388672 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.9436445236206055 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.944836139678955 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.942584037780762 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.943691730499268 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.935976505279541 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.937545299530029 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.940749645233154 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.94005823135376 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.940617084503174 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.953882217407227 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.945059776306152 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.935088634490967 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.941315174102783 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.942779541015625 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.9452738761901855 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.948841571807861 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.94928503036499 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.945047855377197 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.935410499572754 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.936846733093262 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.940468788146973 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.947516918182373 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.94775390625 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.960308074951172 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.9513959884643555 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.927347660064697 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.922753810882568 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.93113374710083 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.454537868499756 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.454726696014404 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.450519561767578 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.4437255859375 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.448850631713867 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.450132369995117 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.446074962615967 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.449563980102539 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.4544291496276855 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.448617935180664 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.446349143981934 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.445285797119141 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.443925857543945 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.445038318634033 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.448399066925049 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.453738689422607 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.4486260414123535 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.451790809631348 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.446408271789551 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.450733661651611 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.448801517486572 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.452375411987305 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.452647686004639 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.451661586761475 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.448610782623291 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.447442054748535 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.447699069976807 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.448030948638916 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.455854415893555 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.454732894897461 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.452969074249268 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.4545440673828125 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.447070121765137 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.449777603149414 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.455633640289307 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.453912258148193 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.454641342163086 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.453270435333252 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.455489158630371 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.451198577880859 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.452322959899902 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.452975273132324 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.455671310424805 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.455975532531738 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.456965923309326 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.455340385437012 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.446424961090088 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.451291084289551 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.4530415534973145 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.455935478210449 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.456725120544434 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.452491760253906 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.452752113342285 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.449517726898193 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.452578544616699 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.451826095581055 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.45448637008667 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.454720973968506 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.455731391906738 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.455089092254639 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.452181816101074 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.4526824951171875 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.454453468322754 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.455934524536133 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.456002235412598 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.4546217918396 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.453610420227051 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.452699661254883 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.453961372375488 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.454385757446289 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.452418327331543 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.453369140625 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.4515509605407715 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.4532928466796875 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.452413558959961 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.4538068771362305 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.455025672912598 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.456299304962158 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.454927921295166 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.4534173011779785 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.454418182373047 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.451657295227051 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.451346397399902 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.454096794128418 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.454782009124756 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.45551872253418 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.4534735679626465 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.453575611114502 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.453646183013916 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.454469203948975 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.456052780151367 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.453341484069824 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.454270839691162 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.451605796813965 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.452873706817627 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.453426837921143 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.452221870422363 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.452780246734619 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.455355644226074 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.455962657928467 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.456732273101807 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.453287124633789 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.453049182891846 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.454832077026367 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.454221725463867 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.454924583435059 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.454874515533447 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.451925277709961 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.452725410461426 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.4522833824157715 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.45523738861084 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.45308256149292 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.454366683959961 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.455395698547363 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.452813148498535 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.454213619232178 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.451274871826172 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.454441547393799 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.4550957679748535 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.453302383422852 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.454470634460449 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.453659534454346 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.453427791595459 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.450741767883301 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.453146934509277 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.454835891723633 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.4564948081970215 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.455182075500488 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.454783916473389 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.455849647521973 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.451282501220703 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.455349922180176 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.454250812530518 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.454679012298584 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.454629421234131 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.452934741973877 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.4529619216918945 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.450913429260254 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.453946113586426 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.453378200531006 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.4553914070129395 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.455657958984375 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.454373836517334 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.454061985015869 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.450056552886963 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.453320503234863 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.453465938568115 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.454935073852539 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.455124855041504 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.45426607131958 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.454684734344482 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.450778484344482 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.454276084899902 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.452948093414307 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.455362319946289 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.455121994018555 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.45496940612793 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.454651355743408 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.450726509094238 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.45337438583374 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.452306747436523 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.454626083374023 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.455564498901367 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.454758644104004 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.454658031463623 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.452275276184082 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.452835559844971 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.45163631439209 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.454537868499756 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.455155372619629 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.454677104949951 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.455167293548584 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.4516472816467285 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.453344345092773 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.451472282409668 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.4543914794921875 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.454807758331299 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.4543938636779785 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.454660415649414 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.4515910148620605 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.453164100646973 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.452962875366211 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.455028533935547 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.45471715927124 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.454559803009033 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.454847812652588 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.451287746429443 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.453179836273193 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.453149795532227 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.453858852386475 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.454281330108643 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.455502033233643 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.454575538635254 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.451561450958252 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.452361583709717 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.4535813331604 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.454469203948975 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.454554557800293 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.452547073364258 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.454784870147705 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.451087951660156 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.453288555145264 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.452445983886719 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.453505992889404 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.454779624938965 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.456064701080322 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.454588890075684 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.452078342437744 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.453407287597656 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.452796936035156 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.455315113067627 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.4545488357543945 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.455374240875244 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.4543914794921875 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.450343608856201 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.45657205581665 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.453243732452393 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.455255031585693 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.4553728103637695 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.457914352416992 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.456103801727295 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.44964075088501 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.447453498840332 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.448609352111816 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.10332727432251 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.103443145751953 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.102327823638916 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.10093879699707 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.102288246154785 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.10227108001709 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.102169513702393 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.102914333343506 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.1028876304626465 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.1018571853637695 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.102959156036377 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.101169109344482 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.100903511047363 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.10158109664917 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.102158546447754 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.102851390838623 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.103203773498535 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.100645065307617 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.100838661193848 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.1026201248168945 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.1004252433776855 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.102511405944824 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.102550506591797 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.102677345275879 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.101796627044678 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.1003804206848145 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.102692127227783 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.1017165184021 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.103428840637207 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.102806091308594 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.10255241394043 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.102294445037842 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.101330280303955 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.101095199584961 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.102182388305664 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.10371732711792 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.103259086608887 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.10362434387207 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.102778911590576 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.103330135345459 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.102243423461914 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.102015018463135 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.103821754455566 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.103170394897461 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.102950096130371 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.1029534339904785 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.100493907928467 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.101402759552002 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.102095127105713 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.1033759117126465 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.104278087615967 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.10400390625 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.104101181030273 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.101878643035889 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.1019816398620605 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.10313081741333 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.1031084060668945 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.103360176086426 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.1029582023620605 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.103797435760498 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.1028923988342285 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.102817058563232 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.1025390625 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.103361129760742 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.103934288024902 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.1032819747924805 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.10377836227417 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.103052139282227 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.103724479675293 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.1033244132995605 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.103392601013184 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.1027607917785645 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.10310173034668 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.102715015411377 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.1021318435668945 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.102659225463867 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.102339267730713 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.103653907775879 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.103553295135498 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.103222846984863 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.102321147918701 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.1020355224609375 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.102360248565674 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.103721618652344 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.10411262512207 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.103764057159424 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.102407932281494 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.103240013122559 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.102255344390869 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.102858543395996 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.102774620056152 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.102561950683594 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.102424621582031 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.10245943069458 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.102545738220215 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.102554798126221 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.103653430938721 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.103150844573975 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.103906631469727 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.103859901428223 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.102217674255371 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.103619575500488 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.102270126342773 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.1033854484558105 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.103147029876709 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.103157043457031 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.103323459625244 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.102534294128418 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.102122783660889 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.102392673492432 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.102656364440918 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.103926658630371 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.103625774383545 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.103555679321289 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.1027302742004395 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.102412223815918 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.101873874664307 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.103229999542236 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.102924346923828 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.1027679443359375 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.103058815002441 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.103218078613281 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.1030073165893555 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.101222038269043 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.102621555328369 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.102445602416992 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.103686809539795 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.103755474090576 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.103569984436035 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.1022820472717285 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.102065086364746 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.102996826171875 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.103424072265625 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.1032609939575195 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.103483200073242 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.102055072784424 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.102387428283691 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.101625442504883 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.102776527404785 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.102668285369873 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.103758811950684 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.103571891784668 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.102988243103027 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.102940559387207 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.101558685302734 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.103109359741211 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.102682113647461 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.103518009185791 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.103269100189209 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.103158950805664 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.102853298187256 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.101789951324463 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.103322982788086 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.102255344390869 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.103530406951904 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.103815078735352 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.103599548339844 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.103411674499512 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.101712703704834 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.102099418640137 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.102999687194824 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.10357141494751 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.103631496429443 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.103546619415283 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.103321552276611 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.101658821105957 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.102960109710693 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.1025776863098145 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.103583335876465 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.103630065917969 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.103322982788086 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.103185176849365 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.102198123931885 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.102322101593018 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.103001117706299 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.103229999542236 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.103567600250244 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.103597640991211 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.103576183319092 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.101297378540039 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.102327823638916 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.1029582023620605 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.103095054626465 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.103282928466797 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.103132724761963 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.102866172790527 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.102301597595215 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.101816177368164 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.1020965576171875 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.10333776473999 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.103382587432861 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.103857040405273 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.103193759918213 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.102137088775635 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.103390693664551 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.102590560913086 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.103017330169678 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.103185653686523 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.102924346923828 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.1027631759643555 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.1010870933532715 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.102463245391846 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.103203773498535 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.102989673614502 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.103216648101807 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.104397773742676 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.103231430053711 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.101535320281982 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.1034979820251465 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.1034040451049805 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.1031293869018555 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.10416316986084 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.103411674499512 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.103137969970703 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.101171016693115 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.100432872772217 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.101370811462402 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.102499961853027 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.103721618652344 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.105226516723633 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.103948593139648 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.100316047668457 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.098746299743652 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.1007561683654785 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.964634895324707 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.964853286743164 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.964779853820801 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.96457576751709 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.964901447296143 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.965076446533203 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.963896751403809 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.9644904136657715 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.964446067810059 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.964075088500977 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.9636616706848145 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.9634599685668945 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.963460922241211 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.963972568511963 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.963679313659668 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.964230060577393 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.963833332061768 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.9647674560546875 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.964493274688721 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.963866233825684 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.964780330657959 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.964466571807861 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.964788913726807 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.963519096374512 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.964328765869141 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.962802886962891 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.96372652053833 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.964566230773926 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.964580535888672 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.9645185470581055 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.964216232299805 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.964838981628418 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.963772296905518 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.965837001800537 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.964780330657959 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.964582443237305 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.964877128601074 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.963987827301025 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.96481466293335 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.963537693023682 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.964683532714844 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.964261054992676 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.964827537536621 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.964755535125732 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.964868545532227 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.964569091796875 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.963584899902344 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.964655876159668 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.9642839431762695 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.9651103019714355 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.964787483215332 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.964884281158447 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.965034008026123 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.963949680328369 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.964010238647461 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.963255882263184 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.964312553405762 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.964426517486572 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.96516227722168 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.963976860046387 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.963647365570068 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.964200019836426 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.964569568634033 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.965219497680664 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.964680194854736 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.964784145355225 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.964580535888672 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.9637675285339355 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.963949680328369 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.964078426361084 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.964617729187012 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.964517593383789 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.964454174041748 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.964346408843994 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.964151859283447 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.964664459228516 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.964797496795654 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.964831352233887 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.9650750160217285 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.96469783782959 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.964787483215332 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.964296340942383 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.96401834487915 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.964768886566162 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.965114593505859 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.965019226074219 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.965167045593262 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.964183330535889 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.964242935180664 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.964613914489746 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.965151309967041 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.964411735534668 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.964498519897461 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.9640607833862305 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.964939117431641 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.963999271392822 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.963999271392822 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.964320182800293 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.964476585388184 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.964928150177002 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.965571880340576 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.964280128479004 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.964073657989502 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.9643940925598145 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.964827537536621 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.964621067047119 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.964763641357422 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.964252948760986 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.964240074157715 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.964327335357666 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.964642524719238 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.964659214019775 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.964810848236084 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.9646830558776855 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.964807510375977 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.964704513549805 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.964130878448486 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.9653496742248535 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.9648542404174805 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.964399337768555 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.964700698852539 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.964797019958496 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.96461820602417 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.964078426361084 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.964522838592529 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.964656352996826 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.965064525604248 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.9648756980896 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.964382171630859 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.964518070220947 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.963754653930664 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.965043067932129 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.9638519287109375 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.964806079864502 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.964855670928955 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.964405059814453 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.964571475982666 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.963390827178955 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.964374542236328 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.964343070983887 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.964828968048096 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.964646816253662 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.9645915031433105 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.964428424835205 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.964176177978516 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.964416980743408 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.964077949523926 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.964686870574951 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.964921474456787 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.964619159698486 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.9647088050842285 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.963682651519775 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.964191913604736 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.9648261070251465 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.964590549468994 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.964537143707275 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.964656352996826 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.964632511138916 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.963538646697998 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.9644293785095215 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.963883399963379 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.964787483215332 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.964715003967285 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.964576244354248 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.964606285095215 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.964107036590576 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.964235305786133 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.964028358459473 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.964752197265625 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.964895248413086 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.964996814727783 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.964977741241455 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.963625431060791 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.9647111892700195 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.964138984680176 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.9649176597595215 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.964878082275391 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.964503765106201 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.964620590209961 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.963753700256348 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.964656352996826 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.964402198791504 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.964504718780518 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.9649176597595215 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.964693546295166 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.964619159698486 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.963278293609619 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.964744567871094 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.964730262756348 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.964637279510498 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.964547157287598 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.964498996734619 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.9650139808654785 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.963581562042236 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.9639692306518555 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.964494228363037 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.964633941650391 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.964561462402344 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.964945316314697 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.964648246765137 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.963507652282715 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.964051723480225 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.96372652053833 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.964704990386963 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.96455717086792 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.965011119842529 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.964638710021973 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.963899612426758 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.964288711547852 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.964208602905273 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.964895725250244 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.964487552642822 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.964959621429443 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.964356899261475 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.963806629180908 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.964376449584961 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.9645514488220215 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.9644083976745605 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.964900016784668 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.965603828430176 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.964944839477539 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.963250637054443 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.9628448486328125 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.9618940353393555 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.888533592224121 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.888609409332275 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.888508319854736 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.888683795928955 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.888564109802246 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.888050079345703 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.888296127319336 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.888637065887451 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.888768196105957 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.888139724731445 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.888930320739746 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.888051986694336 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.887570381164551 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.888113498687744 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.888205051422119 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.888602256774902 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.8882832527160645 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.888096809387207 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.887925148010254 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.888683795928955 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.887927532196045 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.888339996337891 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.8881306648254395 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.8882622718811035 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.887542724609375 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.887788772583008 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.888004302978516 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.887856483459473 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.888277053833008 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.888645648956299 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.888360977172852 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.8880085945129395 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.8879852294921875 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.887504577636719 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.887173175811768 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.888638496398926 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.888370513916016 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.888504981994629 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.887874126434326 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.888915538787842 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.889015197753906 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.888710975646973 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.888538837432861 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.88849401473999 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.888706684112549 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.8884735107421875 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.888428688049316 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.888202667236328 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.888516426086426 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.888519287109375 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.888779163360596 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.888981342315674 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.888237476348877 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.888044834136963 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.887911796569824 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.889214992523193 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.888627529144287 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.888639450073242 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.888596534729004 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.888563632965088 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.888638496398926 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.888349533081055 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.888360977172852 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.888574123382568 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.888939380645752 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.888828277587891 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.888360500335693 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.888573169708252 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.889019012451172 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.889037609100342 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.88852596282959 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.888480186462402 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.888377666473389 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.888641357421875 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.8884477615356445 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.888152122497559 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.888753890991211 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.888671875 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.88850212097168 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.888184547424316 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.888620853424072 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.888172626495361 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.888339042663574 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.888538360595703 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.888589382171631 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.888647556304932 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.888367176055908 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.888630390167236 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.8881964683532715 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.888460636138916 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.888651371002197 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.888354301452637 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.88836145401001 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.888493537902832 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.888124465942383 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.888256072998047 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.888362407684326 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.888558864593506 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.888823986053467 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.888772964477539 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.88839864730835 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.888772010803223 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.888427734375 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.888687610626221 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.888415336608887 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.888566970825195 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.88886022567749 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.888436794281006 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.888871669769287 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.888151168823242 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.888279914855957 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.888452053070068 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.8886847496032715 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.888696670532227 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.888360977172852 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.888774871826172 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.887986183166504 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.888555526733398 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.888382911682129 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.88839054107666 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.888365745544434 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.88832426071167 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.888453006744385 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.888110160827637 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.888450622558594 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.888492584228516 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.888844013214111 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.888654708862305 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.888604640960693 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.888652324676514 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.887988090515137 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.888352870941162 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.888880252838135 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.888491153717041 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.8884758949279785 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.888349533081055 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.888465404510498 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.888376235961914 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.8885602951049805 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.888815879821777 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.888659954071045 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.888651371002197 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.88850212097168 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.888739585876465 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.887884140014648 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.888887882232666 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.888637065887451 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.888782978057861 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.888707637786865 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.888581275939941 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.888576984405518 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.887940406799316 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.8884148597717285 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.88815975189209 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.888671398162842 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.888727188110352 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.8888397216796875 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.888592720031738 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.888171195983887 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.888138771057129 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.88839864730835 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.888693332672119 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.888722896575928 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.888566493988037 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.888630390167236 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.887894630432129 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.888371467590332 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.8883056640625 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.888612747192383 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.8885498046875 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.888686180114746 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.888600826263428 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.888392925262451 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.888243675231934 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.887849807739258 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.8886542320251465 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.888656139373779 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.888689994812012 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.888861656188965 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.88805627822876 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.8883466720581055 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.888495445251465 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.8885979652404785 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.888657093048096 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.888370513916016 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.888408660888672 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.888245582580566 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.887797832489014 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.88852071762085 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.888506889343262 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.888587474822998 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.888690948486328 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.888582706451416 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.888027667999268 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.8884406089782715 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.888558864593506 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.888631820678711 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.888638496398926 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.8885416984558105 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.888225555419922 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.888041973114014 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.888441562652588 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.888904094696045 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.888408660888672 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.888652801513672 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.888426780700684 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.888599395751953 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.887938976287842 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.888474464416504 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.888730525970459 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.8884687423706055 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.888638496398926 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.888818740844727 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.888711452484131 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.887609481811523 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.887635707855225 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.88871431350708 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.888470649719238 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.888686180114746 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.889449596405029 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.888946056365967 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.88793420791626 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.887157440185547 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.8879170417785645 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.846884727478027 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.846755027770996 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.846534729003906 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.846395492553711 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.846962928771973 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.847113132476807 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.846520900726318 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.846809387207031 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.846713066101074 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.846841335296631 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.846555233001709 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.846278190612793 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.846106052398682 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.846847057342529 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.84657621383667 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.846549987792969 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.8466362953186035 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.8466925621032715 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.846368789672852 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.846569538116455 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.846435070037842 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.846747875213623 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.8471479415893555 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.846069812774658 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.846944808959961 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.846384048461914 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.8465576171875 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.846332550048828 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.846738815307617 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.846657752990723 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.8466315269470215 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.846845626831055 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.846436500549316 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.84717321395874 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.846818923950195 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.846718788146973 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.846808910369873 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.846215724945068 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.846886157989502 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.846282005310059 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.846563816070557 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.8466267585754395 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.846828937530518 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.846994876861572 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.8467512130737305 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.846724033355713 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.8460187911987305 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.846347332000732 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.846561431884766 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.846907615661621 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.846936225891113 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.846603870391846 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.847090721130371 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.846323013305664 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.846722602844238 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.846107482910156 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.8468122482299805 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.8466105461120605 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.846934795379639 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.846707344055176 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.846489429473877 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.846768856048584 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.846676349639893 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.846982002258301 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.846992015838623 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.846632957458496 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.846888065338135 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.846412181854248 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.846067905426025 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.846593379974365 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.8468451499938965 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.846786975860596 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.84684419631958 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.847229957580566 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.846533298492432 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.846888065338135 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.846631050109863 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.847040176391602 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.847057819366455 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.84683084487915 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.846927642822266 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.846968650817871 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.846551418304443 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.84695291519165 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.846982002258301 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.846972465515137 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.846938133239746 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.846807956695557 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.846613883972168 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.846781253814697 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.846508502960205 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.84689998626709 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.846874237060547 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.846653938293457 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.846846580505371 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.846853256225586 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.846768856048584 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.846569538116455 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.846914768218994 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.846983909606934 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.847011089324951 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.846548080444336 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.846460819244385 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.846813678741455 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.846814155578613 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.8469462394714355 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.846782684326172 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.84659481048584 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.8464837074279785 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.8469648361206055 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.846916675567627 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.846741199493408 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.8471574783325195 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.846906661987305 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.846940994262695 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.846747875213623 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.846822738647461 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.84714412689209 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.846920967102051 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.846752166748047 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.846938610076904 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.846940040588379 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.846876621246338 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.846527576446533 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.846695423126221 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.8467864990234375 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.846994876861572 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.846986770629883 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.846661567687988 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.846889019012451 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.846558570861816 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.846768379211426 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.846653461456299 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.846931457519531 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.847026824951172 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.846749782562256 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.846560001373291 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.846208572387695 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.846831321716309 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.846343517303467 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.846974849700928 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.8468122482299805 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.846828937530518 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.846536636352539 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.846377372741699 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.8466315269470215 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.846134662628174 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.8467936515808105 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.846747398376465 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.846767902374268 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.8468122482299805 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.84634256362915 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.846466064453125 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.846446990966797 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.846789836883545 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.846935272216797 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.846797466278076 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.846704959869385 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.846429347991943 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.846837520599365 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.846591472625732 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.846845626831055 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.846902370452881 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.8468828201293945 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.846884727478027 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.846377372741699 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.846712589263916 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.846724033355713 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.846932411193848 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.846951484680176 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.846959114074707 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.846892833709717 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.845987319946289 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.84673547744751 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.846661567687988 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.846816539764404 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.8470354080200195 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.846877574920654 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.846774578094482 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.846625328063965 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.846673488616943 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.846678256988525 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.8467936515808105 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.846768856048584 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.846697807312012 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.846809387207031 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.846215724945068 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.847002983093262 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.846602916717529 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.846771717071533 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.846909523010254 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.8467912673950195 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.846930027008057 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.846335411071777 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.846407413482666 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.846812725067139 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.84692907333374 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.846823215484619 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.846906661987305 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.8469085693359375 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.846124649047852 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.846503734588623 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.846322536468506 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.846792221069336 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.84681510925293 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.847112655639648 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.84688663482666 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.846489906311035 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.846936225891113 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.846786975860596 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.846865653991699 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.847081184387207 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.846912860870361 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.846500396728516 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.84630012512207 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.846616744995117 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.84650993347168 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.846712589263916 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.846808910369873 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.847279071807861 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.846980571746826 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.845856189727783 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.8459672927856445 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.845388889312744 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.807320594787598 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.807447910308838 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.807397842407227 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.807557106018066 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.80731725692749 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.8072991371154785 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.807169437408447 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.80741024017334 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.807368278503418 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.806540489196777 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.807317733764648 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.807284832000732 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.806679725646973 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.807029724121094 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.806914806365967 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.80698823928833 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.806814193725586 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.80743408203125 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.807092189788818 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.807408809661865 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.806885242462158 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.807365894317627 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.807035446166992 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.80775260925293 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.8065619468688965 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.806995868682861 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.807311058044434 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.807106971740723 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.807308197021484 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.807418346405029 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.806776523590088 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.807122707366943 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.807094573974609 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.807129859924316 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.806788921356201 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.8074564933776855 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.80728006362915 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.8073344230651855 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.806854724884033 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.807610034942627 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.807583808898926 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.807412147521973 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.807064533233643 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.80736780166626 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.80729866027832 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.807192802429199 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.806939125061035 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.80729866027832 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.807465076446533 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.807305335998535 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.807262897491455 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.807672500610352 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.807365894317627 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.807225227355957 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.807041645050049 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.807146072387695 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.807360649108887 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.807380676269531 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.8074259757995605 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.807681083679199 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.807589530944824 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.80717658996582 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.807328701019287 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.807476997375488 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.807507514953613 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.807347297668457 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.807212829589844 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.8072710037231445 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.80779504776001 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.807518482208252 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.8072991371154785 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.807279109954834 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.807272434234619 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.807150363922119 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.807283878326416 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.807187557220459 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.807396411895752 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.807284832000732 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.807275295257568 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.807399272918701 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.807305812835693 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.80723762512207 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.80735445022583 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.807417869567871 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.807423114776611 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.8074541091918945 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.807394027709961 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.807457447052002 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.807441711425781 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.807389736175537 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.807151794433594 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.807308197021484 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.807285308837891 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.8072099685668945 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.807114601135254 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.807153224945068 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.8076395988464355 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.807408332824707 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.807433605194092 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.807414531707764 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.807464599609375 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.807225227355957 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.807118892669678 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.8074798583984375 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.807446002960205 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.807308197021484 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.807493686676025 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.807308673858643 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.8073601722717285 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.8069047927856445 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.807224273681641 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.80726957321167 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.807433128356934 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.807378768920898 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.807392597198486 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.807464599609375 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.807135105133057 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.807191848754883 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.80746603012085 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.807254791259766 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.807539939880371 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.807183265686035 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.807294845581055 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.8072052001953125 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.807293891906738 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.807287693023682 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.807423114776611 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.807347774505615 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.807369232177734 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.807397842407227 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.807180404663086 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.807460308074951 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.807249069213867 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.8072896003723145 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.807324409484863 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.806993007659912 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.807295799255371 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.807324409484863 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.807216167449951 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.8074049949646 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.807384967803955 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.807507038116455 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.807447910308838 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.8075151443481445 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.806991100311279 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.807608604431152 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.807516098022461 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.80740213394165 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.807366847991943 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.807394027709961 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.807435035705566 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.807272434234619 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.807493686676025 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.807199001312256 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.807322025299072 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.8074564933776855 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.8073296546936035 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.807380199432373 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.807191371917725 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.807180404663086 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.8075056076049805 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.807433128356934 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.807493209838867 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.807223796844482 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.807295799255371 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.807287693023682 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.807112693786621 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.807374477386475 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.807324409484863 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.807441234588623 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.807288646697998 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.8074116706848145 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.807272434234619 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.80704402923584 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.8070878982543945 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.8074564933776855 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.80736780166626 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.807421684265137 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.807408809661865 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.807161331176758 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.807325839996338 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.807275295257568 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.80742883682251 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.807389736175537 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.807441234588623 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.807344436645508 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.807398796081543 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.806964874267578 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.807369232177734 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.807373046875 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.807372093200684 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.807512283325195 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.807385444641113 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.807251453399658 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.807164669036865 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.807197570800781 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.807240009307861 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.807430267333984 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.807344913482666 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.80710506439209 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.806970596313477 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.806989669799805 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.8075642585754395 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.807403564453125 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.807308197021484 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.807372093200684 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.807277202606201 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.807153701782227 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.807036876678467 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.807571887969971 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.807326793670654 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.807351112365723 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.8075714111328125 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.807255268096924 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.80708646774292 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.807107925415039 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.807385444641113 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.807175159454346 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.807455062866211 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.807769298553467 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.807490348815918 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.807016372680664 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.806805610656738 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.806951999664307 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.7860870361328125 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.786165714263916 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.786190032958984 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.785765647888184 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.7859272956848145 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.78639030456543 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.785793304443359 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.786105632781982 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.785983562469482 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.7862372398376465 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.785848140716553 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.785799026489258 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.785938262939453 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.785671234130859 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.786051273345947 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.786233425140381 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.786227703094482 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.785523891448975 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.785677909851074 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.785831928253174 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.7858195304870605 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.786016464233398 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.786160945892334 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.78556489944458 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.786416053771973 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.7861504554748535 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.7858428955078125 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.785478591918945 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.78593111038208 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.785983562469482 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.786014556884766 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.785986423492432 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.785490989685059 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.786184310913086 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.7858195304870605 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.785972595214844 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.785904884338379 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.78585958480835 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.786273002624512 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.785771369934082 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.785844802856445 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.786144256591797 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.786036491394043 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.786085605621338 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.786350727081299 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.785879135131836 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.7859344482421875 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.785741806030273 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.785911560058594 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.78611421585083 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.786179542541504 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.786022186279297 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.785970687866211 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.785727024078369 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.786327838897705 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.785758972167969 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.78601598739624 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.78586483001709 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.786107540130615 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.785608291625977 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.7859110832214355 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.785917282104492 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.786117076873779 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.78607702255249 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.7861433029174805 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.785947322845459 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.785985946655273 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.7856950759887695 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.785669803619385 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.785665035247803 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.78601598739624 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.785956382751465 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.786189079284668 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.786167144775391 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.7859015464782715 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.78593635559082 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.785836219787598 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.78613805770874 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.785987854003906 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.786139488220215 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.785860061645508 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.785974025726318 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.7858567237854 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.78590202331543 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.7861127853393555 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.786235809326172 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.78599214553833 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.785956859588623 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.78582763671875 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.786026477813721 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.786167144775391 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.7860002517700195 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.785965919494629 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.785858154296875 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.785942554473877 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.786120891571045 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.7859601974487305 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.785771369934082 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.786098003387451 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.786062240600586 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.786027908325195 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.7860846519470215 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.785863399505615 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.78599214553833 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.786008358001709 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.786103248596191 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.786043643951416 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.785855293273926 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.785950660705566 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.786082744598389 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.786101818084717 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.786012649536133 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.786051273345947 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.786034107208252 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.785958290100098 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.7859930992126465 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.7859086990356445 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.7862467765808105 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.785863399505615 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.786041259765625 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.786000728607178 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.785933017730713 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.785943984985352 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.7859673500061035 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.785989284515381 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.785783767700195 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.786108016967773 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.786021709442139 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.786001682281494 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.785774230957031 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.785782814025879 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.785972595214844 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.785923957824707 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.786153316497803 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.786073207855225 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.786062240600586 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.785933017730713 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.785512924194336 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.786040306091309 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.785582542419434 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.786125659942627 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.786070346832275 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.785903453826904 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.785881519317627 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.7858052253723145 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.785869598388672 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.785634517669678 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.7860894203186035 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.785984992980957 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.78590202331543 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.785776615142822 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.785739898681641 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.785686016082764 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.7858710289001465 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.786015510559082 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.785984992980957 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.786074638366699 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.7859673500061035 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.7857794761657715 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.785918712615967 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.785735130310059 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.786016464233398 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.786036014556885 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.786085605621338 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.786025047302246 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.785719394683838 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.786073684692383 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.785758018493652 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.786116123199463 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.7859954833984375 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.7860894203186035 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.786044120788574 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.785635471343994 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.786078453063965 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.785892486572266 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.7859907150268555 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.786092281341553 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.785958290100098 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.785780429840088 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.7859063148498535 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.785931587219238 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.7859907150268555 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.786015510559082 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.78598690032959 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.785983562469482 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.786022186279297 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.785456657409668 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.786066055297852 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.7858195304870605 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.786002159118652 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.786025524139404 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.785871982574463 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.786020755767822 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.7857747077941895 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.785984516143799 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.786062717437744 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.785949230194092 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.786081314086914 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.7859272956848145 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.7860589027404785 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.785386562347412 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.7861127853393555 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.785547256469727 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.785964488983154 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.7859063148498535 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.786127090454102 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.785897731781006 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.785623550415039 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.785971164703369 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.78568696975708 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.785957336425781 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.7860236167907715 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.785878658294678 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.785817623138428 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.785684108734131 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.7856855392456055 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.785557270050049 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.78585958480835 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.785964488983154 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.786198139190674 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.7861008644104 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.785236835479736 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.785416126251221 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.785285472869873 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.763876914978027 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.763910293579102 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.763853549957275 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.76401424407959 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.763963222503662 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.7636260986328125 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.764211177825928 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.763911724090576 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.763829708099365 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.763540744781494 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.763612747192383 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.763721466064453 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.76350212097168 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.763619899749756 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.763597011566162 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.763787746429443 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.763580799102783 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.7638840675354 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.763800144195557 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.763984680175781 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.763599395751953 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.763930797576904 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.76369571685791 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.7638163566589355 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.7631916999816895 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.763453960418701 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.763749122619629 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.7637505531311035 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.763873100280762 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.763938903808594 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.763756275177002 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.763725280761719 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.763880729675293 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.763558864593506 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.763548851013184 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.763647556304932 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.763906478881836 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.763967037200928 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.763612747192383 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.763794422149658 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.764153003692627 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.763688087463379 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.763852119445801 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.763820648193359 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.763698101043701 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.7637410163879395 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.763702869415283 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.764163970947266 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.763774394989014 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.763908863067627 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.763912200927734 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.7638421058654785 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.763820648193359 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.763578414916992 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.763699054718018 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.764079570770264 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.763862133026123 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.763889312744141 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.7640557289123535 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.763990879058838 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.763952255249023 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.763833045959473 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.763771057128906 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.763866901397705 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.76412296295166 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.763780117034912 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.7637505531311035 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.763862133026123 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.763949394226074 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.764062404632568 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.763815402984619 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.763830184936523 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.763818740844727 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.76371431350708 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.763657569885254 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.76369571685791 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.764039039611816 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.763833045959473 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.763803958892822 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.763761520385742 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.764018535614014 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.76369571685791 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.763611793518066 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.763716220855713 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.763824939727783 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.763736724853516 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.763860702514648 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.763862133026123 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.763857841491699 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.763759613037109 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.763803005218506 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.7638092041015625 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.7638349533081055 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.763949871063232 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.763761043548584 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.763696193695068 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.763683319091797 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.763885498046875 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.763844013214111 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.7638373374938965 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.763776779174805 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.763764381408691 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.763606071472168 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.763893127441406 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.763791084289551 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.763823986053467 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.763840675354004 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.763827800750732 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.763771057128906 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.763636112213135 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.763758659362793 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.763873100280762 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.76384973526001 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.763851165771484 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.7639641761779785 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.763798713684082 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.763840675354004 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.763706684112549 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.763923168182373 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.763768196105957 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.763861179351807 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.763614177703857 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.763744831085205 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.763579368591309 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.763778209686279 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.763814449310303 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.763911724090576 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.763951301574707 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.763959884643555 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.7639641761779785 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.763771057128906 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.763907432556152 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.763689994812012 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.7637834548950195 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.7639617919921875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.763702392578125 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.7637858390808105 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.763891696929932 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.763716220855713 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.76397180557251 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.763879776000977 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.763941764831543 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.763826370239258 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.763874530792236 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.763741493225098 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.764047145843506 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.7637410163879395 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.76384973526001 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.763911724090576 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.76379919052124 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.76381778717041 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.763568878173828 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.763959884643555 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.763635158538818 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.7637200355529785 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.763876438140869 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.763875961303711 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.763876914978027 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.7636542320251465 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.763672351837158 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.763879776000977 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.763868808746338 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.763914108276367 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.7638654708862305 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.7637410163879395 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.763660907745361 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.763449668884277 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.7637619972229 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.76393985748291 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.763997554779053 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.763876914978027 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.763798236846924 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.763744831085205 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.763727188110352 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.763614177703857 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.763899326324463 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.763771057128906 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.763908863067627 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.7639899253845215 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.76370096206665 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.763743877410889 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.763840198516846 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.763787746429443 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.76381254196167 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.763828754425049 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.76360559463501 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.763652801513672 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.763684272766113 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.763845443725586 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.763819217681885 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.76375675201416 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.763904094696045 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.76383638381958 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.763662338256836 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.763668060302734 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.763660907745361 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.763773441314697 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.763828754425049 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.763808250427246 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.763736724853516 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.763669490814209 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.763576984405518 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.763879776000977 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.763696670532227 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.763768196105957 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.763823509216309 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.763830184936523 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.763809680938721 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.7637152671813965 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.763912677764893 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.763919830322266 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.763867378234863 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.764018535614014 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.7637739181518555 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.763439178466797 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.76362943649292 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.7639055252075195 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.763725280761719 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.763826370239258 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.763906002044678 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.763828277587891 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.763727188110352 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.76338005065918 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.76322603225708 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.750020980834961 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.750077724456787 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.750213623046875 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.7500786781311035 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.7501044273376465 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.750396728515625 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.74979829788208 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.750003814697266 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.750091075897217 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.750096321105957 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.750409126281738 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.750068664550781 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.749920845031738 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.749948024749756 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.749916076660156 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.749958038330078 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.7501540184021 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.749876976013184 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.7501115798950195 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.749826908111572 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.749764919281006 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.750082492828369 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.750151634216309 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.749709606170654 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.750182628631592 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.750070095062256 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.74973201751709 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.749866008758545 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.750063419342041 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.749924182891846 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.749812602996826 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.749690532684326 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.749947547912598 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.750251293182373 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.749931812286377 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.750001907348633 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.749978542327881 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.7501020431518555 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.750133514404297 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.749849796295166 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.749995231628418 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.750255584716797 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.75007438659668 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.750039100646973 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.750225067138672 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.7498674392700195 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.750048637390137 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.74960994720459 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.750172138214111 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.750096321105957 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.750021934509277 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.7498884201049805 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.750110149383545 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.749950408935547 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.749808311462402 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.749939441680908 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.750105857849121 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.7499566078186035 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.75016450881958 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.750014305114746 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.749876499176025 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.7500810623168945 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.750097274780273 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.750137805938721 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.7501068115234375 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.750007629394531 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.749920845031738 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.749889373779297 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.750068664550781 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.749770164489746 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.74998140335083 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.7499895095825195 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.750071048736572 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.750212669372559 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.749993801116943 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.750055313110352 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.74984073638916 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.75002384185791 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.749930381774902 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.749964714050293 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.749937057495117 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.749963283538818 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.750051021575928 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.750151634216309 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.750094890594482 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.750199794769287 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.750016212463379 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.749950885772705 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.749963283538818 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.750150680541992 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.74990177154541 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.750068187713623 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.749967575073242 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.750001430511475 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.7498674392700195 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.750048637390137 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.750049591064453 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.749871730804443 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.750004768371582 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.750156402587891 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.7500319480896 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.750095844268799 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.749948978424072 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.749963283538818 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.750036239624023 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.749994277954102 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.750004768371582 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.7499799728393555 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.749864101409912 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.749906063079834 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.750154972076416 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.749868869781494 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.750053882598877 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.749982833862305 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.749892234802246 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.750008583068848 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.74986457824707 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.750156879425049 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.7498779296875 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.7501044273376465 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.750033378601074 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.750000476837158 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.7498297691345215 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.750145435333252 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.750127792358398 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.749939918518066 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.750089645385742 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.750092029571533 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.749913692474365 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.749995231628418 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.74984884262085 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.749996185302734 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.750014305114746 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.7500786781311035 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.750009536743164 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.750150203704834 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.75006628036499 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.7498250007629395 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.750083923339844 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.749686241149902 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.750180244445801 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.749969005584717 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.749982833862305 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.750037670135498 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.7498908042907715 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.749860763549805 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.749767780303955 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.75007963180542 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.750010013580322 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.750041961669922 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.750009536743164 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.749959945678711 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.749973773956299 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.7500200271606445 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.750053882598877 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.750007629394531 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.750096321105957 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.750121116638184 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.74991512298584 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.7500762939453125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.749960422515869 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.7500200271606445 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.750077247619629 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.75005578994751 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.750024795532227 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.749948024749756 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.750172138214111 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.75 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.750148773193359 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.750057697296143 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.7499799728393555 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.7500786781311035 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.74978494644165 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.749974250793457 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.750060081481934 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.749953746795654 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.750132083892822 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.750023365020752 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.749988079071045 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.74991512298584 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.749953746795654 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.749972820281982 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.750073432922363 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.750045299530029 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.749997138977051 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.750068187713623 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.749803066253662 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.7501068115234375 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.749828815460205 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.749999046325684 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.750051021575928 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.750063896179199 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.750000476837158 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.749790668487549 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.749943256378174 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.750059127807617 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.749959468841553 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.7500319480896 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.750086784362793 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.749938488006592 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.7497076988220215 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.75007963180542 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.749924659729004 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.750041961669922 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.749931812286377 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.7501678466796875 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.749998569488525 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.749776363372803 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.750130653381348 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.749953746795654 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.750038146972656 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.750070095062256 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.749917507171631 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.7498884201049805 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.7498250007629395 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.7501678466796875 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.7499165534973145 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.749966621398926 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.750113487243652 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.750253200531006 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.750173568725586 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.749752521514893 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.74944543838501 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.749748229980469 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.735749244689941 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.735605716705322 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.735559463500977 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.735401630401611 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.735648155212402 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.735724925994873 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.735809803009033 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.735662937164307 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.735661506652832 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.735718250274658 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.735525608062744 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.735543251037598 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.735413074493408 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.735805034637451 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.7355475425720215 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.73565673828125 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.735165596008301 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.735630512237549 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.735355377197266 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.735599517822266 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.73553466796875 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.735842704772949 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.735633850097656 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.73579740524292 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.735411643981934 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.735323429107666 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.735809803009033 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.735527038574219 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.735580921173096 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.735686779022217 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.735784530639648 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.735810279846191 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.735538482666016 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.735507488250732 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.735589027404785 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.735638618469238 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.735682964324951 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.735683441162109 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.735445499420166 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.735910415649414 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.735760688781738 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.735584259033203 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.735650539398193 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.735589027404785 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.735595226287842 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.73568058013916 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.7354302406311035 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.735755920410156 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.735533237457275 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.7357025146484375 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.735727787017822 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.735914707183838 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.735564708709717 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.735570907592773 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.735718250274658 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.735691070556641 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.735559463500977 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.735669136047363 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.736020565032959 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.735628604888916 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.735891819000244 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.73552131652832 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.73578405380249 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.735596656799316 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.735816478729248 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.735847473144531 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.735689640045166 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.735713958740234 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.735567092895508 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.735745429992676 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.7356743812561035 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.73566198348999 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.735579967498779 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.735451698303223 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.735611915588379 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.7354865074157715 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.735877990722656 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.735663414001465 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.735743999481201 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.735640048980713 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.735723495483398 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.735437870025635 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.735717296600342 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.73552942276001 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.7357048988342285 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.735743045806885 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.735779762268066 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.735907077789307 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.735551834106445 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.735812664031982 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.735816478729248 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.735585689544678 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.735698699951172 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.735727787017822 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.735485076904297 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.735496520996094 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.735637187957764 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.73578405380249 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.735725402832031 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.7356953620910645 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.735657215118408 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.73576545715332 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.735666275024414 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.735670566558838 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.735718250274658 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.735681533813477 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.735720634460449 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.735648155212402 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.735612869262695 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.7354960441589355 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.735509872436523 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.7357869148254395 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.735675811767578 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.735697269439697 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.735825061798096 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.735678672790527 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.7356276512146 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.735659599304199 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.735790729522705 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.735655307769775 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.7357354164123535 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.735655307769775 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.735683917999268 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.735528945922852 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.7355241775512695 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.735678672790527 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.735630989074707 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.735649585723877 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.735651016235352 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.735783576965332 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.73565673828125 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.735672473907471 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.7355170249938965 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.735677242279053 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.735748291015625 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.735722064971924 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.735543251037598 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.735703945159912 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.735637187957764 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.735703468322754 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.7356367111206055 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.735689640045166 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.735782623291016 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.735724925994873 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.735607624053955 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.7357988357543945 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.735667705535889 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.735719680786133 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.7357048988342285 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.7355146408081055 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.735562801361084 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.735489845275879 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.735600471496582 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.735648155212402 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.735593795776367 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.735612869262695 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.735564708709717 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.735640048980713 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.735483646392822 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.735544681549072 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.735790729522705 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.7357330322265625 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.73561429977417 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.735673904418945 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.735618591308594 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.735533714294434 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.735453128814697 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.735683917999268 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.735688209533691 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.735688209533691 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.735729217529297 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.7357401847839355 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.735551834106445 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.735549449920654 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.735442161560059 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.735749244689941 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.735632419586182 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.735667705535889 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.735859394073486 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.735560894012451 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.735610485076904 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.735659599304199 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.735598564147949 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.735591888427734 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.735774517059326 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.735651016235352 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.735477447509766 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.7354536056518555 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.735658168792725 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.735630035400391 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.735575199127197 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.735724449157715 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.735669136047363 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.735570907592773 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.735556125640869 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.7355146408081055 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.735653877258301 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.735621452331543 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.735678672790527 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.7357940673828125 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.7357072830200195 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.735198497772217 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.735698699951172 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.735628604888916 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.735651016235352 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.735590934753418 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.735592365264893 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.735528469085693 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.735500812530518 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.735600471496582 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.735644340515137 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.73559045791626 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.735889911651611 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.735602855682373 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.735527038574219 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.735183238983154 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.735662937164307 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.735490322113037 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.735570430755615 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.735771179199219 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.735713958740234 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.735530376434326 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.735454559326172 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.735284328460693 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.727272033691406 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.727263927459717 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.727447986602783 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.727210521697998 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.7273149490356445 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.727382659912109 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.727174282073975 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.727188587188721 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.727271556854248 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.727020740509033 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.727312088012695 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.727245807647705 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.72697114944458 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.727160930633545 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.727227210998535 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.727433681488037 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.727734565734863 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.727424621582031 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.727383136749268 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.727261543273926 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.727002143859863 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.727267742156982 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.72728967666626 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.727218151092529 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.7272820472717285 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.7273101806640625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.727075576782227 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.727088451385498 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.727281093597412 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.727230548858643 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.727171421051025 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.727014064788818 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.7270002365112305 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.7271623611450195 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.7271857261657715 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.727338790893555 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.727336406707764 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.727290630340576 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.727341651916504 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.726978302001953 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.727092742919922 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.727373123168945 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.727230072021484 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.727346897125244 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.727179527282715 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.727208614349365 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.727406024932861 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.727031230926514 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.727427959442139 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.727264404296875 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.727366924285889 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.727170944213867 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.727219104766846 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.727188587188721 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.727202415466309 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.727203845977783 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.727184295654297 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.727207660675049 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.727180480957031 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.727263927459717 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.727007865905762 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.727344512939453 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.727258205413818 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.72737979888916 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.727283954620361 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.727219581604004 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.7271528244018555 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.726965427398682 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.727278232574463 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.727016448974609 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.727213382720947 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.727233409881592 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.727285385131836 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.727283954620361 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.727242469787598 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.727166652679443 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.727079391479492 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.727243900299072 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.727213382720947 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.7272257804870605 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.727101802825928 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.72724723815918 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.727171421051025 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.727391719818115 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.727233409881592 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.727422714233398 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.727375030517578 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.727240562438965 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.727254390716553 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.727351665496826 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.727051734924316 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.727200984954834 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.727278232574463 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.727243900299072 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.727210521697998 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.727277755737305 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.727265357971191 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.727119445800781 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.727256774902344 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.727424621582031 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.72728967666626 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.7272162437438965 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.727060794830322 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.727225303649902 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.727285861968994 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.727351665496826 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.727291584014893 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.727375030517578 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.727297782897949 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.7272210121154785 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.727382659912109 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.727233409881592 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.727297782897949 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.727248191833496 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.727179050445557 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.727337837219238 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.727230548858643 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.727355480194092 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.727076530456543 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.727263927459717 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.727281093597412 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.727174282073975 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.727208614349365 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.727336406707764 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.727266311645508 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.727136135101318 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.727245330810547 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.727322578430176 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.727303981781006 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.727168083190918 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.727179050445557 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.727265357971191 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.727177619934082 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.727233409881592 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.727229595184326 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.727142810821533 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.727259159088135 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.72700309753418 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.727283954620361 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.726864337921143 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.727231502532959 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.727243900299072 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.727282524108887 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.727181911468506 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.727109432220459 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.727280139923096 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.727066993713379 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.727230548858643 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.727263927459717 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.727174758911133 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.727259159088135 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.727238178253174 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.72711181640625 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.727234840393066 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.727338790893555 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.727208614349365 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.727366924285889 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.727307319641113 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.727176666259766 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.727256774902344 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.727062225341797 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.727206707000732 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.727266311645508 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.727261066436768 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.727260589599609 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.727237701416016 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.727290630340576 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.727205276489258 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.727319717407227 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.7273149490356445 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.727163791656494 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.727313995361328 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.727117538452148 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.727297782897949 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.727334022521973 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.7272562980651855 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.7273383140563965 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.727236270904541 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.727217674255371 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.727133274078369 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.727233409881592 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.727234840393066 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.727259635925293 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.727290630340576 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.727233409881592 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.727207660675049 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.727382659912109 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.727365970611572 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.7271857261657715 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.727353572845459 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.727325439453125 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.727242946624756 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.727260589599609 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.72725772857666 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.72722053527832 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.727349281311035 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.727334022521973 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.727223873138428 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.727336406707764 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.727236747741699 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.726943016052246 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.727406978607178 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.727139472961426 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.72732400894165 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.727234840393066 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.727407455444336 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.727261066436768 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.7272748947143555 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.727225303649902 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.727275371551514 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.727323055267334 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.727292060852051 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.727206707000732 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.727131366729736 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.727207183837891 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.727283954620361 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.727217674255371 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.727317810058594 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.727277755737305 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.727339744567871 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.727353572845459 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.727086544036865 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.727168083190918 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.7270188331604 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 5.716471195220947 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 5.7165021896362305 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 5.716497421264648 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 5.716405868530273 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 5.716554641723633 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 5.716458320617676 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 5.716691017150879 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 5.716512203216553 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 5.716405391693115 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 5.716665744781494 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 5.716557025909424 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 5.716489791870117 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 5.716475486755371 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 5.716529369354248 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 5.716424942016602 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 5.716368675231934 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 5.71601676940918 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 5.716370582580566 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 5.7161784172058105 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 5.716451168060303 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 5.716567516326904 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 5.716444969177246 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 5.716471195220947 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 5.716361045837402 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 5.716320037841797 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 5.716214656829834 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 5.716564655303955 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 5.716272354125977 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 5.716392517089844 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 5.716505527496338 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 5.716416835784912 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 5.716427326202393 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 5.716489315032959 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 5.716395854949951 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 5.716413974761963 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 5.7164692878723145 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 5.716431140899658 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 5.7164387702941895 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 5.716334342956543 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 5.7166547775268555 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 5.716627597808838 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 5.716324806213379 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 5.716473579406738 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 5.7164306640625 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 5.716298580169678 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 5.716250896453857 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 5.716250896453857 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 5.716433525085449 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 5.716359615325928 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 5.716538429260254 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 5.716472625732422 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 5.716551303863525 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 5.716439723968506 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 5.716416358947754 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 5.716301441192627 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 5.716520309448242 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 5.716432094573975 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 5.716367244720459 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 5.7166218757629395 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 5.7164154052734375 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 5.716714382171631 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 5.716241359710693 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 5.716564178466797 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 5.716377258300781 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 5.716472148895264 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 5.716655731201172 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 5.716405868530273 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 5.716425895690918 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 5.716358661651611 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 5.716618061065674 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 5.716516494750977 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 5.716457843780518 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 5.716426372528076 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 5.716289520263672 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 5.71643590927124 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 5.7161641120910645 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 5.716543197631836 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 5.716419696807861 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 5.716540336608887 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 5.716250896453857 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 5.716429710388184 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 5.716384410858154 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 5.716546535491943 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 5.716248035430908 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 5.7163872718811035 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 5.716500282287598 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 5.716453552246094 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 5.716541290283203 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 5.716310501098633 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 5.7164998054504395 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 5.71669864654541 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 5.716495513916016 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 5.716455459594727 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 5.716395378112793 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 5.716398239135742 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 5.7163801193237305 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 5.716507911682129 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 5.716575622558594 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 5.716522693634033 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 5.7164106369018555 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 5.716610431671143 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 5.7163896560668945 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 5.7164835929870605 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 5.716461658477783 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 5.716370582580566 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 5.7164788246154785 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 5.716495990753174 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 5.716524600982666 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 5.716317176818848 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 5.716423034667969 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 5.716396808624268 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 5.716536045074463 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 5.716484546661377 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 5.7164506912231445 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 5.716594219207764 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 5.7163920402526855 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 5.7164225578308105 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 5.716315746307373 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 5.716479778289795 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 5.716516494750977 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 5.716393947601318 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 5.716498851776123 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 5.71648645401001 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 5.716279983520508 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 5.716372013092041 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 5.716432571411133 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 5.716487884521484 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 5.71643590927124 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 5.7163872718811035 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 5.716500282287598 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 5.716414928436279 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 5.716492176055908 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 5.716322898864746 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 5.716555595397949 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 5.716487884521484 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 5.716487407684326 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 5.716372013092041 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 5.716485023498535 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 5.716375827789307 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 5.716655731201172 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 5.716384410858154 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 5.716467380523682 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 5.716480255126953 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 5.716492652893066 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 5.716387748718262 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 5.716408729553223 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 5.716521263122559 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 5.716517925262451 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 5.716440200805664 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 5.71645975112915 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 5.716413974761963 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 5.716322898864746 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 5.716640472412109 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 5.716489315032959 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 5.716421604156494 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 5.716449737548828 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 5.716375350952148 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 5.7164788246154785 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 5.7164788246154785 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 5.716489315032959 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 5.716472625732422 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 5.7164835929870605 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 5.716504096984863 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 5.7164788246154785 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 5.7164812088012695 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 5.716255187988281 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 5.7164998054504395 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 5.716384410858154 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 5.716497421264648 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 5.7164387702941895 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 5.716585159301758 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 5.716421127319336 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 5.716434478759766 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 5.716467380523682 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 5.716280460357666 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 5.716424942016602 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 5.716375827789307 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 5.716519832611084 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 5.7164306640625 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 5.716427326202393 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 5.716678142547607 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 5.716527462005615 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 5.716350555419922 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 5.716449737548828 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 5.716461658477783 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 5.716452598571777 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 5.716210842132568 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 5.716273307800293 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 5.716616630554199 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 5.716398239135742 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 5.716460227966309 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 5.716630935668945 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 5.716404438018799 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 5.716335296630859 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 5.71648645401001 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 5.716475963592529 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 5.716441631317139 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 5.71636962890625 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 5.7164306640625 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 5.716403484344482 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 5.716465950012207 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 5.716305732727051 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 5.716561794281006 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 5.716446399688721 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 5.7164483070373535 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 5.7163310050964355 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 5.716424465179443 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 5.716359615325928 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 5.7164306640625 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 5.7163310050964355 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 5.716396808624268 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 5.71647834777832 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 5.716632843017578 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 5.716471195220947 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 5.716372013092041 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 5.716257095336914 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 5.7162981033325195 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 5.716318130493164 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 5.716458320617676 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 5.716487884521484 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 5.716555595397949 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 5.716307163238525 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 5.716017723083496 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 5.716212272644043 |
| } |
| ] |
| } |
| ] |
| } |
