| { |
| "original_ppl": 13.899003028869629, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 115978.984375 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 117935.8984375 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 130077.734375 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 126700.8671875 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 190005.09375 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 186942.875 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 138211.484375 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 115656.5703125 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 121126.0390625 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 99458.8046875 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 188835.96875 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 135572.09375 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 112415.7265625 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 65246.640625 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 118356.6171875 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 117979.9921875 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 186615.125 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 74303.953125 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 122958.671875 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 113680.765625 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 134391.40625 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 115220.6171875 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 115232.703125 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 315601.09375 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 35685.0703125 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 98983.40625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 101194.8515625 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 225972.5625 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 117586.6328125 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 116418.1484375 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 132355.8125 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 133306.265625 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 184612.578125 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 146822.453125 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 77345.875 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 115174.0390625 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 107967.375 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 147275.5625 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 152333.484375 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 146245.15625 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 239209.640625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 188546.25 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 115871.4140625 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 117346.4453125 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 136709.171875 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 139127.53125 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 114270.4375 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 169269.53125 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 182424.53125 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 116332.90625 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 115020.8046875 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 124089.5546875 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 97838.4296875 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 131759.734375 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 125108.734375 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 131944.96875 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 115727.84375 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 112676.9765625 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 119787.40625 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 145996.140625 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 113413.625 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 77397.2265625 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 125113.984375 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 117445.640625 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 113556.8125 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 109536.8515625 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 122423.359375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 138422.53125 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 56440.8515625 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 165643.390625 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 116976.9453125 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 117724.1953125 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 86124.9921875 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 66275.765625 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 111009.875 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 92160.3203125 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 93803.1484375 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 118710.2109375 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 116078.4609375 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 152364.734375 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 142038.234375 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 141696.75 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 130260.5859375 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 106812.7578125 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 115725.640625 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 114445.7109375 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 150791.828125 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 132950.265625 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 167486.15625 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 136850.3125 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 137191.890625 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 115154.4921875 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 114831.40625 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 107067.8203125 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 165852.5 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 125789.46875 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 105294.6640625 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 94707.6875 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 114872.484375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 115165.25 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 93756.4609375 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 206517.84375 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 142689.09375 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 133557.46875 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 105951.421875 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 114843.34375 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 114862.4140625 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 90556.28125 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 120032.8125 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 110944.2578125 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 141141.359375 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 116927.3125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 114495.4921875 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 115596.9140625 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 106099.75 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 115500.9375 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 131227.40625 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 114856.3828125 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 135165.4375 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 114792.4296875 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 115122.96875 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 156268.4375 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 149494.6875 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 109665.3125 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 113299.1484375 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 97967.1796875 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 115980.3125 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 114208.015625 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 105210.046875 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 126556.4375 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 126279.1484375 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 125695.21875 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 114645.9375 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 115147.4609375 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 115042.3046875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 92032.3515625 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 103167.3203125 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 133661.0625 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 135656.9375 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 122762.5234375 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 116163.0703125 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 116294.6484375 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 112999.8125 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 92730.90625 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 126445.3203125 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 124262.8125 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 93129.9921875 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 114325.1640625 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 115789.0078125 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 108390.5703125 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 210341.546875 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 120934.5546875 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 101062.9140625 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 137005.703125 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 115685.6953125 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 114676.0078125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 117970.3203125 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 115191.8359375 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 117455.6015625 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 88665.1328125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 106945.265625 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 115541.921875 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 115466.90625 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 47576.62109375 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 153542.453125 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 114219.34375 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 76318.890625 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 144987.96875 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 1104.77734375 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 1117.5347900390625 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 1085.104248046875 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 772.0201416015625 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 707.7988891601562 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 729.6810302734375 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 767.7919921875 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 1104.3433837890625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 1100.5977783203125 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 975.0816040039062 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 1177.628173828125 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 890.8675537109375 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 953.7259521484375 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 1043.837158203125 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 1106.888671875 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 1103.126953125 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 1018.44970703125 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 1080.5126953125 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 848.2011108398438 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 969.7440795898438 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 1040.535400390625 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 1105.650634765625 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 1016.8173217773438 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 1042.3221435546875 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 1332.68505859375 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 968.755126953125 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 1115.97607421875 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 1111.03076171875 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 1100.5535888671875 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 1106.374267578125 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 1056.422119140625 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 1129.7744140625 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 1077.334716796875 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 1031.3544921875 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 950.3606567382812 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 1132.03759765625 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 1057.854248046875 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 1009.041015625 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 1234.8411865234375 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 1061.481201171875 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 1018.9150390625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 1095.8421630859375 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 1104.73095703125 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 1083.9547119140625 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 1136.5213623046875 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 1073.5491943359375 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 1056.22216796875 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 1065.378662109375 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 1097.4366455078125 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 1103.91259765625 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 1074.64208984375 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 1128.4024658203125 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 1038.6903076171875 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 1095.4525146484375 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 999.6639404296875 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 1089.017822265625 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 1104.580810546875 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 1098.0616455078125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 1079.3143310546875 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 1118.2340087890625 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 1070.6536865234375 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 1110.5751953125 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 1070.402587890625 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 1110.7764892578125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 1131.66845703125 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 1140.68896484375 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 1028.2171630859375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 1098.1480712890625 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 1021.2586059570312 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 932.7628784179688 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 1104.95068359375 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 1105.21728515625 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 1094.091552734375 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 936.4127197265625 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 1085.2047119140625 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 1035.9390869140625 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 1093.98828125 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 1133.919921875 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 1083.7330322265625 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 1077.087158203125 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 1077.116943359375 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 1058.9185791015625 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 1015.3550415039062 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 1061.5986328125 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 1103.0523681640625 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 1108.2164306640625 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 1076.389892578125 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 1001.43212890625 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 1112.908203125 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 1042.40966796875 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 1210.527099609375 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 1114.3468017578125 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 1109.5843505859375 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 1113.113037109375 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 1031.4444580078125 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 1080.88525390625 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 1081.2296142578125 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 1146.0684814453125 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 1107.8533935546875 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 1097.6942138671875 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 1082.7237548828125 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 1060.6837158203125 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 1085.4737548828125 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 1102.57958984375 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 1085.2523193359375 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 1106.208984375 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 1120.32666015625 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 1127.364013671875 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 1125.56884765625 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 1067.0594482421875 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 1037.7684326171875 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 1078.5355224609375 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 1101.0738525390625 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 1129.101806640625 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 1010.1222534179688 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 1075.365966796875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 1079.19091796875 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 1117.4888916015625 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 1089.1732177734375 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 1115.8536376953125 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 1111.6851806640625 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 1084.6429443359375 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 1133.7852783203125 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 1093.816162109375 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 1064.03125 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 1097.9757080078125 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 1110.746337890625 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 1104.7994384765625 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 1109.0955810546875 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 1162.3387451171875 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 1078.2613525390625 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 1103.8699951171875 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 1119.7039794921875 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 1111.099609375 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 1107.1044921875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 1079.3978271484375 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 1102.4886474609375 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 1120.1856689453125 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 1070.478515625 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 1096.9327392578125 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 1106.4886474609375 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 1100.1156005859375 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 1094.5120849609375 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 1126.524658203125 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 1089.9425048828125 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 1150.4691162109375 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 1110.9305419921875 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 1099.633544921875 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 1101.825439453125 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 1106.230712890625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 1113.47509765625 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 1101.0975341796875 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 1155.2440185546875 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 1116.1910400390625 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 1104.1595458984375 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 1099.4454345703125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 1104.2738037109375 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 1070.5985107421875 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 1113.9180908203125 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 1015.337646484375 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 1115.7642822265625 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 1108.263427734375 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 1108.5399169921875 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 955.1649780273438 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 1113.8489990234375 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 1045.75927734375 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 1069.7076416015625 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 925.7997436523438 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 39.35506057739258 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 39.32781219482422 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 39.344608306884766 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 39.02629089355469 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 38.6014289855957 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 38.14826202392578 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 38.43531799316406 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 39.28401184082031 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 39.30514907836914 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 39.816959381103516 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 40.805973052978516 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 38.77671432495117 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 39.26633071899414 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 38.64971923828125 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 39.341270446777344 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 39.077335357666016 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 39.261043548583984 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 38.458290100097656 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 37.59619140625 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 38.17994689941406 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 38.458473205566406 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 38.706703186035156 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 36.84185791015625 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 37.98421859741211 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 38.619083404541016 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 38.61041259765625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 39.18088912963867 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 39.098472595214844 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 39.23974609375 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 39.61964416503906 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 39.3549690246582 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 39.145118713378906 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 38.93742370605469 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 39.02334976196289 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 38.851627349853516 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 39.36909866333008 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 39.270347595214844 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 38.94121170043945 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 39.12631607055664 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 38.85331726074219 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 39.37434768676758 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 39.036033630371094 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 39.34165573120117 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 39.064945220947266 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 39.20331573486328 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 39.3511962890625 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 38.58588790893555 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 39.03982162475586 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 39.29458999633789 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 39.37360763549805 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 38.484375 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 38.8640251159668 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 38.253318786621094 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 38.72230911254883 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 39.323848724365234 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 39.26142120361328 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 39.20205307006836 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 39.283935546875 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 39.101192474365234 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 39.273616790771484 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 38.77415466308594 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 39.34434509277344 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 38.73604965209961 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 39.305641174316406 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 39.09168243408203 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 39.427833557128906 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 39.2554817199707 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 38.622859954833984 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 38.93667984008789 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 38.95016098022461 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 39.359718322753906 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 38.94915008544922 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 39.1114387512207 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 39.50409698486328 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 38.824607849121094 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 39.110198974609375 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 39.579612731933594 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 39.32085418701172 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 39.225059509277344 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 38.929542541503906 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 39.6527214050293 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 39.20378112792969 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 38.965396881103516 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 39.70588684082031 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 39.363224029541016 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 39.47519302368164 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 39.46685791015625 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 39.407508850097656 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 39.23819351196289 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 39.23158645629883 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 39.517364501953125 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 39.290706634521484 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 39.40549850463867 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 39.4801025390625 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 39.33195877075195 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 39.275123596191406 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 39.18341064453125 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 39.46139907836914 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 39.30316162109375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 39.251953125 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 39.366641998291016 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 39.12328338623047 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 39.178287506103516 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 39.13130569458008 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 39.432003021240234 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 39.340370178222656 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 39.32317352294922 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 39.40216064453125 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 39.365570068359375 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 39.081546783447266 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 39.12284469604492 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 39.26258850097656 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 39.46735763549805 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 39.48711013793945 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 39.49180221557617 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 39.29378128051758 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 39.07826614379883 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 39.180362701416016 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 38.841392517089844 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 39.355838775634766 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 39.4254035949707 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 39.33624267578125 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 39.51812744140625 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 39.14421081542969 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 39.18684768676758 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 39.18109130859375 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 39.307891845703125 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 39.263694763183594 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 39.25142288208008 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 39.18424987792969 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 39.21150207519531 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 39.30417251586914 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 39.21913146972656 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 39.292076110839844 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 39.24518966674805 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 39.344356536865234 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 39.31454086303711 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 39.11281204223633 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 39.200714111328125 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 38.98899841308594 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 39.37110137939453 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 39.317264556884766 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 39.16870880126953 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 39.41497802734375 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 39.161006927490234 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 39.041778564453125 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 39.23173904418945 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 39.33845520019531 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 39.31966781616211 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 39.354217529296875 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 39.35450744628906 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 39.13120651245117 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 39.06975173950195 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 39.14295959472656 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 39.3288459777832 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 39.30983352661133 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 39.37760543823242 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 39.17430877685547 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 39.225669860839844 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 39.250953674316406 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 39.138572692871094 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 39.34307098388672 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 39.3690071105957 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 38.6165771484375 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 39.2541618347168 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 38.97572708129883 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 38.59062576293945 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 37.79502868652344 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 19.779138565063477 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 19.775676727294922 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 19.770112991333008 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 19.74167251586914 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 19.707183837890625 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 19.707674026489258 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 19.63474464416504 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 19.77523422241211 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 19.76637077331543 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 19.80510139465332 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 19.828384399414062 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 19.718727111816406 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 19.743885040283203 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 19.80364227294922 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 19.7819766998291 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 19.789459228515625 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 19.767101287841797 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 19.749011993408203 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 19.663976669311523 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 19.71395492553711 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 19.73113441467285 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 19.763832092285156 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 19.79463005065918 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 19.62027931213379 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 19.77900505065918 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 19.70415687561035 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 19.768369674682617 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 19.75008201599121 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 19.777774810791016 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 19.794885635375977 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 19.775798797607422 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 19.75157928466797 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 19.732017517089844 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 19.754131317138672 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 19.736234664916992 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 19.788288116455078 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 19.773229598999023 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 19.764251708984375 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 19.7569522857666 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 19.69136619567871 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 19.752803802490234 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 19.754301071166992 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 19.784259796142578 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 19.775474548339844 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 19.78266143798828 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 19.78754234313965 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 19.70989227294922 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 19.737512588500977 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 19.75505828857422 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 19.799283981323242 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 19.595666885375977 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 19.77581024169922 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 19.79781150817871 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 19.737442016601562 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 19.726085662841797 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 19.716650009155273 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 19.784608840942383 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 19.779481887817383 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 19.7554988861084 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 19.764514923095703 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 19.71770668029785 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 19.7531795501709 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 19.7639217376709 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 19.782787322998047 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 19.76932144165039 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 19.768030166625977 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 19.79119110107422 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 19.717927932739258 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 19.773277282714844 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 19.79021644592285 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 19.78032112121582 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 19.761714935302734 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 19.775487899780273 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 19.797565460205078 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 19.716480255126953 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 19.763996124267578 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 19.78944969177246 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 19.773893356323242 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 19.776586532592773 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 19.76559829711914 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 19.702566146850586 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 19.757862091064453 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 19.742891311645508 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 19.743349075317383 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 19.774324417114258 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 19.772764205932617 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 19.753416061401367 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 19.73722267150879 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 19.748517990112305 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 19.773574829101562 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 19.769363403320312 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 19.794649124145508 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 19.77198028564453 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 19.781579971313477 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 19.75636100769043 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 19.768020629882812 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 19.75920867919922 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 19.756240844726562 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 19.77730369567871 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 19.76465606689453 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 19.77499771118164 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 19.747068405151367 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 19.758365631103516 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 19.771236419677734 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 19.77338981628418 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 19.779863357543945 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 19.817787170410156 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 19.765438079833984 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 19.7608585357666 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 19.730852127075195 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 19.767780303955078 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 19.753896713256836 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 19.792997360229492 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 19.78935432434082 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 19.77300453186035 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 19.79607391357422 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 19.71941375732422 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 19.7377872467041 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 19.730682373046875 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 19.778623580932617 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 19.781599044799805 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 19.743513107299805 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 19.812572479248047 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 19.743038177490234 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 19.782270431518555 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 19.744253158569336 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 19.78270721435547 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 19.767587661743164 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 19.7668514251709 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 19.75877571105957 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 19.755285263061523 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 19.76682472229004 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 19.764354705810547 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 19.780778884887695 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 19.754865646362305 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 19.787656784057617 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 19.74881935119629 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 19.725767135620117 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 19.749446868896484 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 19.74549674987793 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 19.77736473083496 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 19.758352279663086 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 19.764623641967773 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 19.82379913330078 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 19.735795974731445 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 19.75492286682129 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 19.745967864990234 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 19.778430938720703 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 19.788623809814453 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 19.757654190063477 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 19.80455780029297 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 19.750858306884766 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 19.745655059814453 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 19.73949432373047 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 19.773344039916992 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 19.77733612060547 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 19.785409927368164 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 19.775432586669922 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 19.764759063720703 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 19.698705673217773 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 19.747600555419922 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 19.778581619262695 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 19.77695083618164 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 19.696483612060547 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 19.76226806640625 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 19.74675750732422 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 19.703603744506836 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 19.516366958618164 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 16.630685806274414 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 16.627500534057617 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 16.631406784057617 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 16.618005752563477 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 16.602291107177734 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 16.597368240356445 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 16.586889266967773 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 16.627073287963867 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 16.634178161621094 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 16.626371383666992 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 16.664024353027344 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 16.6167049407959 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 16.62947654724121 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 16.61944007873535 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 16.628286361694336 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 16.630712509155273 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 16.630868911743164 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 16.627859115600586 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 16.59368133544922 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 16.611072540283203 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 16.61209487915039 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 16.628631591796875 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 16.604551315307617 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 16.59253692626953 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 16.63457679748535 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 16.592971801757812 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 16.61699867248535 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 16.61562728881836 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 16.631200790405273 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 16.635284423828125 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 16.63636016845703 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 16.622753143310547 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 16.606184005737305 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 16.616777420043945 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 16.62289047241211 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 16.630998611450195 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 16.630531311035156 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 16.62473487854004 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 16.642414093017578 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 16.615798950195312 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 16.627944946289062 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 16.62205696105957 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 16.63237762451172 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 16.625757217407227 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 16.62980842590332 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 16.62651824951172 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 16.60041618347168 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 16.609739303588867 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 16.62714385986328 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 16.634368896484375 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 16.56149673461914 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 16.626995086669922 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 16.63007164001465 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 16.609481811523438 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 16.630590438842773 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 16.6151123046875 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 16.630672454833984 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 16.6357421875 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 16.620988845825195 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 16.629135131835938 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 16.61407470703125 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 16.62395477294922 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 16.6292781829834 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 16.625030517578125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 16.62008285522461 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 16.625757217407227 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 16.62205696105957 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 16.610754013061523 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 16.63167381286621 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 16.621564865112305 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 16.630470275878906 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 16.62807273864746 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 16.631839752197266 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 16.60504722595215 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 16.61274528503418 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 16.615049362182617 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 16.625621795654297 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 16.63251304626465 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 16.617847442626953 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 16.634159088134766 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 16.627687454223633 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 16.62582778930664 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 16.632112503051758 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 16.627643585205078 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 16.62935447692871 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 16.635658264160156 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 16.63436508178711 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 16.62236976623535 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 16.623994827270508 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 16.62853240966797 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 16.62305450439453 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 16.63251304626465 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 16.63064193725586 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 16.623014450073242 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 16.63410758972168 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 16.624967575073242 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 16.633520126342773 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 16.63284683227539 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 16.62908363342285 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 16.626304626464844 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 16.6295108795166 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 16.618759155273438 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 16.62211036682129 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 16.625282287597656 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 16.619108200073242 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 16.629114151000977 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 16.63429069519043 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 16.630903244018555 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 16.622848510742188 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 16.618009567260742 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 16.624765396118164 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 16.61850929260254 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 16.637554168701172 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 16.614490509033203 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 16.635717391967773 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 16.635967254638672 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 16.622970581054688 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 16.621959686279297 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 16.61798095703125 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 16.629793167114258 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 16.631895065307617 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 16.620437622070312 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 16.638097763061523 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 16.61531639099121 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 16.6163387298584 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 16.614290237426758 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 16.631608963012695 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 16.617443084716797 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 16.625282287597656 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 16.6259708404541 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 16.621984481811523 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 16.62305450439453 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 16.623022079467773 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 16.63270378112793 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 16.625900268554688 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 16.629356384277344 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 16.626646041870117 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 16.610998153686523 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 16.61231231689453 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 16.612825393676758 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 16.63055419921875 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 16.62926483154297 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 16.621702194213867 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 16.64854621887207 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 16.615711212158203 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 16.61214256286621 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 16.601547241210938 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 16.629804611206055 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 16.629024505615234 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 16.626480102539062 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 16.643024444580078 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 16.618288040161133 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 16.61366844177246 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 16.614381790161133 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 16.629928588867188 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 16.631935119628906 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 16.63134765625 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 16.6223087310791 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 16.623445510864258 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 16.629274368286133 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 16.618263244628906 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 16.62987518310547 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 16.631284713745117 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 16.614221572875977 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 16.632417678833008 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 16.604145050048828 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 16.581981658935547 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 16.605316162109375 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 15.527266502380371 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 15.52575969696045 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 15.525885581970215 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 15.519804954528809 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 15.520051956176758 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 15.510549545288086 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 15.513763427734375 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 15.524959564208984 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 15.526588439941406 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 15.530157089233398 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 15.537994384765625 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 15.515793800354004 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 15.516578674316406 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 15.526836395263672 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 15.524785995483398 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 15.527861595153809 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 15.528583526611328 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 15.519311904907227 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 15.510608673095703 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 15.512909889221191 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 15.517599105834961 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 15.527044296264648 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 15.53763484954834 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 15.522446632385254 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 15.530767440795898 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 15.519268989562988 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 15.519981384277344 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 15.525640487670898 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 15.526825904846191 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 15.52847957611084 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 15.52184009552002 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 15.525592803955078 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 15.522631645202637 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 15.522273063659668 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 15.518617630004883 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 15.529257774353027 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 15.52783203125 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 15.527865409851074 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 15.520033836364746 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 15.51518726348877 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 15.5269775390625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 15.516911506652832 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 15.526784896850586 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 15.527114868164062 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 15.525848388671875 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 15.533723831176758 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 15.521225929260254 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 15.5265474319458 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 15.52258014678955 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 15.52814769744873 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 15.517051696777344 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 15.531012535095215 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 15.525267601013184 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 15.520463943481445 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 15.520441055297852 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 15.528613090515137 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 15.523926734924316 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 15.527436256408691 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 15.530046463012695 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 15.522758483886719 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 15.520271301269531 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 15.528528213500977 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 15.52187728881836 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 15.529410362243652 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 15.524386405944824 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 15.52705192565918 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 15.535486221313477 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 15.516996383666992 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 15.52095890045166 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 15.522165298461914 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 15.526308059692383 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 15.52802848815918 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 15.522198677062988 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 15.523053169250488 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 15.520000457763672 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 15.529088020324707 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 15.524829864501953 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 15.523239135742188 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 15.526406288146973 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 15.52641773223877 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 15.524615287780762 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 15.520218849182129 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 15.520456314086914 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 15.523841857910156 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 15.527091979980469 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 15.52742862701416 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 15.524578094482422 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 15.525833129882812 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 15.518879890441895 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 15.517047882080078 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 15.522509574890137 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 15.525985717773438 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 15.524112701416016 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 15.521469116210938 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 15.524282455444336 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 15.522784233093262 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 15.523042678833008 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 15.52314281463623 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 15.526055335998535 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 15.524823188781738 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 15.526769638061523 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 15.52293872833252 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 15.52143383026123 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 15.524726867675781 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 15.525568008422852 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 15.526529312133789 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 15.527762413024902 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 15.527403831481934 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 15.523435592651367 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 15.517821311950684 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 15.517677307128906 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 15.520981788635254 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 15.530257225036621 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 15.523201942443848 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 15.527095794677734 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 15.529080390930176 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 15.519153594970703 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 15.516112327575684 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 15.518842697143555 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 15.527036666870117 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 15.527580261230469 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 15.521921157836914 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 15.531364440917969 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 15.523993492126465 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 15.526573181152344 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 15.51697826385498 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 15.5274076461792 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 15.520277976989746 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 15.526213645935059 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 15.526506423950195 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 15.521610260009766 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 15.522984504699707 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 15.519335746765137 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 15.524919509887695 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 15.524666786193848 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 15.527114868164062 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 15.524785995483398 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 15.520648956298828 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 15.522273063659668 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 15.519352912902832 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 15.526267051696777 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 15.525666236877441 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 15.521954536437988 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 15.536116600036621 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 15.516816139221191 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 15.52022647857666 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 15.520026206970215 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 15.526347160339355 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 15.527788162231445 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 15.523849487304688 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 15.540205955505371 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 15.520940780639648 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 15.521547317504883 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 15.519020080566406 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 15.525577545166016 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 15.527107238769531 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 15.526836395263672 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 15.52531909942627 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 15.521078109741211 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 15.51115608215332 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 15.520895957946777 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 15.526680946350098 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 15.527192115783691 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 15.515379905700684 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 15.523483276367188 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 15.51502513885498 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 15.499074935913086 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 15.502264976501465 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 14.990837097167969 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 14.990540504455566 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 14.991348266601562 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 14.99056625366211 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 14.984272956848145 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 14.97730827331543 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 14.983526229858398 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 14.991548538208008 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 14.991205215454102 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 14.99117660522461 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 14.99532413482666 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 14.98586654663086 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 14.990368843078613 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 14.98536205291748 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 14.991151809692383 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 14.990293502807617 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 14.990604400634766 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 14.990225791931152 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 14.982900619506836 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 14.98953914642334 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 14.985352516174316 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 14.992059707641602 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 14.984663009643555 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 14.989876747131348 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 14.98706340789795 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 14.982101440429688 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 14.990533828735352 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 14.984945297241211 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 14.991423606872559 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 14.991273880004883 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 14.995030403137207 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 14.993203163146973 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 14.981433868408203 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 14.986988067626953 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 14.98453426361084 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 14.993127822875977 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 14.989947319030762 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 14.99177360534668 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 14.99257755279541 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 14.986644744873047 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 14.990168571472168 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 14.99202823638916 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 14.99117660522461 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 14.989897727966309 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 14.992249488830566 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 14.989439964294434 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 14.983701705932617 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 14.991827011108398 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 14.98825740814209 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 14.992284774780273 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 14.987946510314941 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 14.988710403442383 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 14.991769790649414 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 14.986170768737793 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 14.99288558959961 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 14.98873233795166 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 14.99137020111084 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 14.991540908813477 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 14.989126205444336 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 14.988832473754883 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 14.986227989196777 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 14.986717224121094 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 14.994129180908203 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 14.99143123626709 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 14.990082740783691 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 14.990691184997559 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 14.989368438720703 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 14.987141609191895 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 14.991813659667969 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 14.989750862121582 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 14.9913911819458 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 14.991252899169922 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 14.98904037475586 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 14.99138355255127 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 14.986334800720215 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 14.98686695098877 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 14.990644454956055 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 14.99190616607666 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 14.987128257751465 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 14.987189292907715 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 14.98904037475586 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 14.990200996398926 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 14.984638214111328 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 14.989339828491211 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 14.99145221710205 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 14.992742538452148 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 14.99244499206543 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 14.990397453308105 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 14.988017082214355 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 14.989421844482422 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 14.988703727722168 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 14.992795944213867 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 14.991019248962402 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 14.991124153137207 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 14.99026870727539 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 14.99178409576416 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 14.989433288574219 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 14.99083423614502 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 14.99130916595459 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 14.990540504455566 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 14.990341186523438 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 14.988525390625 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 14.98939323425293 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 14.989982604980469 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 14.988249778747559 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 14.99071979522705 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 14.990047454833984 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 14.990976333618164 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 14.990861892700195 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 14.988842964172363 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 14.991755485534668 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 14.98727035522461 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 14.993106842041016 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 14.991899490356445 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 14.991951942443848 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 14.992799758911133 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 14.98901081085205 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 14.98926830291748 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 14.985437393188477 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 14.989901542663574 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 14.990633010864258 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 14.981523513793945 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 14.997273445129395 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 14.987335205078125 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 14.990443229675293 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 14.98945426940918 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 14.990877151489258 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 14.986078262329102 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 14.990289688110352 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 14.991284370422363 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 14.98768138885498 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 14.986574172973633 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 14.988832473754883 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 14.99185562133789 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 14.990362167358398 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 14.990408897399902 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 14.990143775939941 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 14.984517097473145 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 14.98213005065918 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 14.987496376037598 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 14.991456031799316 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 14.990287780761719 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 14.98926067352295 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 14.996332168579102 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 14.986891746520996 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 14.984373092651367 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 14.983086585998535 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 14.990571975708008 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 14.9915771484375 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 14.988832473754883 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 14.996646881103516 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 14.985391616821289 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 14.984622955322266 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 14.988239288330078 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 14.990877151489258 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 14.99118423461914 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 14.991945266723633 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 14.98695182800293 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 14.988786697387695 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 14.983065605163574 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 14.986617088317871 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 14.991556167602539 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 14.991354942321777 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 14.987725257873535 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 14.993558883666992 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 14.981600761413574 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 14.960899353027344 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 14.973709106445312 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 14.653050422668457 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 14.65310287475586 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 14.652637481689453 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 14.648613929748535 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 14.649794578552246 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 14.648845672607422 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 14.641141891479492 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 14.651426315307617 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 14.651841163635254 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 14.65207576751709 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 14.66532039642334 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 14.649138450622559 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 14.649948120117188 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 14.651931762695312 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 14.651583671569824 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 14.652965545654297 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 14.652435302734375 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 14.649815559387207 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 14.646858215332031 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 14.64636516571045 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 14.651422500610352 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 14.652637481689453 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 14.655020713806152 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 14.651265144348145 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 14.653658866882324 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 14.649406433105469 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 14.645254135131836 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 14.654370307922363 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 14.65196704864502 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 14.652495384216309 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 14.652183532714844 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 14.652729034423828 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 14.651876449584961 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 14.65058708190918 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 14.650416374206543 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 14.652827262878418 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 14.653120040893555 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 14.653008460998535 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 14.654696464538574 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 14.649267196655273 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 14.65112590789795 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 14.646867752075195 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 14.65184497833252 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 14.652124404907227 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 14.652295112609863 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 14.655369758605957 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 14.65360164642334 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 14.649032592773438 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 14.652990341186523 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 14.65326976776123 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 14.648061752319336 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 14.655363082885742 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 14.652351379394531 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 14.651485443115234 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 14.650727272033691 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 14.650014877319336 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 14.651261329650879 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 14.652700424194336 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 14.652937889099121 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 14.652295112609863 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 14.651185035705566 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 14.654698371887207 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 14.651568412780762 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 14.65267276763916 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 14.651683807373047 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 14.653633117675781 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 14.655447006225586 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 14.649606704711914 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 14.64868450164795 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 14.650046348571777 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 14.652167320251465 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 14.65339183807373 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 14.65170955657959 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 14.650486946105957 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 14.649706840515137 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 14.65312385559082 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 14.653501510620117 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 14.649986267089844 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 14.651956558227539 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 14.654544830322266 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 14.65461540222168 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 14.650849342346191 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 14.654492378234863 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 14.652953147888184 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 14.652637481689453 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 14.652928352355957 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 14.652554512023926 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 14.652029991149902 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 14.649201393127441 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 14.648419380187988 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 14.651683807373047 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 14.652400970458984 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 14.652704238891602 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 14.652511596679688 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 14.651418685913086 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 14.652504920959473 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 14.651257514953613 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 14.652920722961426 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 14.652026176452637 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 14.652246475219727 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 14.653727531433105 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 14.652155876159668 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 14.651206016540527 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 14.650766372680664 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 14.651229858398438 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 14.652700424194336 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 14.653388977050781 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 14.653206825256348 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 14.64965534210205 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 14.64824390411377 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 14.64859676361084 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 14.650273323059082 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 14.653633117675781 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 14.652603149414062 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 14.653075218200684 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 14.654419898986816 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 14.648543357849121 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 14.651310920715332 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 14.650723457336426 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 14.6530122756958 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 14.654255867004395 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 14.650325775146484 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 14.659116744995117 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 14.650097846984863 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 14.650590896606445 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 14.64894962310791 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 14.652400970458984 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 14.65001106262207 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 14.652470588684082 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 14.65207576751709 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 14.65153694152832 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 14.652603149414062 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 14.650471687316895 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 14.652448654174805 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 14.652896881103516 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 14.653328895568848 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 14.65114974975586 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 14.649249076843262 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 14.649571418762207 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 14.647821426391602 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 14.652257919311523 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 14.652191162109375 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 14.651935577392578 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 14.657756805419922 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 14.649239540100098 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 14.649032592773438 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 14.647660255432129 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 14.652448654174805 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 14.653238296508789 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 14.653319358825684 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 14.65864086151123 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 14.651212692260742 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 14.65107250213623 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 14.649787902832031 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 14.652348518371582 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 14.653057098388672 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 14.653273582458496 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 14.653735160827637 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 14.65081787109375 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 14.651206016540527 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 14.649032592773438 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 14.652959823608398 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 14.652774810791016 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 14.650734901428223 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 14.649138450622559 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 14.648313522338867 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 14.649151802062988 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 14.638998985290527 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 14.474125862121582 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 14.473880767822266 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 14.473986625671387 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 14.470726013183594 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 14.47212028503418 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 14.468912124633789 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 14.470277786254883 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 14.474090576171875 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 14.474239349365234 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 14.474032402038574 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 14.475661277770996 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 14.469836235046387 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 14.473183631896973 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 14.470657348632812 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 14.474114418029785 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 14.474045753479004 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 14.473379135131836 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 14.473811149597168 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 14.469033241271973 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 14.472375869750977 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 14.471429824829102 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 14.473214149475098 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 14.473038673400879 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 14.476536750793457 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 14.471192359924316 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 14.471540451049805 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 14.4757719039917 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 14.46876049041748 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 14.47502613067627 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 14.474273681640625 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 14.473320960998535 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 14.47424602508545 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 14.468698501586914 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 14.471750259399414 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 14.469398498535156 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 14.474034309387207 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 14.473376274108887 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 14.47463321685791 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 14.475811958312988 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 14.472196578979492 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 14.473970413208008 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 14.473196029663086 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 14.473803520202637 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 14.474387168884277 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 14.474052429199219 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 14.474480628967285 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 14.470070838928223 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 14.474149703979492 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 14.470599174499512 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 14.474762916564941 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 14.470844268798828 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 14.473725318908691 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 14.474480628967285 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 14.473776817321777 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 14.476606369018555 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 14.473052024841309 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 14.474414825439453 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 14.473655700683594 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 14.47309398651123 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 14.47372817993164 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 14.4701566696167 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 14.4697265625 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 14.473038673400879 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 14.473245620727539 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 14.474336624145508 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 14.474052429199219 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 14.474528312683105 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 14.4721097946167 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 14.475750923156738 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 14.471940994262695 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 14.474076271057129 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 14.474525451660156 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 14.473465919494629 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 14.47424602508545 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 14.47232723236084 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 14.468738555908203 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 14.472319602966309 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 14.476040840148926 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 14.473320960998535 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 14.474605560302734 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 14.472789764404297 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 14.47312068939209 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 14.46982192993164 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 14.472921371459961 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 14.473873138427734 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 14.474493980407715 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 14.473962783813477 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 14.471519470214844 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 14.471861839294434 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 14.472437858581543 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 14.472437858581543 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 14.473831176757812 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 14.474122047424316 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 14.475212097167969 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 14.475109100341797 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 14.473687171936035 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 14.474163055419922 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 14.47329330444336 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 14.474370002746582 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 14.473779678344727 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 14.473344802856445 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 14.472713470458984 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 14.472030639648438 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 14.47249698638916 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 14.476399421691895 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 14.474032402038574 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 14.473845481872559 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 14.473546028137207 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 14.474038124084473 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 14.473593711853027 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 14.47170639038086 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 14.47188949584961 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 14.47465991973877 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 14.472996711730957 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 14.474287033081055 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 14.47352409362793 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 14.471781730651855 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 14.471429824829102 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 14.471975326538086 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 14.47314167022705 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 14.472789764404297 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 14.47376537322998 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 14.47677230834961 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 14.472813606262207 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 14.47355842590332 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 14.473438262939453 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 14.473459243774414 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 14.473007202148438 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 14.473976135253906 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 14.473572731018066 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 14.472458839416504 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 14.473417282104492 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 14.472779273986816 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 14.47415542602539 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 14.47348690032959 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 14.47374153137207 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 14.47409439086914 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 14.472220420837402 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 14.471636772155762 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 14.473645210266113 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 14.47445011138916 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 14.472879409790039 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 14.473031044006348 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 14.478340148925781 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 14.470595359802246 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 14.471495628356934 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 14.472465515136719 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 14.473666191101074 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 14.47406005859375 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 14.473261833190918 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 14.477134704589844 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 14.471450805664062 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 14.470736503601074 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 14.47077465057373 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 14.473597526550293 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 14.47425651550293 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 14.474135398864746 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 14.471212387084961 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 14.472078323364258 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 14.4684419631958 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 14.470643043518066 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 14.474004745483398 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 14.473869323730469 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 14.472817420959473 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 14.475798606872559 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 14.4662446975708 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 14.45545768737793 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 14.466903686523438 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 14.323567390441895 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 14.323575019836426 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 14.323260307312012 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 14.323502540588379 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 14.321624755859375 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 14.319371223449707 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 14.321955680847168 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 14.323089599609375 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 14.32313060760498 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 14.323636054992676 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 14.326126098632812 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 14.32292652130127 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 14.321566581726074 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 14.323363304138184 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 14.323765754699707 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 14.323868751525879 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 14.324285507202148 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 14.323256492614746 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 14.322361946105957 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 14.321457862854004 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 14.320354461669922 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 14.323834419250488 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 14.325385093688965 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 14.324158668518066 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 14.325087547302246 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 14.319087982177734 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 14.319190979003906 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 14.325786590576172 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 14.32313060760498 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 14.32345199584961 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 14.323834419250488 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 14.323343276977539 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 14.322359085083008 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 14.324336051940918 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 14.324728965759277 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 14.32351016998291 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 14.323598861694336 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 14.323506355285645 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 14.322361946105957 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 14.321037292480469 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 14.3219633102417 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 14.323266983032227 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 14.323578834533691 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 14.323375701904297 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 14.323431968688965 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 14.324018478393555 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 14.323246955871582 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 14.321723937988281 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 14.325692176818848 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 14.323857307434082 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 14.323225975036621 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 14.323653221130371 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 14.324448585510254 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 14.321173667907715 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 14.322761535644531 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 14.322830200195312 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 14.322467803955078 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 14.32398796081543 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 14.323390007019043 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 14.323683738708496 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 14.323575019836426 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 14.32494068145752 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 14.322823524475098 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 14.32407283782959 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 14.322769165039062 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 14.32359504699707 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 14.324230194091797 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 14.321586608886719 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 14.320706367492676 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 14.323158264160156 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 14.323369979858398 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 14.322745323181152 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 14.32269287109375 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 14.321316719055176 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 14.321320533752441 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 14.325592994689941 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 14.326082229614258 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 14.322990417480469 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 14.323089599609375 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 14.323969841003418 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 14.322980880737305 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 14.323290824890137 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 14.32399845123291 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 14.323984146118164 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 14.323738098144531 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 14.323383331298828 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 14.323806762695312 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 14.323004722595215 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 14.322044372558594 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 14.320539474487305 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 14.322611808776855 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 14.324080467224121 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 14.323915481567383 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 14.32351016998291 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 14.322307586669922 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 14.322864532470703 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 14.321823120117188 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 14.323567390441895 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 14.323413848876953 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 14.323240280151367 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 14.323991775512695 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 14.325206756591797 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 14.322867393493652 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 14.323403358459473 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 14.32162094116211 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 14.32354736328125 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 14.32494831085205 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 14.323848724365234 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 14.32252311706543 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 14.320996284484863 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 14.322591781616211 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 14.320637702941895 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 14.324670791625977 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 14.323946952819824 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 14.324288368225098 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 14.324134826660156 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 14.322406768798828 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 14.322236061096191 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 14.319344520568848 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 14.323943138122559 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 14.324469566345215 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 14.323991775512695 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 14.325818061828613 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 14.3215970993042 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 14.322318077087402 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 14.32121467590332 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 14.32351016998291 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 14.323274612426758 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 14.323718070983887 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 14.323266983032227 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 14.322334289550781 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 14.322324752807617 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 14.321986198425293 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 14.323328971862793 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 14.323506355285645 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 14.323834419250488 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 14.323759078979492 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 14.321808815002441 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 14.320747375488281 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 14.32025146484375 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 14.323492050170898 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 14.323724746704102 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 14.32319450378418 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 14.324934005737305 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 14.322495460510254 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 14.322290420532227 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 14.320690155029297 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 14.323857307434082 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 14.324175834655762 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 14.322225570678711 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 14.323861122131348 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 14.32261848449707 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 14.321304321289062 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 14.321627616882324 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 14.32410717010498 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 14.323912620544434 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 14.323734283447266 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 14.323738098144531 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 14.32270336151123 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 14.323240280151367 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 14.321778297424316 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 14.323397636413574 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 14.323780059814453 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 14.323363304138184 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 14.320903778076172 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 14.321115493774414 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 14.32013988494873 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 14.315937042236328 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 14.210070610046387 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 14.209817886352539 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 14.210013389587402 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 14.20782470703125 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 14.20869255065918 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 14.208407402038574 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 14.204289436340332 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 14.210173606872559 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 14.209820747375488 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 14.210186004638672 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 14.209634780883789 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 14.207513809204102 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 14.209488868713379 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 14.211172103881836 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 14.210318565368652 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 14.210288047790527 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 14.210224151611328 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 14.210143089294434 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 14.207533836364746 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 14.210433959960938 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 14.207694053649902 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 14.209634780883789 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 14.209956169128418 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 14.20884895324707 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 14.209956169128418 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 14.209936141967773 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 14.209895133972168 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 14.208374977111816 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 14.210386276245117 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 14.209990501403809 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 14.210305213928223 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 14.209733009338379 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 14.205870628356934 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 14.209086418151855 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 14.207733154296875 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 14.210285186767578 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 14.209909439086914 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 14.209620475769043 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 14.210163116455078 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 14.210596084594727 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 14.210392951965332 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 14.209607124328613 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 14.209592819213867 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 14.210108757019043 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 14.209734916687012 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 14.209807395935059 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 14.20782470703125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 14.208428382873535 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 14.207608222961426 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 14.210196495056152 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 14.209756851196289 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 14.210002899169922 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 14.210054397583008 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 14.209653854370117 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 14.209966659545898 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 14.210063934326172 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 14.210277557373047 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 14.210021018981934 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 14.209539413452148 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 14.209627151489258 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 14.207574844360352 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 14.207557678222656 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 14.209420204162598 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 14.209962844848633 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 14.210413932800293 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 14.209979057312012 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 14.210094451904297 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 14.209086418151855 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 14.211463928222656 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 14.209186553955078 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 14.210508346557617 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 14.210901260375977 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 14.209580421447754 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 14.209756851196289 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 14.209742546081543 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 14.206151962280273 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 14.207844734191895 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 14.210951805114746 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 14.209417343139648 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 14.210444450378418 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 14.209113121032715 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 14.209738731384277 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 14.205329895019531 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 14.208906173706055 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 14.210074424743652 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 14.210681915283203 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 14.210078239440918 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 14.20850944519043 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 14.20874309539795 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 14.20895004272461 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 14.209528923034668 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 14.210143089294434 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 14.210261344909668 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 14.210189819335938 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 14.21151065826416 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 14.20934009552002 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 14.210285186767578 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 14.210942268371582 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 14.210386276245117 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 14.210204124450684 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 14.210196495056152 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 14.208658218383789 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 14.208414077758789 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 14.209803581237793 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 14.210261344909668 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 14.210121154785156 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 14.209803581237793 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 14.210000991821289 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 14.210769653320312 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 14.20941162109375 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 14.208344459533691 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 14.210521697998047 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 14.210270881652832 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 14.210224151611328 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 14.210094451904297 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 14.210711479187012 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 14.20964527130127 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 14.210082054138184 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 14.209803581237793 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 14.209776878356934 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 14.209749221801758 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 14.209972381591797 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 14.212307929992676 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 14.209397315979004 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 14.210765838623047 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 14.210244178771973 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 14.210196495056152 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 14.210189819335938 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 14.210021018981934 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 14.209884643554688 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 14.209708213806152 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 14.210074424743652 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 14.208854675292969 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 14.209681510925293 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 14.210088729858398 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 14.209607124328613 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 14.210925102233887 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 14.208486557006836 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 14.209102630615234 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 14.210772514343262 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 14.210139274597168 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 14.210386276245117 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 14.209562301635742 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 14.212565422058105 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 14.208407402038574 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 14.207686424255371 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 14.208027839660645 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 14.210000991821289 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 14.209959983825684 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 14.208939552307129 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 14.212982177734375 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 14.208597183227539 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 14.208991050720215 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 14.2079439163208 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 14.209936141967773 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 14.209921836853027 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 14.210067749023438 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 14.208964347839355 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 14.208139419555664 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 14.205484390258789 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 14.208597183227539 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 14.209972381591797 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 14.210151672363281 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 14.210021018981934 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 14.211352348327637 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 14.206429481506348 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 14.198408126831055 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 14.2058744430542 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 14.116667747497559 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 14.116345405578613 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 14.116018295288086 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 14.115629196166992 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 14.113899230957031 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 14.11443042755127 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 14.115252494812012 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 14.117109298706055 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 14.116459846496582 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 14.116253852844238 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 14.118896484375 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 14.115571975708008 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 14.1143159866333 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 14.114941596984863 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 14.116368293762207 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 14.115991592407227 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 14.116321563720703 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 14.115864753723145 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 14.115264892578125 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 14.113380432128906 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 14.117142677307129 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 14.116253852844238 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 14.116332054138184 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 14.116167068481445 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 14.117324829101562 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 14.113872528076172 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 14.117162704467773 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 14.117681503295898 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 14.11626148223877 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 14.116644859313965 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 14.116921424865723 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 14.116362571716309 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 14.116288185119629 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 14.117130279541016 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 14.116022109985352 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 14.116446495056152 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 14.11616039276123 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 14.116341590881348 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 14.11717700958252 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 14.11398696899414 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 14.115489959716797 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 14.115516662597656 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 14.115817070007324 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 14.116591453552246 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 14.116167068481445 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 14.115924835205078 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 14.116096496582031 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 14.115127563476562 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 14.117142677307129 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 14.116409301757812 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 14.115991592407227 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 14.116442680358887 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 14.115809440612793 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 14.115665435791016 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 14.117294311523438 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 14.115315437316895 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 14.116544723510742 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 14.116241455078125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 14.11668872833252 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 14.116719245910645 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 14.11572551727295 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 14.115487098693848 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 14.115496635437012 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 14.116782188415527 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 14.116779327392578 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 14.116413116455078 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 14.117594718933105 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 14.115443229675293 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 14.114214897155762 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 14.116477012634277 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 14.115756034851074 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 14.11611270904541 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 14.1162748336792 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 14.114713668823242 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 14.114367485046387 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 14.116948127746582 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 14.116907119750977 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 14.116083145141602 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 14.116314888000488 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 14.116776466369629 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 14.117278099060059 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 14.115629196166992 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 14.115608215332031 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 14.116507530212402 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 14.116278648376465 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 14.116130828857422 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 14.11661148071289 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 14.116530418395996 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 14.114729881286621 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 14.113717079162598 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 14.115675926208496 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 14.116705894470215 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 14.116423606872559 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 14.116409301757812 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 14.116678237915039 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 14.116530418395996 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 14.116089820861816 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 14.115187644958496 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 14.116914749145508 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 14.116241455078125 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 14.117074966430664 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 14.116409301757812 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 14.116591453552246 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 14.115944862365723 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 14.116366386413574 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 14.116378784179688 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 14.11662483215332 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 14.11662483215332 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 14.115484237670898 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 14.115432739257812 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 14.117193222045898 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 14.114645957946777 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 14.117055892944336 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 14.116453170776367 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 14.116667747497559 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 14.117025375366211 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 14.116214752197266 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 14.114763259887695 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 14.114511489868164 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 14.116618156433105 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 14.11662483215332 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 14.11611270904541 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 14.11746597290039 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 14.116005897521973 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 14.115066528320312 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 14.114033699035645 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 14.116301536560059 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 14.116167068481445 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 14.11662483215332 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 14.116329193115234 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 14.115279197692871 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 14.115594863891602 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 14.115762710571289 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 14.116564750671387 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 14.116362571716309 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 14.11629867553711 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 14.11622142791748 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 14.11539363861084 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 14.114397048950195 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 14.114080429077148 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 14.116671562194824 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 14.116708755493164 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 14.115134239196777 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 14.11710262298584 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 14.115215301513672 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 14.114949226379395 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 14.114665985107422 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 14.11671257019043 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 14.116499900817871 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 14.11575984954834 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 14.11655044555664 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 14.115508079528809 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 14.114269256591797 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 14.115043640136719 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 14.116564750671387 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 14.116486549377441 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 14.116409301757812 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 14.11712646484375 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 14.116598129272461 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 14.115911483764648 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 14.115311622619629 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 14.116382598876953 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 14.116332054138184 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 14.116028785705566 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 14.115608215332031 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 14.115339279174805 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 14.115903854370117 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 14.111879348754883 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 14.040955543518066 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 14.041177749633789 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 14.040969848632812 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 14.039824485778809 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 14.041268348693848 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 14.039895057678223 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 14.036086082458496 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 14.04057788848877 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 14.041204452514648 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 14.041589736938477 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 14.04111385345459 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 14.041337966918945 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 14.041823387145996 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 14.041507720947266 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 14.041056632995605 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 14.040664672851562 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 14.041276931762695 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 14.041937828063965 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 14.040312767028809 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 14.042027473449707 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 14.03901481628418 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 14.041186332702637 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 14.041130065917969 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 14.041241645812988 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 14.041655540466309 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 14.041796684265137 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 14.039620399475098 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 14.039138793945312 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 14.041532516479492 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 14.041394233703613 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 14.041173934936523 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 14.041655540466309 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 14.040009498596191 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 14.03970718383789 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 14.03852653503418 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 14.041060447692871 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 14.040872573852539 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 14.041180610656738 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 14.041053771972656 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 14.041695594787598 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 14.041876792907715 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 14.040456771850586 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 14.040902137756348 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 14.040788650512695 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 14.041214942932129 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 14.041328430175781 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 14.039885520935059 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 14.040413856506348 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 14.039628028869629 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 14.04092025756836 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 14.04082202911377 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 14.040885925292969 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 14.0424222946167 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 14.040487289428711 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 14.041351318359375 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 14.042760848999023 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 14.04055118560791 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 14.040617942810059 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 14.041394233703613 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 14.040390968322754 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 14.039915084838867 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 14.039895057678223 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 14.041280746459961 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 14.041099548339844 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 14.04111385345459 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 14.04130744934082 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 14.0408935546875 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 14.040149688720703 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 14.042546272277832 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 14.040531158447266 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 14.041220664978027 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 14.04086685180664 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 14.041067123413086 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 14.042171478271484 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 14.041046142578125 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 14.039377212524414 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 14.039694786071777 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 14.041715621948242 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 14.041110038757324 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 14.040939331054688 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 14.040199279785156 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 14.041348457336426 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 14.039457321166992 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 14.040732383728027 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 14.040641784667969 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 14.041231155395508 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 14.04112720489502 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 14.04029369354248 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 14.040668487548828 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 14.038566589355469 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 14.040668487548828 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 14.041133880615234 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 14.04114055633545 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 14.040955543518066 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 14.041160583496094 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 14.040905952453613 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 14.040430068969727 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 14.040695190429688 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 14.04114055633545 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 14.041190147399902 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 14.041398048400879 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 14.040925979614258 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 14.040009498596191 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 14.041043281555176 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 14.0401029586792 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 14.041783332824707 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 14.041207313537598 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 14.040986061096191 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 14.04137134552002 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 14.039885520935059 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 14.039091110229492 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 14.041876792907715 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 14.041507720947266 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 14.041260719299316 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 14.041295051574707 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 14.041060447692871 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 14.040464401245117 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 14.04057788848877 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 14.040889739990234 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 14.041163444519043 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 14.040939331054688 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 14.039875030517578 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 14.041903495788574 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 14.040705680847168 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 14.041827201843262 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 14.041605949401855 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 14.041247367858887 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 14.040224075317383 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 14.040955543518066 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 14.041003227233887 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 14.040969848632812 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 14.040928840637207 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 14.041729927062988 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 14.040939331054688 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 14.041254043579102 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 14.040836334228516 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 14.04163932800293 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 14.040413856506348 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 14.040732383728027 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 14.040812492370605 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 14.041019439697266 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 14.041120529174805 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 14.040949821472168 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 14.04316234588623 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 14.039700508117676 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 14.040339469909668 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 14.041314125061035 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 14.040885925292969 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 14.041328430175781 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 14.041186332702637 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 14.042004585266113 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 14.040905952453613 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 14.040989875793457 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 14.040413856506348 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 14.041153907775879 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 14.041133880615234 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 14.040943145751953 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 14.040437698364258 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 14.040156364440918 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 14.038251876831055 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 14.040122985839844 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 14.041180610656738 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 14.04114055633545 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 14.04112720489502 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 14.041237831115723 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 14.0374116897583 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 14.032137870788574 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 14.037672996520996 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 13.98147201538086 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 13.981685638427734 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 13.981865882873535 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 13.980772018432617 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 13.980805397033691 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 13.980755805969238 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 13.982878684997559 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 13.982378959655762 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 13.981549263000488 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 13.981325149536133 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 13.98318862915039 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 13.98005199432373 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 13.982245445251465 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 13.980545043945312 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 13.981738090515137 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 13.981678009033203 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 13.98162841796875 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 13.980958938598633 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 13.980619430541992 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 13.980338096618652 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 13.981532096862793 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 13.981385231018066 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 13.981842994689941 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 13.981779098510742 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 13.981085777282715 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 13.980165481567383 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 13.981557846069336 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 13.981377601623535 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 13.981411933898926 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 13.981335639953613 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 13.98121166229248 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 13.980831146240234 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 13.981441497802734 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 13.982308387756348 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 13.982288360595703 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 13.981271743774414 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 13.981745719909668 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 13.98161792755127 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 13.982012748718262 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 13.980598449707031 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 13.980681419372559 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 13.980985641479492 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 13.981839179992676 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 13.981592178344727 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 13.981338500976562 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 13.981489181518555 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 13.9812593460083 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 13.97991943359375 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 13.981435775756836 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 13.981515884399414 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 13.98145866394043 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 13.981805801391602 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 13.981696128845215 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 13.980772018432617 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 13.980998992919922 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 13.980857849121094 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 13.981411933898926 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 13.981768608093262 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 13.981518745422363 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 13.981888771057129 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 13.98082447052002 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 13.980611801147461 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 13.981812477111816 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 13.981715202331543 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 13.981492042541504 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 13.981278419494629 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 13.982912063598633 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 13.981025695800781 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 13.98028564453125 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 13.981549263000488 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 13.981644630432129 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 13.981557846069336 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 13.981497764587402 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 13.979851722717285 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 13.979965209960938 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 13.981572151184082 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 13.981658935546875 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 13.980828285217285 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 13.981025695800781 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 13.981524467468262 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 13.98216438293457 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 13.980588912963867 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 13.980232238769531 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 13.981598854064941 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 13.981437683105469 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 13.981632232666016 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 13.981172561645508 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 13.9812593460083 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 13.980225563049316 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 13.97988224029541 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 13.981058120727539 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 13.981371879577637 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 13.981377601623535 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 13.981925964355469 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 13.981385231018066 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 13.981752395629883 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 13.98163890838623 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 13.981745719909668 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 13.981598854064941 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 13.981669425964355 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 13.981325149536133 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 13.981888771057129 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 13.982038497924805 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 13.981501579284668 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 13.981605529785156 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 13.981325149536133 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 13.981321334838867 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 13.981435775756836 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 13.98084545135498 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 13.980478286743164 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 13.982695579528809 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 13.97989273071289 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 13.981818199157715 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 13.98128604888916 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 13.98147201538086 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 13.982381820678711 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 13.981578826904297 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 13.982341766357422 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 13.981008529663086 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 13.981505393981934 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 13.982065200805664 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 13.981858253479004 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 13.981779098510742 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 13.980831146240234 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 13.980658531188965 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 13.980244636535645 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 13.981108665466309 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 13.980831146240234 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 13.98143196105957 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 13.981115341186523 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 13.981338500976562 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 13.981058120727539 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 13.980371475219727 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 13.981557846069336 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 13.981362342834473 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 13.981165885925293 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 13.981868743896484 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 13.980831146240234 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 13.979632377624512 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 13.97991943359375 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 13.981411933898926 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 13.981609344482422 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 13.981489181518555 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 13.982388496398926 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 13.98128890991211 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 13.978678703308105 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 13.97984504699707 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 13.98163890838623 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 13.981524467468262 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 13.981549263000488 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 13.982452392578125 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 13.980815887451172 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 13.980025291442871 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 13.980865478515625 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 13.981142044067383 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 13.981678009033203 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 13.981532096862793 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 13.981184959411621 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 13.981745719909668 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 13.97927188873291 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 13.980978012084961 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 13.981338500976562 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 13.98163890838623 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 13.981232643127441 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 13.981218338012695 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 13.981087684631348 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 13.980695724487305 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 13.978025436401367 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 13.942641258239746 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 13.942085266113281 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 13.94222640991211 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 13.942159652709961 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 13.941866874694824 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 13.941654205322266 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 13.939201354980469 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 13.941761016845703 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 13.942142486572266 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 13.941999435424805 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 13.941877365112305 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 13.942658424377441 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 13.940300941467285 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 13.941879272460938 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 13.942039489746094 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 13.942425727844238 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 13.94218921661377 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 13.942784309387207 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 13.941184997558594 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 13.942242622375488 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 13.94168758392334 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 13.942627906799316 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 13.942285537719727 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 13.942365646362305 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 13.941511154174805 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 13.94180679321289 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 13.941737174987793 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 13.942724227905273 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 13.942435264587402 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 13.942012786865234 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 13.942298889160156 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 13.942110061645508 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 13.940775871276855 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 13.940287590026855 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 13.941368103027344 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 13.942411422729492 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 13.942205429077148 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 13.9425048828125 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 13.942388534545898 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 13.942012786865234 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 13.942744255065918 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 13.941390991210938 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 13.94233226776123 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 13.942378044128418 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 13.941600799560547 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 13.942338943481445 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 13.94155502319336 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 13.942748069763184 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 13.94218635559082 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 13.942342758178711 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 13.942371368408203 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 13.942485809326172 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 13.942910194396973 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 13.942238807678223 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 13.942166328430176 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 13.941762924194336 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 13.942487716674805 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 13.9425048828125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 13.941863059997559 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 13.942471504211426 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 13.940640449523926 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 13.940673828125 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 13.941953659057617 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 13.942212104797363 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 13.94238567352295 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 13.941766738891602 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 13.94155502319336 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 13.941793441772461 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 13.942201614379883 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 13.941401481628418 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 13.942253112792969 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 13.942193031311035 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 13.941829681396484 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 13.943177223205566 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 13.941873550415039 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 13.941222190856934 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 13.94214916229248 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 13.942219734191895 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 13.941896438598633 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 13.941912651062012 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 13.941849708557129 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 13.942527770996094 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 13.940443992614746 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 13.941956520080566 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 13.942046165466309 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 13.941963195800781 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 13.942166328430176 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 13.94179630279541 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 13.94148063659668 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 13.940114974975586 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 13.942670822143555 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 13.942421913146973 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 13.941987037658691 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 13.942126274108887 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 13.942265510559082 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 13.942012786865234 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 13.94183349609375 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 13.941889762878418 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 13.942399024963379 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 13.941803932189941 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 13.94197940826416 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 13.942797660827637 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 13.94114875793457 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 13.941588401794434 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 13.942577362060547 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 13.942621231079102 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 13.941886901855469 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 13.942312240600586 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 13.94254207611084 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 13.94200611114502 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 13.940381050109863 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 13.942976951599121 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 13.9425048828125 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 13.941813468933105 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 13.942092895507812 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 13.94167423248291 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 13.94162368774414 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 13.941743850708008 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 13.941041946411133 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 13.942055702209473 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 13.942028999328613 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 13.941935539245605 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 13.943302154541016 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 13.942155838012695 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 13.942764282226562 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 13.942365646362305 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 13.942028999328613 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 13.942245483398438 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 13.94200611114502 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 13.941995620727539 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 13.941916465759277 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 13.94196605682373 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 13.942342758178711 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 13.942046165466309 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 13.942245483398438 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 13.942269325256348 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 13.941627502441406 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 13.941993713378906 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 13.941710472106934 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 13.94234848022461 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 13.941999435424805 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 13.9420166015625 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 13.941972732543945 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 13.943582534790039 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 13.940696716308594 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 13.940876007080078 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 13.941856384277344 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 13.941893577575684 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 13.94184398651123 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 13.941999435424805 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 13.943323135375977 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 13.941514015197754 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 13.942272186279297 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 13.94183349609375 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 13.942452430725098 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 13.94183349609375 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 13.941960334777832 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 13.94128131866455 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 13.9414644241333 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 13.942020416259766 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 13.941062927246094 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 13.942253112792969 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 13.94196605682373 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 13.942032814025879 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 13.942361831665039 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 13.939929008483887 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 13.934708595275879 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 13.940805435180664 |
| } |
| ] |
| } |
| ] |
| } |
