OUTPUT of run 1 of at most 100: ``` WEIGHT-ONLY ANALYSIS: no inputs, activations, or forward passes. Keys and shapes: [('W_fc.0', (1536, 384)), ('W_fc.1', (1536, 384)), ('W_fc.2', (1536, 384)), ('W_fc.3', (1536, 384)), ('W_k.0', (384, 384)), ('W_k.1', (384, 384)), ('W_k.2', (384, 384)), ('W_k.3', (384, 384)), ('W_o.0', (384, 384)), ('W_o.1', (384, 384)), ('W_o.2', (384, 384)), ('W_o.3', (384, 384)), ('W_proj.0', (384, 1536)), ('W_proj.1', (384, 1536)), ('W_proj.2', (384, 1536)), ('W_proj.3', (384, 1536)), ('W_q.0', (384, 384)), ('W_q.1', (384, 384)), ('W_q.2', (384, 384)), ('W_q.3', (384, 384)), ('W_v.0', (384, 384)), ('W_v.1', (384, 384)), ('W_v.2', (384, 384)), ('W_v.3', (384, 384)), ('ln1_b.0', (384,)), ('ln1_b.1', (384,)), ('ln1_b.2', (384,)), ('ln1_b.3', (384,)), ('ln1_w.0', (384,)), ('ln1_w.1', (384,)), ('ln1_w.2', (384,)), ('ln1_w.3', (384,)), ('ln2_b.0', (384,)), ('ln2_b.1', (384,)), ('ln2_b.2', (384,)), ('ln2_b.3', (384,)), ('ln2_w.0', (384,)), ('ln2_w.1', (384,)), ('ln2_w.2', (384,)), ('ln2_w.3', (384,))] LAYER 0 ln1_w min/mean/max/std/norm: [-0.001 0.2403 1.0341 0.2436 6.7051] ln1_b min/mean/max/std/norm: [-0.0959 0.0008 0.091 0.0287 0.5635] ln2_w min/mean/max/std/norm: [ 0.0044 0.8006 1.5072 0.3295 16.9648] ln2_b min/mean/max/std/norm: [-0.4986 0.0022 0.3331 0.0683 1.3392] W_q norm/topSV/r50/r90: 32.2912 8.2519 9 30 W_k norm/topSV/r50/r90: 30.3909 7.6464 10 35 W_v norm/topSV/r50/r90: 14.836 4.339 14 97 W_o norm/topSV/r50/r90: 16.3713 7.1033 7 49 W_fc norm/topSV/r50/r90: 46.2485 18.1759 16 79 W_proj norm/topSV/r50/r90: 39.3073 9.8257 27 99 LAYER 1 ln1_w min/mean/max/std/norm: [0.1361 0.4575 1.1239 0.1022 9.1868] ln1_b min/mean/max/std/norm: [-1.4713 -0.005 1.1544 0.1672 3.2771] ln2_w min/mean/max/std/norm: [-0.0019 1.0429 1.4856 0.2003 20.8093] ln2_b min/mean/max/std/norm: [-0.3597 0.0005 0.3686 0.1364 2.6732] W_q norm/topSV/r50/r90: 32.3126 9.9032 15 54 W_k norm/topSV/r50/r90: 29.2506 13.3072 8 53 W_v norm/topSV/r50/r90: 27.3529 5.2845 22 76 W_o norm/topSV/r50/r90: 25.9233 4.9657 42 113 W_fc norm/topSV/r50/r90: 79.1264 21.4881 24 98 W_proj norm/topSV/r50/r90: 75.7102 8.1935 105 279 LAYER 2 ln1_w min/mean/max/std/norm: [ 0.2078 0.7052 1.0144 0.0749 13.8969] ln1_b min/mean/max/std/norm: [-1.3819 -0.001 1.0527 0.1525 2.9893] ln2_w min/mean/max/std/norm: [ 0.0011 1.3123 1.4655 0.1255 25.8329] ln2_b min/mean/max/std/norm: [-0.3761 0.005 0.3886 0.1537 3.0128] W_q norm/topSV/r50/r90: 39.3455 5.7962 53 169 W_k norm/topSV/r50/r90: 37.7688 10.6627 46 164 W_v norm/topSV/r50/r90: 33.3048 3.7062 62 179 W_o norm/topSV/r50/r90: 32.8397 4.1025 67 189 W_fc norm/topSV/r50/r90: 78.9667 15.3634 82 256 W_proj norm/topSV/r50/r90: 76.831 9.4036 87 266 LAYER 3 ln1_w min/mean/max/std/norm: [ 0.2944 0.7546 0.8701 0.0542 14.8247] ln1_b min/mean/max/std/norm: [-0.6069 0.0121 0.6502 0.0849 1.6801] ln2_w min/mean/max/std/norm: [ 0.4364 1.2778 1.4306 0.0808 25.0891] ln2_b min/mean/max/std/norm: [-0.6728 0.0043 0.3068 0.1276 2.5016] W_q norm/topSV/r50/r90: 32.9482 4.5598 56 176 W_k norm/topSV/r50/r90: 32.2094 6.0903 61 184 W_v norm/topSV/r50/r90: 39.7941 5.0598 52 170 W_o norm/topSV/r50/r90: 40.4549 9.5668 41 157 W_fc norm/topSV/r50/r90: 77.7757 14.1624 97 277 W_proj norm/topSV/r50/r90: 82.395 26.3094 9 24 HEAD PRODUCTS QK=Q.T@K/sqrt(64); OV=O@V. sym=/||A||F^2; trN=trace(A)/nuclear_norm(A). Columns: head; QK(norm,topSV,r50,r90,sym,trN); OV(norm,topSV,r50,r90,sym,trN); leading nonzero eigenvalues by magnitude. L0H0 [np.float64(0.381), np.float64(0.3137), 1, 12, np.float64(0.0561), np.float64(-0.1312)] [np.float64(12.9041), np.float64(10.2524), 1, 4, np.float64(0.0078), np.float64(-0.0051)] QKev [-0.068+0.j -0.041+0.008j -0.041-0.008j -0.016+0.j -0.003+0.01j ] OVev [ 1.173+0.j -0.432+0.575j -0.432-0.575j -0.38 +0.j 0.274+0.244j] L0H1 [np.float64(0.2297), np.float64(0.1371), 2, 27, np.float64(0.1165), np.float64(-0.1169)] [np.float64(10.1485), np.float64(7.662), 1, 4, np.float64(0.0184), np.float64(-0.0264)] QKev [-0.07 +0.j -0.034+0.j -0.009+0.01j -0.009-0.01j 0.012+0.j ] OVev [ 0.981+0.j -0.924+0.j -0.229+0.315j -0.229-0.315j 0.388+0.j ] L0H2 [np.float64(22.8044), np.float64(7.4848), 6, 14, np.float64(-0.0304), np.float64(0.0382)] [np.float64(13.223), np.float64(4.0009), 7, 16, np.float64(0.0098), np.float64(-0.069)] QKev [-0.867+5.787j -0.867-5.787j 1.92 +4.997j 1.92 -4.997j 4.415+2.676j] OVev [-0.698+0.122j -0.698-0.122j -0.473+0.j -0.32 +0.318j -0.32 -0.318j] L0H3 [np.float64(1.4189), np.float64(0.821), 2, 9, np.float64(0.1641), np.float64(-0.169)] [np.float64(7.3079), np.float64(4.305), 2, 5, np.float64(0.0159), np.float64(-0.0816)] QKev [ 0.304+0.j -0.211+0.064j -0.211-0.064j 0.194+0.048j 0.194-0.048j] OVev [-0.691+0.j -0.332+0.027j -0.332-0.027j 0.262+0.j -0.204+0.j ] L0H4 [np.float64(0.8259), np.float64(0.7527), 1, 2, np.float64(0.1641), np.float64(-0.2512)] [np.float64(7.3117), np.float64(4.8786), 2, 6, np.float64(0.0082), np.float64(-0.0381)] QKev [-0.309+0.j -0.103+0.j -0.07 +0.j -0.021+0.j -0.014+0.009j] OVev [-0.467+0.j 0.36 +0.j -0.264+0.j -0.221+0.j 0.107+0.141j] L0H5 [np.float64(0.3223), np.float64(0.2737), 1, 15, np.float64(0.0624), np.float64(-0.0847)] [np.float64(13.0384), np.float64(10.3142), 1, 4, np.float64(0.0039), np.float64(-0.0023)] QKev [-0.079+0.j -0.016+0.j -0.012+0.j 0.006+0.008j 0.006-0.008j] OVev [ 0.827+0.j -0.41 +0.498j -0.41 -0.498j 0.423+0.j 0.104+0.189j] L1H0 [np.float64(2.9582), np.float64(1.5768), 2, 10, np.float64(0.0806), np.float64(-0.2659)] [np.float64(21.7066), np.float64(8.0989), 5, 15, np.float64(0.0885), np.float64(-0.2035)] QKev [-0.383+0.j -0.356+0.105j -0.356-0.105j -0.24 +0.059j -0.24 -0.059j] OVev [-2.776+0.j -2.372+0.124j -2.372-0.124j -2.193+0.345j -2.193-0.345j] L1H1 [np.float64(4.4391), np.float64(2.4561), 2, 11, np.float64(0.1022), np.float64(-0.3042)] [np.float64(22.226), np.float64(6.5833), 9, 22, np.float64(0.0682), np.float64(-0.2227)] QKev [-0.684+0.j -0.515+0.096j -0.515-0.096j -0.478+0.019j -0.478-0.019j] OVev [-2.052+0.j -1.972+0.206j -1.972-0.206j -1.509-0.28j -1.509+0.28j ] L1H2 [np.float64(5.2226), np.float64(2.9227), 2, 11, np.float64(0.1095), np.float64(-0.3463)] [np.float64(19.5165), np.float64(6.1346), 8, 19, np.float64(0.0494), np.float64(-0.2067)] QKev [-0.769+0.j -0.621+0.287j -0.621-0.287j -0.656+0.j -0.608+0.j ] OVev [-1.708+0.229j -1.708-0.229j -1.42 +0.326j -1.42 -0.326j -1.325+0.j ] L1H3 [np.float64(4.6046), np.float64(2.6384), 2, 11, np.float64(0.0956), np.float64(-0.2981)] [np.float64(21.4133), np.float64(7.7675), 6, 15, np.float64(0.0452), np.float64(-0.1952)] QKev [-0.822+0.j -0.576+0.j -0.488+0.012j -0.488-0.012j -0.405+0.085j] OVev [-1.88 +0.j -1.706+0.128j -1.706-0.128j -1.526+0.j -1.438+0.j ] L1H4 [np.float64(4.6727), np.float64(2.8698), 2, 11, np.float64(0.0851), np.float64(-0.3096)] [np.float64(22.0658), np.float64(6.4984), 8, 18, np.float64(0.0675), np.float64(-0.228)] QKev [-0.566+0.06j -0.566-0.06j -0.55 +0.j -0.451+0.174j -0.451-0.174j] OVev [-2.065+0.j -2.045+0.265j -2.045-0.265j -1.945+0.j -1.467+0.482j] L1H5 [np.float64(4.5264), np.float64(2.6178), 2, 11, np.float64(0.0644), np.float64(-0.2713)] [np.float64(22.4638), np.float64(6.2081), 9, 20, np.float64(0.0683), np.float64(-0.2384)] QKev [-0.52 +0.114j -0.52 -0.114j -0.524+0.j -0.377+0.287j -0.377-0.287j] OVev [-2.115+0.j -2.009+0.j -1.932+0.409j -1.932-0.409j -1.638+0.108j] L2H0 [np.float64(4.8218), np.float64(1.4573), 11, 33, np.float64(-0.0144), np.float64(0.0075)] [np.float64(22.8142), np.float64(4.8981), 15, 41, np.float64(0.1187), np.float64(-0.3236)] QKev [0.111+0.4j 0.111-0.4j 0.118+0.375j 0.118-0.375j 0.237+0.157j] OVev [-2.112+0.j -2.02 +0.513j -2.02 -0.513j -2.002+0.082j -2.002-0.082j] L2H1 [np.float64(4.1385), np.float64(1.1546), 13, 39, np.float64(0.0163), np.float64(0.0398)] [np.float64(24.8446), np.float64(5.4684), 16, 42, np.float64(0.0796), np.float64(-0.2716)] QKev [ 0.264+0.j 0.233+0.087j 0.233-0.087j 0.245+0.j -0.167+0.139j] OVev [-2.182+0.269j -2.182-0.269j -1.599+1.285j -1.599-1.285j -1.913+0.j ] L2H2 [np.float64(3.1536), np.float64(1.11), 11, 39, np.float64(0.0121), np.float64(-0.0914)] [np.float64(28.7074), np.float64(7.8628), 12, 38, np.float64(0.0723), np.float64(-0.2283)] QKev [ 0.031+0.213j 0.031-0.213j 0.174+0.113j 0.174-0.113j -0.196+0.j ] OVev [-2.963+0.j -2.564+0.636j -2.564-0.636j -2.556+0.j -2.024+0.j ] L2H3 [np.float64(4.1027), np.float64(1.0661), 13, 39, np.float64(-0.0033), np.float64(-0.0502)] [np.float64(24.5055), np.float64(6.1803), 14, 41, np.float64(0.0956), np.float64(-0.2784)] QKev [ 0.11 +0.404j 0.11 -0.404j -0.234+0.j -0.222+0.061j -0.222-0.061j] OVev [-2.719+0.j -2.27 +0.499j -2.27 -0.499j -1.906+0.032j -1.906-0.032j] L2H4 [np.float64(4.6034), np.float64(1.1879), 13, 37, np.float64(-0.0074), np.float64(-0.0285)] [np.float64(23.2786), np.float64(5.0699), 16, 41, np.float64(0.08), np.float64(-0.2651)] QKev [ 0.105+0.409j 0.105-0.409j -0.253+0.j 0.161+0.192j 0.161-0.192j] OVev [-1.945+0.j -1.822+0.26j -1.822-0.26j -1.585+0.715j -1.585-0.715j] L2H5 [np.float64(4.4193), np.float64(1.2007), 12, 36, np.float64(-0.008), np.float64(-0.0415)] [np.float64(23.3416), np.float64(4.8404), 16, 42, np.float64(0.0994), np.float64(-0.2923)] QKev [ 0.083+0.407j 0.083-0.407j -0.274+0.j -0.158+0.216j -0.158-0.216j] OVev [-2.304+0.j -1.972+0.j -1.941+0.17j -1.941-0.17j -1.757+0.454j] L3H0 [np.float64(3.9054), np.float64(0.7905), 18, 46, np.float64(0.0512), np.float64(0.2014)] [np.float64(29.6831), np.float64(8.3838), 13, 41, np.float64(0.1181), np.float64(-0.3382)] QKev [0.321+0.j 0.289+0.028j 0.289-0.028j 0.243+0.137j 0.243-0.137j] OVev [-3.166+0.235j -3.166-0.235j -2.676+0.j -2.512+0.j -2.339+0.41j ] L3H1 [np.float64(3.2236), np.float64(0.5714), 21, 50, np.float64(0.2114), np.float64(0.4453)] [np.float64(53.3976), np.float64(13.8537), 10, 23, np.float64(0.0622), np.float64(-0.2166)] QKev [0.301+0.13j 0.301-0.13j 0.309-0.096j 0.309+0.096j 0.304+0.j ] OVev [-5.342+0.j -4.693+0.548j -4.693-0.548j -3.938+0.j -3.693+0.482j] L3H2 [np.float64(2.6302), np.float64(0.808), 12, 42, np.float64(-0.0016), np.float64(0.0211)] [np.float64(35.2432), np.float64(11.8908), 9, 33, np.float64(0.0488), np.float64(-0.1901)] QKev [ 0.053+0.216j 0.053-0.216j 0.148+0.075j 0.148-0.075j -0.063+0.142j] OVev [-3.171+0.559j -3.171-0.559j -2.821+0.468j -2.821-0.468j -2.168+1.06j ] L3H3 [np.float64(2.7937), np.float64(0.8563), 14, 45, np.float64(-0.0148), np.float64(0.0092)] [np.float64(35.9947), np.float64(12.4344), 9, 37, np.float64(0.0483), np.float64(-0.2096)] QKev [ 0.041+0.26j 0.041-0.26j -0.036+0.163j -0.036-0.163j 0.155+0.024j] OVev [-3.108+0.j -2.921+1.018j -2.921-1.018j -2.591+0.j -2.092+1.046j] L3H4 [np.float64(3.2493), np.float64(0.919), 15, 45, np.float64(0.0008), np.float64(0.0666)] [np.float64(31.7775), np.float64(9.5258), 12, 39, np.float64(0.0392), np.float64(-0.1946)] QKev [0.115+0.169j 0.115-0.169j 0.175+0.059j 0.175-0.059j 0.18 +0.j ] OVev [-2.931+0.j -2.449+0.j -2.121+0.j -1.369+1.511j -1.369-1.511j] L3H5 [np.float64(1.4813), np.float64(0.4831), 10, 40, np.float64(0.0244), np.float64(0.0166)] [np.float64(59.2935), np.float64(21.9791), 6, 18, np.float64(0.0647), np.float64(-0.1821)] QKev [0.135+0.058j 0.135-0.058j 0.134+0.j 0.042+0.087j 0.042-0.087j] OVev [-8.508+0.j -6.584+0.j -5.094+0.838j -5.094-0.838j -3.835+1.273j] ```