OUTPUT of run 2 of at most 100: ``` GAIN-WEIGHTED PRODUCTS: A=diag(ln1_w) Q.T K diag(ln1_w)/8, B=O V diag(ln1_w) These are matrix products only; no LayerNorm, attention, or MLP evaluations. L0H0: A_norm=0.0546, sr=1.13, symfrac=0.715, B_norm=9.043, top_s=[0.0514 0.0114 0.0071 0.0046 0.0043 0.0036], eig=[ 0.0072+0.j -0.004 +0.j -0.0024+0.j -0.0005+0.0014j -0.0005-0.0014j 0.0002+0.0012j 0.0002-0.0012j -0.0011+0.0004j] L0H1: A_norm=0.0295, sr=1.40, symfrac=0.724, B_norm=6.966, top_s=[0.025 0.0095 0.0053 0.0044 0.0035 0.0031], eig=[-0.0043+0.0011j -0.0043-0.0011j 0.002 +0.j -0.0015+0.0007j -0.0015-0.0007j -0.0012+0.j -0.0008+0.0008j -0.0008-0.0008j] L0H2: A_norm=4.0784, sr=8.31, symfrac=0.690, B_norm=2.540, top_s=[1.415 1.2749 1.2149 1.2061 1.159 1.0942], eig=[ 0.6322+0.8142j 0.6322-0.8142j -0.23 +0.9352j -0.23 -0.9352j 0.3332+0.8752j 0.3332-0.8752j -0.7788+0.3341j -0.7788-0.3341j] L0H3: A_norm=0.3946, sr=1.21, symfrac=0.860, B_norm=1.766, top_s=[0.3588 0.1191 0.0776 0.0425 0.0326 0.0273], eig=[ 0.2617+0.j 0.0777+0.j 0.0101+0.0309j 0.0101-0.0309j -0.0173+0.0052j -0.0173-0.0052j -0.0175+0.j -0.0148+0.0026j] L0H4: A_norm=0.1063, sr=1.42, symfrac=0.709, B_norm=4.034, top_s=[0.0892 0.0536 0.0165 0.0065 0.0052 0.0041], eig=[-0.0279+0.0279j -0.0279-0.0279j -0.0078+0.j -0.0022+0.j -0.0011+0.0014j -0.0011-0.0014j -0.0015+0.j 0.0005-0.0011j] L0H5: A_norm=0.0776, sr=1.04, symfrac=0.762, B_norm=9.311, top_s=[0.0759 0.0108 0.0046 0.0037 0.0034 0.0029], eig=[-0.0311+0.j 0.0007+0.0017j 0.0007-0.0017j -0.0013+0.0003j -0.0013-0.0003j -0.0005+0.0011j -0.0005-0.0011j 0.0006-0.0008j] L1H0: A_norm=0.8874, sr=2.20, symfrac=0.710, B_norm=10.122, top_s=[0.5978 0.4823 0.2477 0.194 0.1316 0.1118], eig=[ 0.0276+0.1058j 0.0276-0.1058j -0.0784+0.j -0.0652+0.0282j -0.0652-0.0282j -0.055 +0.007j -0.055 -0.007j -0.0469+0.j ] L1H1: A_norm=1.3157, sr=2.23, symfrac=0.714, B_norm=10.333, top_s=[0.8818 0.6663 0.4114 0.2669 0.203 0.1785], eig=[ 0.069 +0.1912j 0.069 -0.1912j -0.1533+0.j -0.1067+0.0244j -0.1067-0.0244j -0.1076+0.0063j -0.1076-0.0063j -0.0862+0.j ] L1H2: A_norm=1.5512, sr=2.43, symfrac=0.713, B_norm=9.181, top_s=[0.9947 0.8604 0.3835 0.3254 0.2644 0.2271], eig=[ 0.0955+0.246j 0.0955-0.246j -0.1678+0.j -0.1413+0.0645j -0.1413-0.0645j -0.1349+0.0089j -0.1349-0.0089j -0.1036+0.0298j] L1H3: A_norm=1.3759, sr=2.24, symfrac=0.713, B_norm=10.051, top_s=[0.9196 0.7386 0.4078 0.2536 0.2205 0.1718], eig=[ 0.0931+0.2031j 0.0931-0.2031j -0.1754+0.j -0.1129+0.0083j -0.1129-0.0083j -0.1071+0.j -0.0401+0.091j -0.0401-0.091j ] L1H4: A_norm=1.4507, sr=2.34, symfrac=0.711, B_norm=10.401, top_s=[0.9477 0.8684 0.3597 0.259 0.1933 0.1725], eig=[ 0.0609+0.2141j 0.0609-0.2141j -0.1348+0.0091j -0.1348-0.0091j -0.1275+0.j -0.0988+0.j -0.0816+0.0532j -0.0816-0.0532j] L1H5: A_norm=1.3383, sr=2.24, symfrac=0.706, B_norm=10.620, top_s=[0.8932 0.7393 0.3722 0.2296 0.204 0.1849], eig=[ 0.0873+0.2208j 0.0873-0.2208j -0.1207+0.0123j -0.1207-0.0123j -0.1 +0.0043j -0.1 -0.0043j -0.0743+0.0552j -0.0743-0.0552j] L2H0: A_norm=2.4382, sr=12.31, symfrac=0.695, B_norm=16.307, top_s=[0.6949 0.6297 0.6024 0.5633 0.5301 0.5065], eig=[ 0.0526+0.306j 0.0526-0.306j 0.0363+0.183j 0.0363-0.183j 0.1184+0.0804j 0.1184-0.0804j -0.0766+0.0978j -0.0766-0.0978j] L2H1: A_norm=2.0681, sr=15.42, symfrac=0.711, B_norm=17.722, top_s=[0.5267 0.4494 0.4325 0.4171 0.4097 0.3951], eig=[ 0.0271+0.1441j 0.0271-0.1441j 0.1258+0.0113j 0.1258-0.0113j 0.1092+0.0374j 0.1092-0.0374j -0.084 +0.0709j -0.084 -0.0709j] L2H2: A_norm=1.6133, sr=7.06, symfrac=0.706, B_norm=20.575, top_s=[0.6072 0.503 0.4013 0.3172 0.2979 0.2866], eig=[ 0.0363+0.1743j 0.0363-0.1743j -0.1058+0.j 0.0675+0.0552j 0.0675-0.0552j -0.0836-0.0147j -0.0836+0.0147j -0.0761+0.0139j] L2H3: A_norm=2.0871, sr=13.79, symfrac=0.698, B_norm=17.506, top_s=[0.562 0.5256 0.4916 0.4403 0.4116 0.3945], eig=[ 0.0718+0.3068j 0.0718-0.3068j -0.0764+0.084j -0.0764-0.084j -0.1022-0.0418j -0.1022+0.0418j -0.1042+0.0055j -0.1042-0.0055j] L2H4: A_norm=2.3444, sr=13.94, symfrac=0.698, B_norm=16.624, top_s=[0.6279 0.6043 0.5322 0.5082 0.467 0.4514], eig=[ 0.0613+0.3024j 0.0613-0.3024j 0.0769+0.1016j 0.0769-0.1016j -0.0811-0.091j -0.0811+0.091j -0.1183+0.j -0.1109+0.0277j] L2H5: A_norm=2.2511, sr=12.43, symfrac=0.697, B_norm=16.684, top_s=[0.6384 0.5397 0.4937 0.4762 0.4679 0.4519], eig=[ 0.0458+0.3023j 0.0458-0.3023j -0.1412+0.j -0.0716+0.1085j -0.0716-0.1085j -0.1192+0.0373j -0.1192-0.0373j 0.0509+0.1089j] L3H0: A_norm=2.2327, sr=24.45, symfrac=0.725, B_norm=22.598, top_s=[0.4515 0.4199 0.4181 0.4053 0.4008 0.3946], eig=[0.1821+0.j 0.1411+0.0785j 0.1411-0.0785j 0.1581-0.0188j 0.1581+0.0188j 0.1434+0.0562j 0.1434-0.0562j 0.1412+0.0357j] L3H1: A_norm=1.8309, sr=31.24, symfrac=0.778, B_norm=40.869, top_s=[0.3276 0.3174 0.3092 0.3059 0.3008 0.2979], eig=[0.171 +0.0673j 0.171 -0.0673j 0.1739+0.0596j 0.1739-0.0596j 0.1486+0.0815j 0.1486-0.0815j 0.1641-0.0312j 0.1641+0.0312j] L3H2: A_norm=1.5058, sr=10.23, symfrac=0.706, B_norm=26.882, top_s=[0.4708 0.4111 0.3556 0.3321 0.3087 0.2904], eig=[ 0.0301+0.1338j 0.0301-0.1338j 0.081 +0.0387j 0.081 -0.0387j -0.0348+0.0816j -0.0348-0.0816j 0.0823+0.j -0.0765+0.j ] L3H3: A_norm=1.5992, sr=10.34, symfrac=0.701, B_norm=27.431, top_s=[0.4974 0.4025 0.3358 0.3039 0.2956 0.2949], eig=[ 0.0233+0.1587j 0.0233-0.1587j 0.099 +0.j -0.0228+0.0945j -0.0228-0.0945j 0.0762+0.0363j 0.0762-0.0363j 0.0778+0.j ] L3H4: A_norm=1.8560, sr=12.32, symfrac=0.707, B_norm=24.198, top_s=[0.5288 0.4125 0.3981 0.3736 0.3603 0.3361], eig=[0.0677+0.0916j 0.0677-0.0916j 0.0027+0.1082j 0.0027-0.1082j 0.0982+0.0395j 0.0982-0.0395j 0.1025+0.j 0.0706+0.0664j] L3H5: A_norm=0.8349, sr=10.36, symfrac=0.716, B_norm=45.401, top_s=[0.2594 0.2349 0.2122 0.1963 0.1906 0.1595], eig=[0.0754+0.035j 0.0754-0.035j 0.0779+0.j 0.021 +0.0476j 0.021 -0.0476j 0.0222+0.0351j 0.0222-0.0351j 0.0376+0.0108j] PAIRWISE QK COSINES WITHIN EACH LAYER L 0 [[ 1. 0.209 -0.0156 -0.0138 0.1151 0.0181] [ 0.209 1. -0.0039 -0.0356 0.2091 -0.0084] [-0.0156 -0.0039 1. 0.2393 -0.0061 -0.0256] [-0.0138 -0.0356 0.2393 1. -0.0536 -0.0579] [ 0.1151 0.2091 -0.0061 -0.0536 1. 0.1721] [ 0.0181 -0.0084 -0.0256 -0.0579 0.1721 1. ]] L 1 [[1. 0.578 0.4169 0.4486 0.4554 0.4652] [0.578 1. 0.5504 0.4889 0.5564 0.5281] [0.4169 0.5504 1. 0.6087 0.6676 0.5649] [0.4486 0.4889 0.6087 1. 0.6442 0.5634] [0.4554 0.5564 0.6676 0.6442 1. 0.5778] [0.4652 0.5281 0.5649 0.5634 0.5778 1. ]] L 2 [[1. 0.0783 0.1327 0.1034 0.112 0.1062] [0.0783 1. 0.0931 0.0855 0.0767 0.0704] [0.1327 0.0931 1. 0.1188 0.1265 0.1248] [0.1034 0.0855 0.1188 1. 0.1198 0.1131] [0.112 0.0767 0.1265 0.1198 1. 0.1074] [0.1062 0.0704 0.1248 0.1131 0.1074 1. ]] L 3 [[1. 0.0706 0.0444 0.036 0.0755 0.03 ] [0.0706 1. 0.0208 0.0236 0.0301 0.045 ] [0.0444 0.0208 1. 0.0946 0.1035 0.0545] [0.036 0.0236 0.0946 1. 0.0836 0.0652] [0.0755 0.0301 0.1035 0.0836 1. 0.0493] [0.03 0.045 0.0545 0.0652 0.0493 1. ]] L0H2 FULL NONZERO SINGULAR SPECTRUM [1.415 1.2749 1.2149 1.2061 1.159 1.0942 1.0863 1.0219 0.9558 0.9159 0.8613 0.7816 0.7279 0.7214 0.5146 0.4919 0.4289 0.3761 0.3219 0.29 0.189 0.1656 0.1282 0.1157 0.1096 0.0981 0.0751 0.0561 0.0492 0.0435 0.0371 0.0336 0.0302 0.0272 0.0242 0.0196 0.0195 0.0157 0.0145 0.013 0.0111 0.0096 0.0088 0.0086 0.0077 0.0073 0.0065 0.0064 0.0059 0.0048 0.0043 0.0039 0.0034 0.0032 0.0031 0.0025 0.0024 0.002 0.0019 0.0018 0.0016 0.0015 0.0014 0.0011] L0H2 TOP EIGENVALUES OF UNWEIGHTED Q.T K/8 [-0.8672+5.7867j -0.8672-5.7867j 1.92 +4.997j 1.92 -4.997j 4.415 +2.6762j 4.415 -2.6762j -4.9189+0.j -4.1248+2.3614j -4.1248-2.3614j 2.2873+4.0016j 2.2873-4.0016j 3.8125+0.j 3.0783+0.5985j 3.0783-0.5985j -2.3696-1.5219j -2.3696+1.5219j -1.8129+0.j -1.2196+1.0581j -1.2196-1.0581j 1.0283+1.2431j 1.0283-1.2431j -0.764 +0.j 0.1563+0.6708j 0.1563-0.6708j -0.2455+0.579j -0.2455-0.579j 0.6239+0.j -0.4258+0.j 0.1483+0.2689j 0.1483-0.2689j -0.0088-0.1973j -0.0088+0.1973j] RESIDUAL COORDINATE CONCENTRATION L0: top_coordinates=[162 60 35 223 246 4 180 184 300 311 239 136 178 222 72 65]; fraction_top16=0.3677; gains=[1.0341 0.9844 0.937 0.98 0.8815 0.8678 0.9294 0.8446 0.9371 0.9443 0.9385 0.8959 0.8898 0.8362 0.7062 0.8244] L1: top_coordinates=[222 72 78 273 179 302 351 254 136 200 111 211 42 252 314 114]; fraction_top16=0.4125; gains=[1.1239 0.7995 0.7811 0.4613 0.6558 0.7671 0.6989 0.3414 0.4917 0.6776 0.6917 0.6343 0.657 0.5774 0.6457 0.4324] L2: top_coordinates=[222 78 101 94 321 181 100 117 250 126 11 247 122 205 264 225]; fraction_top16=0.1398; gains=[1.0144 0.9057 0.7889 0.817 0.8106 0.7813 0.7841 0.8122 0.813 0.8108 0.7635 0.7678 0.7934 0.8028 0.795 0.7764] L3: top_coordinates=[222 78 187 164 98 177 147 83 178 201 27 232 357 143 295 318]; fraction_top16=0.0724; gains=[0.8271 0.7626 0.8177 0.8323 0.8021 0.8597 0.8001 0.8151 0.8204 0.8132 0.8432 0.8405 0.7982 0.8161 0.81 0.8387] DIRECT OV ALIGNMENT WITH LATER QK (matrix composition only) For each later head: strongest earlier B on key side; enrichment = ||A B||F/(||A||F||B||F/sqrt(384)). L1H0: [('L0H5', np.float64(10.954), np.float64(0.0004)), ('L0H0', np.float64(10.712), np.float64(0.0036)), ('L0H1', np.float64(10.363), np.float64(0.0016)), ('L0H4', np.float64(10.234), np.float64(0.0031)), ('L0H3', np.float64(3.044), np.float64(-0.0061))] L1H1: [('L0H5', np.float64(10.982), np.float64(0.0017)), ('L0H1', np.float64(9.858), np.float64(0.0033)), ('L0H0', np.float64(9.79), np.float64(0.0047)), ('L0H4', np.float64(9.51), np.float64(0.0052)), ('L0H3', np.float64(2.864), np.float64(-0.005))] L1H2: [('L0H5', np.float64(10.822), np.float64(0.002)), ('L0H1', np.float64(9.857), np.float64(0.0042)), ('L0H0', np.float64(9.775), np.float64(0.0051)), ('L0H4', np.float64(9.47), np.float64(0.0057)), ('L0H3', np.float64(3.131), np.float64(-0.0019))] L1H3: [('L0H5', np.float64(11.072), np.float64(0.0025)), ('L0H0', np.float64(10.019), np.float64(0.0058)), ('L0H1', np.float64(9.935), np.float64(0.0047)), ('L0H4', np.float64(9.625), np.float64(0.0062)), ('L0H3', np.float64(2.943), np.float64(-0.0034))] L1H4: [('L0H5', np.float64(11.164), np.float64(0.0041)), ('L0H0', np.float64(10.567), np.float64(0.0075)), ('L0H1', np.float64(10.381), np.float64(0.0068)), ('L0H4', np.float64(10.099), np.float64(0.0083)), ('L0H3', np.float64(3.341), np.float64(-0.0014))] L1H5: [('L0H5', np.float64(10.958), np.float64(-0.001)), ('L0H1', np.float64(9.864), np.float64(0.0018)), ('L0H0', np.float64(9.855), np.float64(0.0039)), ('L0H4', np.float64(9.48), np.float64(0.0038)), ('L0H3', np.float64(3.016), np.float64(-0.0046))] L2H0: [('L0H5', np.float64(4.344), np.float64(0.0037)), ('L0H1', np.float64(3.956), np.float64(0.0031)), ('L0H0', np.float64(3.888), np.float64(0.004)), ('L0H4', np.float64(3.749), np.float64(0.0049)), ('L0H3', np.float64(1.843), np.float64(-0.0099))] L2H1: [('L0H5', np.float64(3.173), np.float64(0.0037)), ('L0H1', np.float64(2.807), np.float64(0.0023)), ('L0H0', np.float64(2.721), np.float64(0.0036)), ('L0H4', np.float64(2.689), np.float64(0.0039)), ('L0H3', np.float64(1.536), np.float64(-0.0066))] L2H2: [('L0H5', np.float64(5.912), np.float64(0.0017)), ('L0H1', np.float64(5.461), np.float64(0.0044)), ('L0H0', np.float64(5.288), np.float64(0.0056)), ('L0H4', np.float64(5.152), np.float64(0.0069)), ('L0H3', np.float64(2.704), np.float64(-0.0064))] L2H3: [('L0H5', np.float64(4.295), np.float64(0.005)), ('L0H1', np.float64(4.068), np.float64(0.004)), ('L0H0', np.float64(4.057), np.float64(0.0056)), ('L0H4', np.float64(3.918), np.float64(0.0059)), ('L0H3', np.float64(1.816), np.float64(-0.0066))] L2H4: [('L0H5', np.float64(4.133), np.float64(0.0051)), ('L0H1', np.float64(4.02), np.float64(0.0055)), ('L0H0', np.float64(4.017), np.float64(0.0073)), ('L0H4', np.float64(3.866), np.float64(0.0077)), ('L0H3', np.float64(1.929), np.float64(-0.0087))] L2H5: [('L0H5', np.float64(4.191), np.float64(0.004)), ('L0H1', np.float64(3.946), np.float64(0.0037)), ('L0H0', np.float64(3.926), np.float64(0.0056)), ('L0H4', np.float64(3.798), np.float64(0.0056)), ('L0H3', np.float64(1.812), np.float64(-0.0095))] L3H0: [('L0H5', np.float64(1.743), np.float64(0.004)), ('L0H0', np.float64(1.607), np.float64(0.0028)), ('L0H1', np.float64(1.584), np.float64(0.003)), ('L0H4', np.float64(1.581), np.float64(0.0044)), ('L1H5', np.float64(1.074), np.float64(-0.0161))] L3H1: [('L0H2', np.float64(1.586), np.float64(0.0598)), ('L1H3', np.float64(1.449), np.float64(0.0392)), ('L2H3', np.float64(1.335), np.float64(0.0248)), ('L2H2', np.float64(1.309), np.float64(0.0243)), ('L2H1', np.float64(1.272), np.float64(0.0638))] L3H2: [('L0H5', np.float64(4.268), np.float64(0.0052)), ('L0H0', np.float64(3.656), np.float64(0.0044)), ('L0H1', np.float64(3.484), np.float64(0.0029)), ('L0H4', np.float64(3.405), np.float64(0.0041)), ('L0H3', np.float64(1.379), np.float64(-0.0042))] L3H3: [('L0H5', np.float64(4.065), np.float64(0.0035)), ('L0H0', np.float64(3.504), np.float64(0.003)), ('L0H1', np.float64(3.441), np.float64(0.0014)), ('L0H4', np.float64(3.373), np.float64(0.0025)), ('L0H3', np.float64(1.357), np.float64(-0.0013))] L3H4: [('L0H5', np.float64(3.368), np.float64(0.0084)), ('L0H0', np.float64(2.886), np.float64(0.0068)), ('L0H1', np.float64(2.849), np.float64(0.0072)), ('L0H4', np.float64(2.789), np.float64(0.0082)), ('L0H3', np.float64(1.28), np.float64(-0.001))] L3H5: [('L0H0', np.float64(1.439), np.float64(-0.0035)), ('L0H3', np.float64(1.414), np.float64(-0.0029)), ('L0H5', np.float64(1.38), np.float64(-0.0022)), ('L0H4', np.float64(1.341), np.float64(-0.0033)), ('L0H1', np.float64(1.281), np.float64(-0.0036))] ```