OUTPUT of run 1 of at most 100: ``` AVAILABLE WEIGHTS [('W_fc.0', (1536, 384)), ('W_fc.1', (1536, 384)), ('W_fc.2', (1536, 384)), ('W_fc.3', (1536, 384)), ('W_k.0', (384, 384)), ('W_k.1', (384, 384)), ('W_k.2', (384, 384)), ('W_k.3', (384, 384)), ('W_o.0', (384, 384)), ('W_o.1', (384, 384)), ('W_o.2', (384, 384)), ('W_o.3', (384, 384)), ('W_proj.0', (384, 1536)), ('W_proj.1', (384, 1536)), ('W_proj.2', (384, 1536)), ('W_proj.3', (384, 1536)), ('W_q.0', (384, 384)), ('W_q.1', (384, 384)), ('W_q.2', (384, 384)), ('W_q.3', (384, 384)), ('W_v.0', (384, 384)), ('W_v.1', (384, 384)), ('W_v.2', (384, 384)), ('W_v.3', (384, 384)), ('ln1_b.0', (384,)), ('ln1_b.1', (384,)), ('ln1_b.2', (384,)), ('ln1_b.3', (384,)), ('ln1_w.0', (384,)), ('ln1_w.1', (384,)), ('ln1_w.2', (384,)), ('ln1_w.3', (384,)), ('ln2_b.0', (384,)), ('ln2_b.1', (384,)), ('ln2_b.2', (384,)), ('ln2_b.3', (384,)), ('ln2_w.0', (384,)), ('ln2_w.1', (384,)), ('ln2_w.2', (384,)), ('ln2_w.3', (384,))] LAYERNORM PARAMETERS: min, median, max, mean, std, L2 norm ln1_w.0 [-0.001 0.1422 1.0341 0.2403 0.2436 6.7051] ln1_b.0 [-0.0959 0.0007 0.091 0.0008 0.0287 0.5635] ln2_w.0 [ 0.0044 0.7876 1.5072 0.8006 0.3295 16.9648] ln2_b.0 [-0.4986 0.0058 0.3331 0.0022 0.0683 1.3392] ln1_w.1 [0.1361 0.4535 1.1239 0.4575 0.1022 9.1868] ln1_b.1 [-1.4712 -0.0071 1.1544 -0.005 0.1672 3.2771] ln2_w.1 [-0.0019 1.0787 1.4856 1.0429 0.2003 20.8093] ln2_b.1 [-0.3597 -0.0041 0.3686 0.0005 0.1364 2.6732] ln1_w.2 [ 0.2078 0.7176 1.0144 0.7052 0.0749 13.8969] ln1_b.2 [-1.3819 -0.0017 1.0527 -0.001 0.1525 2.9893] ln2_w.2 [ 0.0011 1.3247 1.4655 1.3123 0.1255 25.8329] ln2_b.2 [-0.3761 0.0067 0.3886 0.005 0.1537 3.0128] ln1_w.3 [ 0.2944 0.7581 0.8701 0.7546 0.0542 14.8247] ln1_b.3 [-0.6069 0.007 0.6502 0.0121 0.0849 1.6801] ln2_w.3 [ 0.4364 1.2851 1.4306 1.2778 0.0808 25.0891] ln2_b.3 [-0.6728 0.0019 0.3068 0.0043 0.1276 2.5016] MATRIX SPECTRA: Frobenius norm, largest singular value, stable rank, top 8 singular values W_q.0: norm=32.2912 smax=8.2519 stable_rank=15.31 top=[8.2519 8.2232 7.963 7.819 7.68 7.6416 7.5194 7.2054] W_k.0: norm=30.3909 smax=7.6464 stable_rank=15.80 top=[7.6464 7.4363 7.3304 6.9118 6.8285 6.6716 6.4724 6.4464] W_v.0: norm=14.8360 smax=4.3390 stable_rank=11.69 top=[4.339 3.5292 3.2699 3.1656 2.9824 2.6465 2.589 2.5519] W_o.0: norm=16.3713 smax=7.1033 stable_rank=5.31 top=[7.1033 6.6074 4.2683 2.7702 2.5025 2.285 2.2638 2.2184] W_fc.0: norm=46.2486 smax=18.1759 stable_rank=6.47 top=[18.1759 11.0178 8.7366 7.4517 7.2123 7.0621 7.0112 6.6204] W_proj.0: norm=39.3075 smax=9.8257 stable_rank=16.00 top=[9.8257 7.3508 6.2651 5.8693 5.7965 5.6574 5.5163 5.4501] W_q.1: norm=32.3126 smax=9.9032 stable_rank=10.65 top=[9.9032 7.7974 6.7042 5.9516 5.6721 5.6025 5.4021 5.3125] W_k.1: norm=29.2506 smax=13.3072 stable_rank=4.83 top=[13.3072 9.9327 5.8989 5.677 5.0458 4.622 4.4903 4.4044] W_v.1: norm=27.3529 smax=5.2845 stable_rank=26.79 top=[5.2845 4.9661 4.87 4.7328 4.6743 4.5853 4.4113 4.3216] W_o.1: norm=25.9233 smax=4.9657 stable_rank=27.25 top=[4.9657 3.2412 3.157 3.1119 3.0615 3.0531 2.9942 2.9545] W_fc.1: norm=79.1265 smax=21.4881 stable_rank=13.56 top=[21.4881 13.1349 12.8042 12.6247 12.3279 12.1184 11.8504 11.6908] W_proj.1: norm=75.7103 smax=8.1935 stable_rank=85.38 top=[8.1935 7.8941 7.1992 6.9972 6.671 6.4329 6.2745 6.1333] W_q.2: norm=39.3455 smax=5.7962 stable_rank=46.08 top=[5.7962 5.1962 4.8092 4.6159 4.6003 4.5145 4.4419 4.432 ] W_k.2: norm=37.7688 smax=10.6627 stable_rank=12.55 top=[10.6627 8.6244 4.9887 4.3937 4.241 4.0878 3.9564 3.8475] W_v.2: norm=33.3048 smax=3.7062 stable_rank=80.75 top=[3.7062 3.6245 3.5788 3.5334 3.5012 3.4679 3.4285 3.4153] W_o.2: norm=32.8397 smax=4.1025 stable_rank=64.08 top=[4.1025 3.9708 3.558 3.3813 3.3611 3.2925 3.1918 3.1686] W_fc.2: norm=78.9668 smax=15.3634 stable_rank=26.42 top=[15.3634 8.0212 7.8946 7.7247 7.5968 7.4905 7.2574 7.2158] W_proj.2: norm=76.8311 smax=9.4036 stable_rank=66.76 top=[9.4036 9.1867 8.7234 8.4703 8.0398 7.7955 7.6145 7.4501] W_q.3: norm=32.9482 smax=4.5598 stable_rank=52.21 top=[4.5598 4.2388 4.0054 3.9241 3.8529 3.7087 3.7044 3.6911] W_k.3: norm=32.2094 smax=6.0903 stable_rank=27.97 top=[6.0903 4.8989 4.1973 3.4559 3.3531 3.3051 3.2455 3.1608] W_v.3: norm=39.7941 smax=5.0598 stable_rank=61.85 top=[5.0598 4.9045 4.8481 4.769 4.6738 4.6053 4.5722 4.5185] W_o.3: norm=40.4550 smax=9.5668 stable_rank=17.88 top=[9.5668 8.7592 5.3386 5.139 4.9162 4.8063 4.7743 4.7359] W_fc.3: norm=77.7758 smax=14.1624 stable_rank=30.16 top=[14.1624 8.1249 7.9152 7.3262 7.1484 6.8556 6.7628 6.6911] W_proj.3: norm=82.3952 smax=26.3094 stable_rank=9.81 top=[26.3094 20.6845 19.6986 19.2612 18.5937 18.1973 17.8617 17.6671] HEAD WEIGHT PRODUCTS (no inputs or forward passes) QK=Q.T@K/8; OV=O@V. symfrac=||symmetric part||F/||matrix||F; trcos=trace/(sqrt(384)*norm). L0H0: Q,K,V,O norms=[3.305 4.18 6.039 6.666] QK norm=0.381 top=[0.314 0.112 0.087 0.055 0.041 0.039] sr=1.47 symfrac=0.727 trcos=-0.0289; OV norm=12.904 top=[10.252 5.603 3.475 2.818] sr=1.58 trcos=-0.0007 L0H1: Q,K,V,O norms=[3.09 3.658 5.382 6.025] QK norm=0.230 top=[0.137 0.096 0.056 0.043 0.037 0.036] sr=2.81 symfrac=0.747 trcos=-0.0343; OV norm=10.148 top=[7.662 5.172 2.582 2.031] sr=1.75 trcos=-0.0038 L0H2: Q,K,V,O norms=[30.818 28.3 7.765 8.915] QK norm=22.804 top=[7.485 7.16 6.948 6.793 6.404 6.182] sr=9.28 symfrac=0.696 trcos=0.0092; OV norm=13.223 top=[4.001 3.853 3.743 3.544] sr=10.92 trcos=-0.0179 L0H3: Q,K,V,O norms=[6.821 7.243 5.434 5.029] QK norm=1.419 top=[0.821 0.749 0.402 0.331 0.303 0.267] sr=2.99 symfrac=0.763 trcos=-0.0359; OV norm=7.308 top=[4.305 4.046 2.493 2.078] sr=2.88 trcos=-0.0143 L0H4: Q,K,V,O norms=[3.971 4.995 5.208 5.587] QK norm=0.826 top=[0.753 0.255 0.135 0.07 0.054 0.043] sr=1.20 symfrac=0.763 trcos=-0.0365; OV norm=7.312 top=[4.879 3.236 2.548 2.083] sr=2.25 trcos=-0.0067 L0H5: Q,K,V,O norms=[3.19 3.802 6.143 7.162] QK norm=0.322 top=[0.274 0.068 0.049 0.041 0.037 0.037] sr=1.39 symfrac=0.729 trcos=-0.0191; OV norm=13.038 top=[10.314 5.468 3.148 2.935] sr=1.60 trcos=-0.0003 L1H0: Q,K,V,O norms=[10.716 9.827 11.156 9.984] QK norm=2.958 top=[1.577 1.462 1.035 0.877 0.653 0.522] sr=3.52 symfrac=0.735 trcos=-0.0563; OV norm=21.707 top=[8.099 7.58 6.398 6.202] sr=7.18 trcos=-0.0508 L1H1: Q,K,V,O norms=[13.292 12.068 11.72 11.303] QK norm=4.439 top=[2.456 2.132 1.344 1.193 0.918 0.869] sr=3.27 symfrac=0.742 trcos=-0.0647; OV norm=22.226 top=[6.583 6.009 5.739 5.681] sr=11.40 trcos=-0.0654 L1H2: Q,K,V,O norms=[14.476 13.092 10.668 10.272] QK norm=5.223 top=[2.923 2.292 1.508 1.402 1.191 1.074] sr=3.19 symfrac=0.745 trcos=-0.0740; OV norm=19.517 top=[6.135 5.788 5.252 4.987] sr=10.12 trcos=-0.0573 L1H3: Q,K,V,O norms=[13.358 12.153 10.904 9.928] QK norm=4.605 top=[2.638 2.173 1.454 1.142 1.062 0.813] sr=3.05 symfrac=0.740 trcos=-0.0616; OV norm=21.413 top=[7.767 6.864 6.527 6.06 ] sr=7.60 trcos=-0.0483 L1H4: Q,K,V,O norms=[13.518 12.129 11.148 10.794] QK norm=4.673 top=[2.87 2.178 1.43 1.085 0.911 0.776] sr=2.65 symfrac=0.737 trcos=-0.0636; OV norm=22.066 top=[6.498 6.253 5.897 5.784] sr=11.53 trcos=-0.0615 L1H5: Q,K,V,O norms=[13.486 12.132 11.376 11.136] QK norm=4.526 top=[2.618 2.293 1.294 1.053 0.925 0.835] sr=2.99 symfrac=0.730 trcos=-0.0560; OV norm=22.464 top=[6.208 6.019 5.803 5.645] sr=13.09 trcos=-0.0665 L2H0: Q,K,V,O norms=[16.822 16.208 13.115 12.942] QK norm=4.822 top=[1.457 1.249 1.141 1.088 1.043 0.997] sr=10.95 symfrac=0.702 trcos=0.0026; OV norm=22.814 top=[4.898 4.628 4.553 4.4 ] sr=21.70 trcos=-0.1214 L2H1: Q,K,V,O norms=[15.963 15.287 13.768 13.525] QK norm=4.139 top=[1.155 0.893 0.881 0.867 0.816 0.794] sr=12.85 symfrac=0.713 trcos=0.0146; OV norm=24.845 top=[5.468 5.157 4.968 4.755] sr=20.64 trcos=-0.1030 L2H2: Q,K,V,O norms=[13.97 13.239 14.567 14.24 ] QK norm=3.154 top=[1.11 0.945 0.773 0.625 0.589 0.579] sr=8.07 symfrac=0.711 trcos=-0.0328; OV norm=28.707 top=[7.863 7.18 6.901 5.989] sr=13.33 trcos=-0.0831 L2H3: Q,K,V,O norms=[16.121 15.554 13.498 13.391] QK norm=4.103 top=[1.066 0.993 0.907 0.877 0.822 0.797] sr=14.81 symfrac=0.706 trcos=-0.0184; OV norm=24.505 top=[6.18 5.8 5.539 4.932] sr=15.72 trcos=-0.1037 L2H4: Q,K,V,O norms=[16.876 16.188 13.241 13.143] QK norm=4.603 top=[1.188 1.105 1.077 1.014 0.915 0.895] sr=15.02 symfrac=0.704 trcos=-0.0102; OV norm=23.279 top=[5.07 4.774 4.66 4.505] sr=21.08 trcos=-0.0998 L2H5: Q,K,V,O norms=[16.445 15.839 13.339 13.161] QK norm=4.419 top=[1.201 1.063 0.969 0.943 0.934 0.905] sr=13.55 symfrac=0.704 trcos=-0.0148; OV norm=23.342 top=[4.84 4.693 4.49 4.483] sr=23.25 trcos=-0.1110 L3H0: Q,K,V,O norms=[15.747 15.258 14.874 14.87 ] QK norm=3.905 top=[0.79 0.738 0.734 0.707 0.695 0.688] sr=24.41 symfrac=0.725 trcos=0.0782; OV norm=29.683 top=[8.384 7.583 6.42 6.247] sr=12.54 trcos=-0.1253 L3H1: Q,K,V,O norms=[14.22 14.197 17.885 18.331] QK norm=3.224 top=[0.571 0.554 0.552 0.536 0.53 0.525] sr=31.82 symfrac=0.778 trcos=0.1765; OV norm=53.398 top=[13.854 13.612 13.378 12.359] sr=14.86 trcos=-0.0670 L3H2: Q,K,V,O norms=[12.743 12.446 15.523 15.585] QK norm=2.630 top=[0.808 0.724 0.627 0.583 0.54 0.503] sr=10.60 symfrac=0.707 trcos=0.0077; OV norm=35.243 top=[11.891 9.927 8.462 8.285] sr=8.78 trcos=-0.0653 L3H3: Q,K,V,O norms=[13.283 13.057 15.732 16.118] QK norm=2.794 top=[0.856 0.707 0.587 0.539 0.517 0.511] sr=10.64 symfrac=0.702 trcos=0.0035; OV norm=35.995 top=[12.434 10.397 9.009 8.061] sr=8.38 trcos=-0.0739 L3H4: Q,K,V,O norms=[14.356 13.874 15.175 15.3 ] QK norm=3.249 top=[0.919 0.738 0.695 0.661 0.629 0.593] sr=12.50 symfrac=0.707 trcos=0.0253; OV norm=31.778 top=[9.526 8.963 7.011 6.951] sr=11.13 trcos=-0.0709 L3H5: Q,K,V,O norms=[ 9.514 9.231 17.993 18.512] QK norm=1.481 top=[0.483 0.419 0.374 0.349 0.336 0.292] sr=9.40 symfrac=0.716 trcos=0.0060; OV norm=59.294 top=[21.979 19.201 17.421 16.747] sr=7.28 trcos=-0.0517 ```