OUTPUT of run 3 of at most 100: ``` DETAILED SINGULAR VALUES W_q.0 first 80: [4.368 3.131 3.099 3.042 2.994 2.951 2.926 2.881 2.859 2.857 2.804 2.766 2.762 2.692 2.683 2.661 2.637 2.588 2.539 2.509 2.458 2.433 2.403 2.299 2.272 2.173 2.098 2.048 1.954 1.887 1.877 1.865 1.738 1.725 1.665 1.605 1.552 1.535 1.476 1.424 1.378 1.341 1.306 1.264 1.24 1.213 1.154 1.122 1.109 1.068 1.053 1.031 1.017 1.007 0.985 0.957 0.944 0.913 0.888 0.879 0.868 0.864 0.837 0.822 0.795 0.789 0.778 0.769 0.765 0.754 0.742 0.731 0.72 0.711 0.704 0.697 0.691 0.682 0.679 0.672] at indices 96,128,192,256,320,383: [0.572 0.467 0.32 0.205 0.1 0.004] W_k.0 first 80: [4.163 3.952 3.467 3.427 3.364 3.335 3.279 3.256 3.214 3.163 3.115 3.029 2.977 2.901 2.881 2.832 2.798 2.76 2.658 2.584 2.528 2.449 2.353 2.301 2.281 2.206 2.121 2.074 1.917 1.896 1.813 1.753 1.713 1.676 1.573 1.53 1.47 1.431 1.415 1.355 1.3 1.26 1.187 1.169 1.161 1.116 1.108 1.104 1.072 1.032 1.003 0.988 0.965 0.953 0.931 0.911 0.907 0.885 0.873 0.866 0.85 0.841 0.834 0.815 0.792 0.782 0.768 0.749 0.739 0.731 0.726 0.725 0.723 0.716 0.71 0.698 0.69 0.689 0.679 0.666] at indices 96,128,192,256,320,383: [0.585 0.484 0.339 0.215 0.108 0.001] W_v.0 first 80: [5.208 3.364 3.006 2.962 2.815 2.639 2.527 2.46 2.37 2.301 2.25 2.23 2.151 2.135 2.044 2.039 2.001 1.981 1.91 1.875 1.822 1.773 1.749 1.708 1.634 1.611 1.556 1.487 1.453 1.402 1.268 1.213 1.187 1.13 1.109 1.08 1.026 0.992 0.987 0.959 0.933 0.931 0.908 0.892 0.887 0.866 0.864 0.853 0.843 0.827 0.811 0.793 0.783 0.781 0.759 0.748 0.721 0.715 0.71 0.701 0.698 0.675 0.663 0.659 0.652 0.647 0.64 0.626 0.626 0.619 0.614 0.607 0.6 0.596 0.593 0.589 0.585 0.584 0.579 0.573] at indices 96,128,192,256,320,383: [0.515 0.435 0.306 0.195 0.098 0.001] W_fc.0 first 80: [18.118 12.76 11.337 9.323 9.267 9.021 8.898 8.817 8.609 8.4 8.189 8.06 7.819 7.724 7.62 7.543 7.226 7.081 6.979 6.833 6.779 6.682 6.592 6.402 6.39 6.306 6.207 6.18 5.974 5.864 5.789 5.741 5.646 5.579 5.512 5.353 5.196 5.193 5.122 5.053 5.029 4.849 4.646 4.573 4.452 4.163 4.133 3.899 3.888 3.557 3.455 3.347 3.241 3.008 2.921 2.833 2.796 2.657 2.612 2.555 2.418 2.381 2.33 2.3 2.276 2.213 2.162 2.127 2.062 2.002 1.983 1.968 1.927 1.88 1.851 1.832 1.806 1.777 1.766 1.741] at indices 96,128,192,256,320,383: [1.496 1.236 0.95 0.766 0.613 0.433] W_proj.0 first 80: [10.899 7.702 7.173 6.638 6.541 6.288 6.201 6.133 5.941 5.917 5.758 5.641 5.552 5.476 5.393 5.363 5.259 5.206 5.171 5.153 5.057 5.018 4.96 4.88 4.821 4.788 4.77 4.73 4.643 4.579 4.546 4.529 4.505 4.493 4.44 4.415 4.369 4.315 4.293 4.261 4.243 4.185 4.134 4.104 4.064 4.049 4.026 3.988 3.967 3.95 3.933 3.901 3.857 3.804 3.778 3.754 3.74 3.713 3.692 3.683 3.652 3.612 3.607 3.575 3.564 3.522 3.509 3.497 3.47 3.456 3.446 3.417 3.4 3.378 3.337 3.302 3.291 3.269 3.238 3.23 ] at indices 96,128,192,256,320,383: [2.883 2.339 1.479 0.935 0.586 0.321] W_fc.1 first 80: [19.829 11.575 11.522 11.262 11.059 10.875 10.632 10.497 10.363 10.3 10.174 10.05 9.875 9.759 9.703 9.671 9.42 9.191 9.065 8.835 8.81 8.705 8.595 8.58 8.435 8.206 8.123 8.078 7.899 7.826 7.758 7.676 7.635 7.592 7.463 7.395 7.33 7.244 7.185 7.132 7.062 6.82 6.764 6.645 6.591 6.568 6.514 6.48 6.426 6.346 6.271 6.216 6.192 6.148 6.1 6.035 5.947 5.81 5.766 5.762 5.715 5.68 5.646 5.578 5.535 5.439 5.411 5.362 5.325 5.27 5.205 5.152 5.104 5.083 5.02 5. 4.974 4.903 4.884 4.838] at indices 96,128,192,256,320,383: [4.226 3.295 1.966 1.259 0.912 0.551] W_proj.3 first 80: [28.461 22.214 21.34 20.708 19.943 19.662 18.986 18.744 18.295 17.457 17.132 16.561 15.329 14.874 14.328 13.438 12.667 12.455 12.317 11.531 11.047 9.084 8.353 8.24 6.37 5.836 4.626 4.583 3.777 3.61 3.252 3.183 3.086 2.941 2.865 2.801 2.639 2.564 2.538 2.466 2.419 2.404 2.346 2.31 2.246 2.227 2.155 2.147 2.089 2.069 2.065 2.041 1.975 1.936 1.921 1.917 1.857 1.848 1.83 1.82 1.798 1.779 1.764 1.737 1.729 1.719 1.709 1.692 1.68 1.661 1.645 1.635 1.619 1.594 1.589 1.583 1.565 1.55 1.541 1.52 ] at indices 96,128,192,256,320,383: [1.36 1.117 0.821 0.626 0.477 0.321] W_fc.3 first 80: [15.114 8.611 8.486 8.073 7.769 7.529 7.449 7.334 7.094 6.971 6.891 6.808 6.601 6.537 6.454 6.331 6.294 6.232 6.162 6.115 6.058 6.034 5.926 5.878 5.85 5.803 5.786 5.751 5.723 5.713 5.669 5.644 5.591 5.547 5.529 5.512 5.494 5.466 5.454 5.444 5.4 5.348 5.344 5.325 5.294 5.275 5.253 5.237 5.214 5.193 5.186 5.17 5.139 5.13 5.098 5.091 5.061 5.059 5.046 5.036 5.014 5.001 4.972 4.962 4.951 4.912 4.906 4.885 4.869 4.859 4.837 4.808 4.787 4.778 4.77 4.764 4.75 4.736 4.722 4.7 ] at indices 96,128,192,256,320,383: [4.491 4.135 3.521 2.982 2.445 0.818] L0 HEAD Q/K/V SINGULAR VALUES H0 W_q: [2.679 2.108 1.778 1.572 1.462 1.364 1.345 1.302 1.245 1.219 1.192 1.16 ] tail8: [0.335 0.33 0.324 0.315 0.307 0.299 0.284 0.278] H0 W_k: [2.798 2.259 1.759 1.613 1.568 1.472 1.427 1.373 1.327 1.296 1.283 1.218] tail8: [0.345 0.342 0.335 0.321 0.314 0.307 0.287 0.276] H0 W_v: [1.941 1.825 1.681 1.437 1.343 1.314 1.22 1.165 1.151 1.123 1.078 1.013] tail8: [0.286 0.278 0.27 0.264 0.259 0.254 0.251 0.239] H1 W_q: [2.424 1.574 1.242 0.96 0.905 0.802 0.687 0.587 0.566 0.527 0.51 0.509] tail8: [0.239 0.234 0.229 0.224 0.218 0.21 0.207 0.198] H1 W_k: [2.798 1.612 1.417 1.068 0.886 0.857 0.784 0.602 0.579 0.569 0.54 0.528] tail8: [0.263 0.262 0.255 0.247 0.244 0.241 0.233 0.225] H1 W_v: [1.826 1.676 1.062 0.965 0.815 0.696 0.66 0.645 0.625 0.595 0.582 0.565] tail8: [0.264 0.252 0.25 0.242 0.239 0.227 0.225 0.219] H2 W_q: [2.046 1.194 1.147 1.051 0.764 0.701 0.601 0.571 0.535 0.522 0.516 0.482] tail8: [0.237 0.231 0.23 0.218 0.211 0.21 0.204 0.199] H2 W_k: [2.164 1.259 1.13 1.038 0.837 0.752 0.7 0.679 0.597 0.579 0.547 0.536] tail8: [0.265 0.259 0.253 0.249 0.246 0.242 0.232 0.218] H2 W_v: [1.569 1.363 0.945 0.883 0.778 0.688 0.672 0.64 0.609 0.597 0.558 0.551] tail8: [0.26 0.255 0.252 0.243 0.232 0.228 0.222 0.218] H3 W_q: [2.962 2.894 2.873 2.845 2.806 2.792 2.766 2.756 2.703 2.634 2.588 2.561] tail8: [0.573 0.556 0.532 0.519 0.497 0.474 0.443 0.352] H3 W_k: [3.362 3.238 3.209 3.181 3.099 3.086 3.024 2.999 2.951 2.923 2.856 2.765] tail8: [0.505 0.492 0.484 0.453 0.433 0.421 0.396 0.366] H3 W_v: [2.102 1.948 1.903 1.866 1.843 1.819 1.769 1.755 1.674 1.651 1.629 1.621] tail8: [0.318 0.313 0.313 0.298 0.287 0.279 0.271 0.262] H4 W_q: [2.076 1.29 1.127 0.896 0.868 0.699 0.665 0.626 0.597 0.57 0.528 0.524] tail8: [0.241 0.238 0.229 0.226 0.222 0.216 0.209 0.203] H4 W_k: [2.007 1.451 1.356 1.117 0.975 0.833 0.809 0.705 0.682 0.674 0.656 0.581] tail8: [0.275 0.274 0.268 0.262 0.257 0.249 0.246 0.23 ] H4 W_v: [1.252 1.117 0.818 0.786 0.737 0.658 0.626 0.612 0.602 0.565 0.542 0.537] tail8: [0.255 0.25 0.244 0.234 0.232 0.228 0.226 0.216] H5 W_q: [1.229 0.674 0.419 0.398 0.395 0.392 0.382 0.373 0.371 0.366 0.359 0.348] tail8: [0.191 0.188 0.186 0.179 0.176 0.173 0.171 0.166] H5 W_k: [1.181 0.612 0.509 0.503 0.482 0.467 0.463 0.449 0.446 0.442 0.428 0.427] tail8: [0.215 0.212 0.209 0.203 0.2 0.196 0.192 0.17 ] H5 W_v: [4.667 3.073 2.351 2.151 2.011 1.857 1.567 1.379 1.248 1.229 1.111 1.018] tail8: [0.283 0.281 0.276 0.275 0.269 0.265 0.264 0.256] L0 SUBSPACE OVERLAPS H0 W_q energy in V_all top27/top40= 0.323 0.423 FC top27/top40= 0.159 0.225 H3_QK top40= 0.462 H0 W_k energy in V_all top27/top40= 0.352 0.455 FC top27/top40= 0.171 0.242 H3_QK top40= 0.51 H0 W_v energy in V_all top27/top40= 0.661 0.766 FC top27/top40= 0.23 0.436 H3_QK top40= 0.151 H1 W_q energy in V_all top27/top40= 0.332 0.398 FC top27/top40= 0.125 0.155 H3_QK top40= 0.276 H1 W_k energy in V_all top27/top40= 0.308 0.428 FC top27/top40= 0.211 0.263 H3_QK top40= 0.284 H1 W_v energy in V_all top27/top40= 0.511 0.604 FC top27/top40= 0.15 0.204 H3_QK top40= 0.353 H2 W_q energy in V_all top27/top40= 0.258 0.329 FC top27/top40= 0.101 0.131 H3_QK top40= 0.254 H2 W_k energy in V_all top27/top40= 0.245 0.361 FC top27/top40= 0.167 0.226 H3_QK top40= 0.274 H2 W_v energy in V_all top27/top40= 0.43 0.549 FC top27/top40= 0.124 0.178 H3_QK top40= 0.341 H3 W_q energy in V_all top27/top40= 0.094 0.265 FC top27/top40= 0.069 0.103 H3_QK top40= 0.901 H3 W_k energy in V_all top27/top40= 0.137 0.318 FC top27/top40= 0.066 0.094 H3_QK top40= 0.922 H3 W_v energy in V_all top27/top40= 0.773 0.853 FC top27/top40= 0.406 0.663 H3_QK top40= 0.17 H4 W_q energy in V_all top27/top40= 0.243 0.304 FC top27/top40= 0.113 0.147 H3_QK top40= 0.279 H4 W_k energy in V_all top27/top40= 0.22 0.329 FC top27/top40= 0.15 0.204 H3_QK top40= 0.351 H4 W_v energy in V_all top27/top40= 0.298 0.457 FC top27/top40= 0.109 0.149 H3_QK top40= 0.279 H5 W_q energy in V_all top27/top40= 0.232 0.284 FC top27/top40= 0.094 0.124 H3_QK top40= 0.211 H5 W_k energy in V_all top27/top40= 0.191 0.26 FC top27/top40= 0.138 0.202 H3_QK top40= 0.188 H5 W_v energy in V_all top27/top40= 0.852 0.899 FC top27/top40= 0.251 0.336 H3_QK top40= 0.517 RAW QK ROTATION COMPARISON WITH L0H3 L0H0 projected_fraction=0.403 polar_correlations= [(4, np.float64(0.116)), (5, np.float64(0.098)), (3, np.float64(0.078)), (-4, np.float64(0.058))] L0H1 projected_fraction=0.215 polar_correlations= [(3, np.float64(0.067)), (4, np.float64(0.047)), (-2, np.float64(0.03)), (-1, np.float64(0.017))] L0H2 projected_fraction=0.253 polar_correlations= [(3, np.float64(0.049)), (-2, np.float64(0.041)), (4, np.float64(0.022)), (-3, np.float64(0.018))] L0H3 projected_fraction=0.963 polar_correlations= [(1, np.float64(0.784)), (2, np.float64(0.273)), (0, np.float64(0.273)), (-7, np.float64(0.15))] L0H4 projected_fraction=0.335 polar_correlations= [(3, np.float64(0.05)), (2, np.float64(-0.015)), (-7, np.float64(-0.019)), (-2, np.float64(-0.025))] L0H5 projected_fraction=0.300 polar_correlations= [(2, np.float64(0.073)), (3, np.float64(0.043)), (-6, np.float64(0.042)), (-7, np.float64(0.029))] L1H0 projected_fraction=0.024 polar_correlations= [(1, np.float64(0.08)), (3, np.float64(0.053)), (2, np.float64(0.045)), (-8, np.float64(0.033))] L1H1 projected_fraction=0.026 polar_correlations= [(1, np.float64(0.073)), (2, np.float64(0.06)), (3, np.float64(0.052)), (4, np.float64(0.036))] L1H2 projected_fraction=0.023 polar_correlations= [(3, np.float64(0.057)), (1, np.float64(0.044)), (-6, np.float64(0.038)), (-8, np.float64(0.027))] L1H3 projected_fraction=0.027 polar_correlations= [(3, np.float64(0.073)), (2, np.float64(0.072)), (-6, np.float64(0.053)), (1, np.float64(0.052))] L1H4 projected_fraction=0.022 polar_correlations= [(3, np.float64(0.042)), (1, np.float64(0.015)), (2, np.float64(0.009)), (-3, np.float64(0.007))] L1H5 projected_fraction=0.037 polar_correlations= [(1, np.float64(0.081)), (2, np.float64(0.062)), (3, np.float64(0.055)), (-6, np.float64(0.042))] L2H0 projected_fraction=0.023 polar_correlations= [(3, np.float64(0.056)), (1, np.float64(0.055)), (2, np.float64(0.048)), (-6, np.float64(0.025))] L2H1 projected_fraction=0.016 polar_correlations= [(2, np.float64(0.068)), (1, np.float64(0.033)), (-5, np.float64(0.032)), (-7, np.float64(0.028))] L2H2 projected_fraction=0.017 polar_correlations= [(3, np.float64(0.038)), (-5, np.float64(0.037)), (-6, np.float64(0.036)), (2, np.float64(0.031))] L2H3 projected_fraction=0.015 polar_correlations= [(1, np.float64(0.057)), (4, np.float64(0.036)), (3, np.float64(0.036)), (-7, np.float64(0.017))] L2H4 projected_fraction=0.017 polar_correlations= [(2, np.float64(0.07)), (-6, np.float64(0.054)), (3, np.float64(0.05)), (-7, np.float64(0.045))] L2H5 projected_fraction=0.017 polar_correlations= [(2, np.float64(0.07)), (3, np.float64(0.047)), (-6, np.float64(0.025)), (8, np.float64(0.021))] L3H0 projected_fraction=0.015 polar_correlations= [(0, np.float64(0.132)), (1, np.float64(0.072)), (8, np.float64(0.048)), (-1, np.float64(0.043))] L3H1 projected_fraction=0.017 polar_correlations= [(0, np.float64(0.048)), (-7, np.float64(0.035)), (1, np.float64(0.03)), (-5, np.float64(0.028))] L3H2 projected_fraction=0.017 polar_correlations= [(1, np.float64(0.042)), (-6, np.float64(0.014)), (4, np.float64(0.011)), (-5, np.float64(0.01))] L3H3 projected_fraction=0.016 polar_correlations= [(0, np.float64(0.376)), (1, np.float64(0.067)), (8, np.float64(0.067)), (-1, np.float64(0.052))] L3H4 projected_fraction=0.020 polar_correlations= [(0, np.float64(0.048)), (7, np.float64(0.044)), (-6, np.float64(0.042)), (1, np.float64(0.024))] L3H5 projected_fraction=0.016 polar_correlations= [(0, np.float64(0.098)), (-1, np.float64(0.048)), (1, np.float64(0.034)), (8, np.float64(0.03))] LAYER-NORM EXTREME COORDINATES ln1_w.0 [(381, 0.0352), (42, 0.0606), (95, 0.0607), (17, 0.0622), (147, 0.0673), (237, 0.0678), (376, 0.0726), (334, 0.0736)] ln1_b.0 [(312, -0.0954), (146, 0.0906), (12, 0.0856), (104, -0.0841), (289, 0.0826), (156, -0.078), (303, -0.0776), (328, 0.0762)] ln2_w.0 [(372, 0.042), (354, 0.0926), (21, 0.1141), (163, 0.1286), (99, 0.187), (165, 0.2065), (47, 0.2106), (150, 1.775)] ln2_b.0 [(99, -0.6236), (58, 0.4636), (153, 0.2132), (39, -0.202), (18, 0.1816), (29, -0.172), (383, -0.1652), (131, 0.1602)] ln1_w.1 [(52, 0.197), (184, 0.2495), (227, 0.2585), (165, 0.2647), (192, 0.286), (163, 0.2875), (109, 0.2953), (39, 0.2985)] ln1_b.1 [(39, -1.5676), (58, 1.3651), (99, -1.3355), (184, -0.5617), (383, -0.4622), (21, -0.4299), (153, 0.2969), (266, 0.2773)] ln2_w.1 [(21, 0.0019), (99, 0.0568), (163, 0.2201), (58, 0.2217), (39, 0.3414), (165, 0.3562), (52, 0.4584), (242, 1.5196)] ln2_b.1 [(58, 0.417), (375, 0.392), (76, -0.3892), (39, -0.3688), (319, 0.3552), (23, -0.3401), (358, 0.3149), (123, 0.3147)] ln1_w.2 [(383, 0.3175), (184, 0.3373), (39, 0.4223), (336, 0.467), (58, 0.4775), (63, 0.4805), (266, 0.4913), (294, 0.4958)] ln1_b.2 [(99, -1.6589), (58, 1.3318), (39, -1.1991), (383, -0.507), (184, -0.5046), (21, -0.4534), (63, -0.3245), (266, 0.2991)] ln2_w.2 [(21, 0.012), (322, 1.5078), (53, 1.494), (97, 1.4847), (161, 1.4752), (374, 1.4683), (214, 1.4646), (222, 1.4615)] ln2_b.2 [(236, -0.517), (73, -0.4803), (322, 0.4489), (214, -0.4232), (97, 0.4014), (382, -0.3917), (184, -0.3832), (364, 0.3749)] ln1_w.3 [(383, 0.4078), (58, 0.4661), (39, 0.4723), (45, 0.5111), (266, 0.5241), (5, 0.541), (184, 0.5624), (200, 0.5872)] ln1_b.3 [(58, 0.8241), (99, -0.7639), (5, 0.436), (383, -0.3974), (266, 0.3425), (21, -0.3152), (184, -0.2889), (45, 0.2888)] ln2_w.3 [(21, 0.2328), (240, 1.4532), (368, 1.4452), (86, 1.4332), (301, 1.4273), (364, 1.4147), (339, 1.4136), (22, 1.4)] ln2_b.3 [(21, -0.5402), (275, 0.3932), (215, -0.352), (382, -0.3297), (364, 0.3187), (168, 0.3137), (86, -0.3042), (22, -0.3012)] ```