Publish qwen3.8-27b-custom-slq-2026-10-21 (ear099)
28eb87c verified 11 days ago
238 kB
<svg xmlns="http://www.w3.org/2000/svg" width="2130" height="803" viewBox="0 0 2130 803" role="img" aria-labelledby="title description">
<title id="title">Qwen3.8-27B: precision allocation, 99% EAR profile</title>
<desc id="description">Stored bitwidth for 496 individual projections across 64 transformer blocks and 12 projection types. Cell numbers are exported storage widths: 2 through 8, or 16 for BF16. Grey cells indicate absent projections.</desc>
<metadata >{&quot;target&quot;: &quot;ear099&quot;, &quot;allocation_sha256&quot;: &quot;729f291f40faae8402d429471bf6df21d0c7399b667a29c661bc5bb634b3d26d&quot;, &quot;projection_count&quot;: 496, &quot;display&quot;: &quot;stored_bitwidth_only&quot;}</metadata>
<style >text{font-family:DejaVu Sans,Arial,sans-serif;font-variant-numeric:tabular-nums}</style>
<rect x="0" y="0" width="2130" height="803" fill="#ffffff"/>
<rect x="0" y="0" width="2130" height="8" fill="#167f83"/>
<text x="55" y="75" font-size="37" fill="#183342" font-weight="700" text-anchor="start">Qwen3.8-27B · precision allocation · 99% EAR profile</text>
<text x="55" y="137" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Stored bitwidth</text>
<rect x="250" y="114" width="61" height="34" fill="#e5f3ef" rx="3"/>
<text x="280" y="137" font-size="16" fill="#183342" font-weight="650" text-anchor="middle">2</text>
<rect x="326" y="114" width="61" height="34" fill="#cbe6df" rx="3"/>
<text x="356" y="137" font-size="16" fill="#183342" font-weight="650" text-anchor="middle">3</text>
<rect x="402" y="114" width="61" height="34" fill="#a1d3c4" rx="3"/>
<text x="432" y="137" font-size="16" fill="#183342" font-weight="650" text-anchor="middle">4</text>
<rect x="478" y="114" width="61" height="34" fill="#75b7ae" rx="3"/>
<text x="508" y="137" font-size="16" fill="#183342" font-weight="650" text-anchor="middle">5</text>
<rect x="554" y="114" width="61" height="34" fill="#4f9998" rx="3"/>
<text x="584" y="137" font-size="16" fill="#ffffff" font-weight="650" text-anchor="middle">6</text>
<rect x="630" y="114" width="61" height="34" fill="#2e7f83" rx="3"/>
<text x="660" y="137" font-size="16" fill="#ffffff" font-weight="650" text-anchor="middle">7</text>
<rect x="706" y="114" width="61" height="34" fill="#185b69" rx="3"/>
<text x="736" y="137" font-size="16" fill="#ffffff" font-weight="650" text-anchor="middle">8</text>
<rect x="782" y="114" width="61" height="34" fill="#485365" rx="3"/>
<text x="812" y="137" font-size="16" fill="#ffffff" font-weight="650" text-anchor="middle">BF16</text>
<rect x="900" y="114" width="30" height="34" fill="#e8edf0" rx="3"/>
<text x="944" y="137" font-size="18" fill="#536b78" font-weight="400" text-anchor="start">Absent projection</text>
<text x="390" y="183" font-size="13" fill="#536b78" font-weight="650" text-anchor="start">TRANSFORMER BLOCK DEPTH →   (zero-based)</text>
<text x="403.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">0</text>
<text x="429.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">1</text>
<text x="455.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">2</text>
<text x="481.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">3</text>
<text x="507.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">4</text>
<text x="533.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">5</text>
<text x="559.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">6</text>
<text x="585.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">7</text>
<text x="611.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">8</text>
<text x="637.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">9</text>
<text x="663.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">10</text>
<text x="689.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">11</text>
<text x="715.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">12</text>
<text x="741.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">13</text>
<text x="767.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">14</text>
<text x="793.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">15</text>
<text x="819.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">16</text>
<text x="845.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">17</text>
<text x="871.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">18</text>
<text x="897.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">19</text>
<text x="923.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">20</text>
<text x="949.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">21</text>
<text x="975.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">22</text>
<text x="1001.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">23</text>
<text x="1027.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">24</text>
<text x="1053.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">25</text>
<text x="1079.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">26</text>
<text x="1105.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">27</text>
<text x="1131.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">28</text>
<text x="1157.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">29</text>
<text x="1183.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">30</text>
<text x="1209.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">31</text>
<text x="1235.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">32</text>
<text x="1261.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">33</text>
<text x="1287.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">34</text>
<text x="1313.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">35</text>
<text x="1339.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">36</text>
<text x="1365.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">37</text>
<text x="1391.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">38</text>
<text x="1417.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">39</text>
<text x="1443.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">40</text>
<text x="1469.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">41</text>
<text x="1495.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">42</text>
<text x="1521.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">43</text>
<text x="1547.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">44</text>
<text x="1573.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">45</text>
<text x="1599.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">46</text>
<text x="1625.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">47</text>
<text x="1651.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">48</text>
<text x="1677.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">49</text>
<text x="1703.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">50</text>
<text x="1729.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">51</text>
<text x="1755.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">52</text>
<text x="1781.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">53</text>
<text x="1807.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">54</text>
<text x="1833.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">55</text>
<text x="1859.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">56</text>
<text x="1885.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">57</text>
<text x="1911.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">58</text>
<text x="1937.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">59</text>
<text x="1963.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">60</text>
<text x="1989.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">61</text>
<text x="2015.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">62</text>
<text x="2041.0" y="208" font-size="11" fill="#536b78" font-weight="500" text-anchor="middle">63</text>
<text x="55" y="238" font-size="19" fill="#183342" font-weight="650" text-anchor="start">MLP · gate</text>
<text x="55" y="255" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">gate_proj</text>
<g data-layer="model.language_model.layers.0.mlp.gate_proj" data-stored-bits="5">
<title >model.language_model.layers.0.mlp.gate_proj: 5-bit storage</title>
<rect x="390.8" y="218.8" width="24.4" height="42.4" fill="#75b7ae" rx="1.5"/>
<text x="403.0" y="246" font-size="16" fill="#183342" font-weight="700" text-anchor="middle">5</text>
</g>
<g data-layer="model.language_model.layers.1.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.1.mlp.gate_proj: 7-bit storage</title>
<rect x="416.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="429.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.2.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.2.mlp.gate_proj: 7-bit storage</title>
<rect x="442.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="455.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.3.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.3.mlp.gate_proj: 7-bit storage</title>
<rect x="468.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="481.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.4.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.4.mlp.gate_proj: 7-bit storage</title>
<rect x="494.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="507.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.5.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.5.mlp.gate_proj: 7-bit storage</title>
<rect x="520.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="533.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.6.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.6.mlp.gate_proj: 7-bit storage</title>
<rect x="546.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="559.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.7.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.7.mlp.gate_proj: 7-bit storage</title>
<rect x="572.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="585.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.8.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.8.mlp.gate_proj: 7-bit storage</title>
<rect x="598.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="611.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.9.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.9.mlp.gate_proj: 7-bit storage</title>
<rect x="624.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="637.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.10.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.10.mlp.gate_proj: 7-bit storage</title>
<rect x="650.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="663.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.11.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.11.mlp.gate_proj: 7-bit storage</title>
<rect x="676.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="689.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.12.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.12.mlp.gate_proj: 7-bit storage</title>
<rect x="702.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="715.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.13.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.13.mlp.gate_proj: 7-bit storage</title>
<rect x="728.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="741.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.14.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.14.mlp.gate_proj: 7-bit storage</title>
<rect x="754.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="767.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.15.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.15.mlp.gate_proj: 7-bit storage</title>
<rect x="780.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="793.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.16.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.16.mlp.gate_proj: 7-bit storage</title>
<rect x="806.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="819.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.17.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.17.mlp.gate_proj: 8-bit storage</title>
<rect x="832.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="845.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.18.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.18.mlp.gate_proj: 8-bit storage</title>
<rect x="858.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="871.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.19.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.19.mlp.gate_proj: 8-bit storage</title>
<rect x="884.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="897.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.20.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.20.mlp.gate_proj: 8-bit storage</title>
<rect x="910.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="923.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.21.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.21.mlp.gate_proj: 7-bit storage</title>
<rect x="936.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="949.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.22.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.22.mlp.gate_proj: 8-bit storage</title>
<rect x="962.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="975.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.23.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.23.mlp.gate_proj: 8-bit storage</title>
<rect x="988.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1001.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.24.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.24.mlp.gate_proj: 7-bit storage</title>
<rect x="1014.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1027.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.25.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.25.mlp.gate_proj: 7-bit storage</title>
<rect x="1040.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1053.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.26.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.26.mlp.gate_proj: 8-bit storage</title>
<rect x="1066.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1079.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.27.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.27.mlp.gate_proj: 7-bit storage</title>
<rect x="1092.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1105.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.28.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.28.mlp.gate_proj: 7-bit storage</title>
<rect x="1118.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1131.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.29.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.29.mlp.gate_proj: 7-bit storage</title>
<rect x="1144.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1157.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.30.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.30.mlp.gate_proj: 7-bit storage</title>
<rect x="1170.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1183.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.31.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.31.mlp.gate_proj: 7-bit storage</title>
<rect x="1196.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1209.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.32.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.32.mlp.gate_proj: 7-bit storage</title>
<rect x="1222.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1235.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.33.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.33.mlp.gate_proj: 7-bit storage</title>
<rect x="1248.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1261.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.34.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.34.mlp.gate_proj: 7-bit storage</title>
<rect x="1274.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1287.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.35.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.35.mlp.gate_proj: 7-bit storage</title>
<rect x="1300.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1313.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.36.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.36.mlp.gate_proj: 7-bit storage</title>
<rect x="1326.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1339.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.37.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.37.mlp.gate_proj: 7-bit storage</title>
<rect x="1352.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1365.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.38.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.38.mlp.gate_proj: 7-bit storage</title>
<rect x="1378.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1391.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.39.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.39.mlp.gate_proj: 7-bit storage</title>
<rect x="1404.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1417.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.40.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.40.mlp.gate_proj: 7-bit storage</title>
<rect x="1430.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1443.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.41.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.41.mlp.gate_proj: 8-bit storage</title>
<rect x="1456.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1469.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.42.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.42.mlp.gate_proj: 7-bit storage</title>
<rect x="1482.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1495.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.43.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.43.mlp.gate_proj: 7-bit storage</title>
<rect x="1508.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1521.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.44.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.44.mlp.gate_proj: 8-bit storage</title>
<rect x="1534.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1547.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.45.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.45.mlp.gate_proj: 7-bit storage</title>
<rect x="1560.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1573.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.46.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.46.mlp.gate_proj: 7-bit storage</title>
<rect x="1586.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1599.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.47.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.47.mlp.gate_proj: 7-bit storage</title>
<rect x="1612.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1625.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.48.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.48.mlp.gate_proj: 8-bit storage</title>
<rect x="1638.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1651.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.49.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.49.mlp.gate_proj: 8-bit storage</title>
<rect x="1664.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1677.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.50.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.50.mlp.gate_proj: 8-bit storage</title>
<rect x="1690.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1703.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.51.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.51.mlp.gate_proj: 8-bit storage</title>
<rect x="1716.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1729.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.52.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.52.mlp.gate_proj: 8-bit storage</title>
<rect x="1742.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1755.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.53.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.53.mlp.gate_proj: 8-bit storage</title>
<rect x="1768.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1781.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.54.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.54.mlp.gate_proj: 8-bit storage</title>
<rect x="1794.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1807.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.55.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.55.mlp.gate_proj: 7-bit storage</title>
<rect x="1820.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1833.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.56.mlp.gate_proj" data-stored-bits="7">
<title >model.language_model.layers.56.mlp.gate_proj: 7-bit storage</title>
<rect x="1846.8" y="218.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1859.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.57.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.57.mlp.gate_proj: 8-bit storage</title>
<rect x="1872.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1885.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.58.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.58.mlp.gate_proj: 8-bit storage</title>
<rect x="1898.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1911.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.59.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.59.mlp.gate_proj: 8-bit storage</title>
<rect x="1924.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1937.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.60.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.60.mlp.gate_proj: 8-bit storage</title>
<rect x="1950.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1963.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.61.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.61.mlp.gate_proj: 8-bit storage</title>
<rect x="1976.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1989.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.62.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.62.mlp.gate_proj: 8-bit storage</title>
<rect x="2002.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2015.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.63.mlp.gate_proj" data-stored-bits="8">
<title >model.language_model.layers.63.mlp.gate_proj: 8-bit storage</title>
<rect x="2028.8" y="218.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2041.0" y="246" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<text x="55" y="282" font-size="19" fill="#183342" font-weight="650" text-anchor="start">MLP · up</text>
<text x="55" y="299" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">up_proj</text>
<g data-layer="model.language_model.layers.0.mlp.up_proj" data-stored-bits="5">
<title >model.language_model.layers.0.mlp.up_proj: 5-bit storage</title>
<rect x="390.8" y="262.8" width="24.4" height="42.4" fill="#75b7ae" rx="1.5"/>
<text x="403.0" y="290" font-size="16" fill="#183342" font-weight="700" text-anchor="middle">5</text>
</g>
<g data-layer="model.language_model.layers.1.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.1.mlp.up_proj: 7-bit storage</title>
<rect x="416.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="429.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.2.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.2.mlp.up_proj: 7-bit storage</title>
<rect x="442.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="455.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.3.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.3.mlp.up_proj: 7-bit storage</title>
<rect x="468.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="481.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.4.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.4.mlp.up_proj: 7-bit storage</title>
<rect x="494.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="507.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.5.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.5.mlp.up_proj: 7-bit storage</title>
<rect x="520.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="533.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.6.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.6.mlp.up_proj: 7-bit storage</title>
<rect x="546.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="559.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.7.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.7.mlp.up_proj: 7-bit storage</title>
<rect x="572.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="585.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.8.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.8.mlp.up_proj: 7-bit storage</title>
<rect x="598.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="611.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.9.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.9.mlp.up_proj: 7-bit storage</title>
<rect x="624.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="637.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.10.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.10.mlp.up_proj: 7-bit storage</title>
<rect x="650.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="663.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.11.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.11.mlp.up_proj: 7-bit storage</title>
<rect x="676.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="689.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.12.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.12.mlp.up_proj: 7-bit storage</title>
<rect x="702.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="715.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.13.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.13.mlp.up_proj: 7-bit storage</title>
<rect x="728.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="741.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.14.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.14.mlp.up_proj: 7-bit storage</title>
<rect x="754.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="767.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.15.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.15.mlp.up_proj: 7-bit storage</title>
<rect x="780.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="793.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.16.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.16.mlp.up_proj: 7-bit storage</title>
<rect x="806.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="819.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.17.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.17.mlp.up_proj: 8-bit storage</title>
<rect x="832.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="845.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.18.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.18.mlp.up_proj: 8-bit storage</title>
<rect x="858.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="871.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.19.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.19.mlp.up_proj: 8-bit storage</title>
<rect x="884.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="897.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.20.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.20.mlp.up_proj: 8-bit storage</title>
<rect x="910.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="923.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.21.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.21.mlp.up_proj: 7-bit storage</title>
<rect x="936.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="949.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.22.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.22.mlp.up_proj: 8-bit storage</title>
<rect x="962.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="975.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.23.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.23.mlp.up_proj: 8-bit storage</title>
<rect x="988.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1001.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.24.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.24.mlp.up_proj: 7-bit storage</title>
<rect x="1014.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1027.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.25.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.25.mlp.up_proj: 7-bit storage</title>
<rect x="1040.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1053.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.26.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.26.mlp.up_proj: 8-bit storage</title>
<rect x="1066.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1079.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.27.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.27.mlp.up_proj: 7-bit storage</title>
<rect x="1092.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1105.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.28.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.28.mlp.up_proj: 7-bit storage</title>
<rect x="1118.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1131.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.29.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.29.mlp.up_proj: 7-bit storage</title>
<rect x="1144.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1157.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.30.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.30.mlp.up_proj: 7-bit storage</title>
<rect x="1170.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1183.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.31.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.31.mlp.up_proj: 7-bit storage</title>
<rect x="1196.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1209.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.32.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.32.mlp.up_proj: 7-bit storage</title>
<rect x="1222.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1235.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.33.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.33.mlp.up_proj: 7-bit storage</title>
<rect x="1248.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1261.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.34.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.34.mlp.up_proj: 7-bit storage</title>
<rect x="1274.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1287.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.35.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.35.mlp.up_proj: 7-bit storage</title>
<rect x="1300.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1313.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.36.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.36.mlp.up_proj: 7-bit storage</title>
<rect x="1326.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1339.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.37.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.37.mlp.up_proj: 7-bit storage</title>
<rect x="1352.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1365.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.38.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.38.mlp.up_proj: 7-bit storage</title>
<rect x="1378.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1391.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.39.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.39.mlp.up_proj: 7-bit storage</title>
<rect x="1404.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1417.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.40.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.40.mlp.up_proj: 7-bit storage</title>
<rect x="1430.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1443.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.41.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.41.mlp.up_proj: 8-bit storage</title>
<rect x="1456.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1469.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.42.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.42.mlp.up_proj: 7-bit storage</title>
<rect x="1482.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1495.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.43.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.43.mlp.up_proj: 7-bit storage</title>
<rect x="1508.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1521.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.44.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.44.mlp.up_proj: 8-bit storage</title>
<rect x="1534.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1547.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.45.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.45.mlp.up_proj: 7-bit storage</title>
<rect x="1560.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1573.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.46.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.46.mlp.up_proj: 7-bit storage</title>
<rect x="1586.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1599.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.47.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.47.mlp.up_proj: 7-bit storage</title>
<rect x="1612.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1625.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.48.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.48.mlp.up_proj: 8-bit storage</title>
<rect x="1638.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1651.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.49.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.49.mlp.up_proj: 8-bit storage</title>
<rect x="1664.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1677.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.50.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.50.mlp.up_proj: 8-bit storage</title>
<rect x="1690.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1703.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.51.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.51.mlp.up_proj: 8-bit storage</title>
<rect x="1716.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1729.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.52.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.52.mlp.up_proj: 8-bit storage</title>
<rect x="1742.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1755.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.53.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.53.mlp.up_proj: 8-bit storage</title>
<rect x="1768.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1781.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.54.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.54.mlp.up_proj: 8-bit storage</title>
<rect x="1794.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1807.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.55.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.55.mlp.up_proj: 7-bit storage</title>
<rect x="1820.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1833.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.56.mlp.up_proj" data-stored-bits="7">
<title >model.language_model.layers.56.mlp.up_proj: 7-bit storage</title>
<rect x="1846.8" y="262.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1859.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.57.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.57.mlp.up_proj: 8-bit storage</title>
<rect x="1872.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1885.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.58.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.58.mlp.up_proj: 8-bit storage</title>
<rect x="1898.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1911.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.59.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.59.mlp.up_proj: 8-bit storage</title>
<rect x="1924.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1937.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.60.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.60.mlp.up_proj: 8-bit storage</title>
<rect x="1950.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1963.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.61.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.61.mlp.up_proj: 8-bit storage</title>
<rect x="1976.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1989.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.62.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.62.mlp.up_proj: 8-bit storage</title>
<rect x="2002.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2015.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.63.mlp.up_proj" data-stored-bits="8">
<title >model.language_model.layers.63.mlp.up_proj: 8-bit storage</title>
<rect x="2028.8" y="262.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2041.0" y="290" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<text x="55" y="326" font-size="19" fill="#183342" font-weight="650" text-anchor="start">MLP · down</text>
<text x="55" y="343" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">down_proj</text>
<g data-layer="model.language_model.layers.0.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.0.mlp.down_proj: 8-bit storage</title>
<rect x="390.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="403.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.1.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.1.mlp.down_proj: 7-bit storage</title>
<rect x="416.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="429.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.2.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.2.mlp.down_proj: 7-bit storage</title>
<rect x="442.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="455.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.3.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.3.mlp.down_proj: 8-bit storage</title>
<rect x="468.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="481.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.4.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.4.mlp.down_proj: 7-bit storage</title>
<rect x="494.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="507.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.5.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.5.mlp.down_proj: 8-bit storage</title>
<rect x="520.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="533.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.6.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.6.mlp.down_proj: 8-bit storage</title>
<rect x="546.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="559.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.7.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.7.mlp.down_proj: 7-bit storage</title>
<rect x="572.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="585.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.8.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.8.mlp.down_proj: 8-bit storage</title>
<rect x="598.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="611.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.9.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.9.mlp.down_proj: 7-bit storage</title>
<rect x="624.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="637.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.10.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.10.mlp.down_proj: 8-bit storage</title>
<rect x="650.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="663.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.11.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.11.mlp.down_proj: 8-bit storage</title>
<rect x="676.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="689.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.12.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.12.mlp.down_proj: 7-bit storage</title>
<rect x="702.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="715.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.13.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.13.mlp.down_proj: 8-bit storage</title>
<rect x="728.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="741.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.14.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.14.mlp.down_proj: 7-bit storage</title>
<rect x="754.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="767.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.15.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.15.mlp.down_proj: 7-bit storage</title>
<rect x="780.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="793.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.16.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.16.mlp.down_proj: 7-bit storage</title>
<rect x="806.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="819.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.17.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.17.mlp.down_proj: 8-bit storage</title>
<rect x="832.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="845.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.18.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.18.mlp.down_proj: 8-bit storage</title>
<rect x="858.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="871.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.19.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.19.mlp.down_proj: 8-bit storage</title>
<rect x="884.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="897.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.20.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.20.mlp.down_proj: 8-bit storage</title>
<rect x="910.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="923.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.21.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.21.mlp.down_proj: 8-bit storage</title>
<rect x="936.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="949.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.22.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.22.mlp.down_proj: 8-bit storage</title>
<rect x="962.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="975.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.23.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.23.mlp.down_proj: 8-bit storage</title>
<rect x="988.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1001.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.24.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.24.mlp.down_proj: 8-bit storage</title>
<rect x="1014.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1027.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.25.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.25.mlp.down_proj: 8-bit storage</title>
<rect x="1040.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1053.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.26.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.26.mlp.down_proj: 8-bit storage</title>
<rect x="1066.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1079.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.27.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.27.mlp.down_proj: 7-bit storage</title>
<rect x="1092.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1105.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.28.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.28.mlp.down_proj: 8-bit storage</title>
<rect x="1118.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1131.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.29.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.29.mlp.down_proj: 8-bit storage</title>
<rect x="1144.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1157.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.30.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.30.mlp.down_proj: 8-bit storage</title>
<rect x="1170.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1183.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.31.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.31.mlp.down_proj: 7-bit storage</title>
<rect x="1196.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1209.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.32.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.32.mlp.down_proj: 8-bit storage</title>
<rect x="1222.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1235.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.33.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.33.mlp.down_proj: 8-bit storage</title>
<rect x="1248.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1261.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.34.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.34.mlp.down_proj: 8-bit storage</title>
<rect x="1274.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1287.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.35.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.35.mlp.down_proj: 8-bit storage</title>
<rect x="1300.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1313.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.36.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.36.mlp.down_proj: 8-bit storage</title>
<rect x="1326.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1339.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.37.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.37.mlp.down_proj: 8-bit storage</title>
<rect x="1352.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1365.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.38.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.38.mlp.down_proj: 8-bit storage</title>
<rect x="1378.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1391.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.39.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.39.mlp.down_proj: 8-bit storage</title>
<rect x="1404.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1417.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.40.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.40.mlp.down_proj: 8-bit storage</title>
<rect x="1430.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1443.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.41.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.41.mlp.down_proj: 8-bit storage</title>
<rect x="1456.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1469.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.42.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.42.mlp.down_proj: 8-bit storage</title>
<rect x="1482.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1495.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.43.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.43.mlp.down_proj: 8-bit storage</title>
<rect x="1508.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1521.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.44.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.44.mlp.down_proj: 7-bit storage</title>
<rect x="1534.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1547.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.45.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.45.mlp.down_proj: 8-bit storage</title>
<rect x="1560.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1573.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.46.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.46.mlp.down_proj: 8-bit storage</title>
<rect x="1586.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1599.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.47.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.47.mlp.down_proj: 7-bit storage</title>
<rect x="1612.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1625.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.48.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.48.mlp.down_proj: 8-bit storage</title>
<rect x="1638.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1651.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.49.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.49.mlp.down_proj: 8-bit storage</title>
<rect x="1664.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1677.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.50.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.50.mlp.down_proj: 8-bit storage</title>
<rect x="1690.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1703.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.51.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.51.mlp.down_proj: 8-bit storage</title>
<rect x="1716.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1729.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.52.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.52.mlp.down_proj: 8-bit storage</title>
<rect x="1742.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1755.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.53.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.53.mlp.down_proj: 8-bit storage</title>
<rect x="1768.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1781.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.54.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.54.mlp.down_proj: 8-bit storage</title>
<rect x="1794.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1807.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.55.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.55.mlp.down_proj: 8-bit storage</title>
<rect x="1820.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1833.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.56.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.56.mlp.down_proj: 8-bit storage</title>
<rect x="1846.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1859.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.57.mlp.down_proj" data-stored-bits="7">
<title >model.language_model.layers.57.mlp.down_proj: 7-bit storage</title>
<rect x="1872.8" y="306.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1885.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.58.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.58.mlp.down_proj: 8-bit storage</title>
<rect x="1898.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1911.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.59.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.59.mlp.down_proj: 8-bit storage</title>
<rect x="1924.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1937.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.60.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.60.mlp.down_proj: 8-bit storage</title>
<rect x="1950.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1963.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.61.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.61.mlp.down_proj: 8-bit storage</title>
<rect x="1976.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1989.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.62.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.62.mlp.down_proj: 8-bit storage</title>
<rect x="2002.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2015.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.63.mlp.down_proj" data-stored-bits="8">
<title >model.language_model.layers.63.mlp.down_proj: 8-bit storage</title>
<rect x="2028.8" y="306.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2041.0" y="334" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<text x="55" y="382" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Linear attention · QKV</text>
<text x="55" y="399" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">in_proj_qkv</text>
<g data-layer="model.language_model.layers.0.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.0.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="390.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="403.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.1.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.1.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="416.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="429.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.2.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.2.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="442.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="455.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="468.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="481.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.4.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.4.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="494.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="507.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.5.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.5.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="520.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="533.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.6.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.6.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="546.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="559.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="572.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="585.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.8.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.8.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="598.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="611.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.9.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.9.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="624.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="637.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.10.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.10.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="650.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="663.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="676.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="689.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.12.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.12.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="702.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="715.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.13.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.13.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="728.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="741.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.14.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.14.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="754.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="767.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="780.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="793.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.16.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.16.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="806.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="819.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.17.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.17.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="832.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="845.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.18.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.18.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="858.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="871.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="884.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="897.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.20.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.20.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="910.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="923.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.21.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.21.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="936.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="949.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.22.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.22.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="962.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="975.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="988.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1001.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.24.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.24.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1014.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1027.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.25.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.25.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1040.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1053.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.26.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.26.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1066.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1079.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1092.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1105.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.28.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.28.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1118.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1131.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.29.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.29.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1144.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1157.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.30.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.30.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1170.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1183.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1196.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1209.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.32.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.32.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1222.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1235.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.33.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.33.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1248.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1261.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.34.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.34.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1274.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1287.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1300.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1313.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.36.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.36.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1326.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1339.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.37.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.37.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1352.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1365.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.38.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.38.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1378.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1391.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1404.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1417.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.40.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.40.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1430.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1443.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.41.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.41.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1456.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1469.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.42.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.42.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1482.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1495.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1508.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1521.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.44.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.44.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1534.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1547.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.45.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.45.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1560.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1573.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.46.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.46.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1586.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1599.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1612.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1625.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.48.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.48.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1638.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1651.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.49.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.49.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1664.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1677.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.50.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.50.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1690.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1703.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1716.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1729.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.52.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.52.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1742.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1755.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.53.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.53.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1768.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1781.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.54.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.54.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1794.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1807.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1820.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1833.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.56.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.56.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1846.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1859.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.57.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.57.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1872.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1885.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.58.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.58.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1898.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1911.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1924.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1937.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.60.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.60.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="1950.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1963.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.61.linear_attn.in_proj_qkv" data-stored-bits="7">
<title >model.language_model.layers.61.linear_attn.in_proj_qkv: 7-bit storage</title>
<rect x="1976.8" y="362.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1989.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.62.linear_attn.in_proj_qkv" data-stored-bits="8">
<title >model.language_model.layers.62.linear_attn.in_proj_qkv: 8-bit storage</title>
<rect x="2002.8" y="362.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2015.0" y="390" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="2028.8" y="362.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2041.0" y="390" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<text x="55" y="426" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Linear attention · Z</text>
<text x="55" y="443" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">in_proj_z</text>
<g data-layer="model.language_model.layers.0.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.0.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="390.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="403.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.1.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.1.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="416.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="429.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.2.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.2.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="442.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="455.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="468.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="481.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.4.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.4.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="494.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="507.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.5.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.5.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="520.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="533.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.6.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.6.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="546.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="559.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="572.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="585.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.8.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.8.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="598.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="611.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.9.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.9.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="624.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="637.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.10.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.10.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="650.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="663.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="676.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="689.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.12.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.12.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="702.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="715.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.13.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.13.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="728.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="741.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.14.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.14.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="754.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="767.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="780.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="793.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.16.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.16.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="806.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="819.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.17.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.17.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="832.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="845.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.18.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.18.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="858.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="871.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="884.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="897.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.20.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.20.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="910.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="923.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.21.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.21.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="936.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="949.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.22.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.22.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="962.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="975.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="988.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1001.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.24.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.24.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1014.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1027.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.25.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.25.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1040.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1053.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.26.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.26.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1066.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1079.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1092.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1105.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.28.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.28.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1118.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1131.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.29.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.29.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1144.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1157.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.30.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.30.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1170.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1183.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1196.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1209.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.32.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.32.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1222.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1235.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.33.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.33.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1248.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1261.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.34.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.34.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1274.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1287.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1300.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1313.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.36.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.36.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1326.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1339.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.37.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.37.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1352.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1365.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.38.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.38.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1378.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1391.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1404.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1417.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.40.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.40.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1430.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1443.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.41.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.41.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1456.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1469.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.42.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.42.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1482.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1495.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1508.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1521.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.44.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.44.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1534.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1547.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.45.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.45.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1560.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1573.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.46.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.46.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1586.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1599.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1612.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1625.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.48.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.48.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1638.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1651.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.49.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.49.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1664.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1677.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.50.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.50.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1690.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1703.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1716.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1729.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.52.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.52.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1742.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1755.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.53.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.53.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1768.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1781.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.54.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.54.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1794.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1807.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1820.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1833.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.56.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.56.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1846.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1859.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.57.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.57.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1872.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1885.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.58.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.58.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1898.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1911.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1924.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1937.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.60.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.60.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="1950.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1963.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.61.linear_attn.in_proj_z" data-stored-bits="7">
<title >model.language_model.layers.61.linear_attn.in_proj_z: 7-bit storage</title>
<rect x="1976.8" y="406.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1989.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.62.linear_attn.in_proj_z" data-stored-bits="8">
<title >model.language_model.layers.62.linear_attn.in_proj_z: 8-bit storage</title>
<rect x="2002.8" y="406.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2015.0" y="434" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="2028.8" y="406.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2041.0" y="434" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<text x="55" y="470" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Linear attention · output</text>
<text x="55" y="487" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">out_proj</text>
<g data-layer="model.language_model.layers.0.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.0.linear_attn.out_proj: 8-bit storage</title>
<rect x="390.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="403.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.1.linear_attn.out_proj" data-stored-bits="7">
<title >model.language_model.layers.1.linear_attn.out_proj: 7-bit storage</title>
<rect x="416.8" y="450.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="429.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.2.linear_attn.out_proj" data-stored-bits="7">
<title >model.language_model.layers.2.linear_attn.out_proj: 7-bit storage</title>
<rect x="442.8" y="450.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="455.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="468.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="481.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.4.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.4.linear_attn.out_proj: 8-bit storage</title>
<rect x="494.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="507.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.5.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.5.linear_attn.out_proj: 8-bit storage</title>
<rect x="520.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="533.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.6.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.6.linear_attn.out_proj: 8-bit storage</title>
<rect x="546.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="559.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="572.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="585.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.8.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.8.linear_attn.out_proj: 8-bit storage</title>
<rect x="598.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="611.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.9.linear_attn.out_proj" data-stored-bits="7">
<title >model.language_model.layers.9.linear_attn.out_proj: 7-bit storage</title>
<rect x="624.8" y="450.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="637.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.10.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.10.linear_attn.out_proj: 8-bit storage</title>
<rect x="650.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="663.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="676.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="689.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.12.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.12.linear_attn.out_proj: 8-bit storage</title>
<rect x="702.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="715.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.13.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.13.linear_attn.out_proj: 8-bit storage</title>
<rect x="728.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="741.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.14.linear_attn.out_proj" data-stored-bits="7">
<title >model.language_model.layers.14.linear_attn.out_proj: 7-bit storage</title>
<rect x="754.8" y="450.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="767.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="780.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="793.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.16.linear_attn.out_proj" data-stored-bits="7">
<title >model.language_model.layers.16.linear_attn.out_proj: 7-bit storage</title>
<rect x="806.8" y="450.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="819.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<g data-layer="model.language_model.layers.17.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.17.linear_attn.out_proj: 8-bit storage</title>
<rect x="832.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="845.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.18.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.18.linear_attn.out_proj: 8-bit storage</title>
<rect x="858.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="871.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="884.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="897.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.20.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.20.linear_attn.out_proj: 8-bit storage</title>
<rect x="910.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="923.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.21.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.21.linear_attn.out_proj: 8-bit storage</title>
<rect x="936.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="949.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.22.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.22.linear_attn.out_proj: 8-bit storage</title>
<rect x="962.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="975.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="988.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1001.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.24.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.24.linear_attn.out_proj: 8-bit storage</title>
<rect x="1014.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1027.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.25.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.25.linear_attn.out_proj: 8-bit storage</title>
<rect x="1040.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1053.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.26.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.26.linear_attn.out_proj: 8-bit storage</title>
<rect x="1066.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1079.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1092.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1105.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.28.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.28.linear_attn.out_proj: 8-bit storage</title>
<rect x="1118.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1131.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.29.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.29.linear_attn.out_proj: 8-bit storage</title>
<rect x="1144.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1157.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.30.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.30.linear_attn.out_proj: 8-bit storage</title>
<rect x="1170.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1183.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1196.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1209.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.32.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.32.linear_attn.out_proj: 8-bit storage</title>
<rect x="1222.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1235.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.33.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.33.linear_attn.out_proj: 8-bit storage</title>
<rect x="1248.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1261.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.34.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.34.linear_attn.out_proj: 8-bit storage</title>
<rect x="1274.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1287.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1300.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1313.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.36.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.36.linear_attn.out_proj: 8-bit storage</title>
<rect x="1326.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1339.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.37.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.37.linear_attn.out_proj: 8-bit storage</title>
<rect x="1352.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1365.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.38.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.38.linear_attn.out_proj: 8-bit storage</title>
<rect x="1378.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1391.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1404.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1417.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.40.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.40.linear_attn.out_proj: 8-bit storage</title>
<rect x="1430.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1443.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.41.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.41.linear_attn.out_proj: 8-bit storage</title>
<rect x="1456.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1469.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.42.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.42.linear_attn.out_proj: 8-bit storage</title>
<rect x="1482.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1495.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1508.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1521.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.44.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.44.linear_attn.out_proj: 8-bit storage</title>
<rect x="1534.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1547.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.45.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.45.linear_attn.out_proj: 8-bit storage</title>
<rect x="1560.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1573.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.46.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.46.linear_attn.out_proj: 8-bit storage</title>
<rect x="1586.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1599.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1612.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1625.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.48.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.48.linear_attn.out_proj: 8-bit storage</title>
<rect x="1638.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1651.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.49.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.49.linear_attn.out_proj: 8-bit storage</title>
<rect x="1664.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1677.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.50.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.50.linear_attn.out_proj: 8-bit storage</title>
<rect x="1690.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1703.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1716.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1729.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.52.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.52.linear_attn.out_proj: 8-bit storage</title>
<rect x="1742.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1755.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.53.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.53.linear_attn.out_proj: 8-bit storage</title>
<rect x="1768.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1781.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.54.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.54.linear_attn.out_proj: 8-bit storage</title>
<rect x="1794.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1807.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1820.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1833.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.56.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.56.linear_attn.out_proj: 8-bit storage</title>
<rect x="1846.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1859.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.57.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.57.linear_attn.out_proj: 8-bit storage</title>
<rect x="1872.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1885.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.58.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.58.linear_attn.out_proj: 8-bit storage</title>
<rect x="1898.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1911.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1924.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1937.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.60.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.60.linear_attn.out_proj: 8-bit storage</title>
<rect x="1950.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1963.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.61.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.61.linear_attn.out_proj: 8-bit storage</title>
<rect x="1976.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1989.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<g data-layer="model.language_model.layers.62.linear_attn.out_proj" data-stored-bits="8">
<title >model.language_model.layers.62.linear_attn.out_proj: 8-bit storage</title>
<rect x="2002.8" y="450.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2015.0" y="478" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="2028.8" y="450.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2041.0" y="478" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<text x="55" y="514" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Linear attention · control A</text>
<text x="55" y="531" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">in_proj_a</text>
<g data-layer="model.language_model.layers.0.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.0.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="390.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="403.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.1.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.1.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="416.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="429.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.2.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.2.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="442.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="455.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="468.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="481.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.4.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.4.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="494.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="507.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.5.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.5.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="520.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="533.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.6.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.6.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="546.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="559.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="572.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="585.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.8.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.8.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="598.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="611.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.9.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.9.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="624.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="637.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.10.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.10.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="650.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="663.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="676.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="689.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.12.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.12.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="702.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="715.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.13.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.13.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="728.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="741.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.14.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.14.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="754.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="767.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="780.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="793.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.16.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.16.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="806.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="819.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.17.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.17.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="832.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="845.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.18.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.18.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="858.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="871.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="884.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="897.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.20.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.20.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="910.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="923.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.21.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.21.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="936.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="949.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.22.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.22.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="962.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="975.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="988.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1001.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.24.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.24.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1014.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1027.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.25.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.25.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1040.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1053.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.26.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.26.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1066.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1079.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1092.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1105.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.28.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.28.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1118.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1131.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.29.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.29.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1144.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1157.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.30.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.30.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1170.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1183.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1196.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1209.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.32.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.32.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1222.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1235.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.33.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.33.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1248.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1261.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.34.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.34.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1274.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1287.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1300.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1313.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.36.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.36.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1326.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1339.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.37.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.37.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1352.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1365.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.38.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.38.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1378.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1391.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1404.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1417.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.40.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.40.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1430.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1443.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.41.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.41.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1456.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1469.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.42.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.42.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1482.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1495.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1508.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1521.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.44.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.44.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1534.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1547.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.45.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.45.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1560.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1573.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.46.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.46.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1586.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1599.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1612.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1625.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.48.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.48.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1638.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1651.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.49.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.49.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1664.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1677.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.50.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.50.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1690.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1703.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1716.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1729.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.52.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.52.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1742.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1755.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.53.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.53.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1768.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1781.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.54.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.54.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1794.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1807.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1820.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1833.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.56.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.56.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1846.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1859.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.57.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.57.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1872.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1885.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.58.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.58.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1898.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1911.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1924.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1937.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.60.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.60.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1950.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1963.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.61.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.61.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="1976.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1989.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.62.linear_attn.in_proj_a" data-stored-bits="16">
<title >model.language_model.layers.62.linear_attn.in_proj_a: 16-bit storage</title>
<rect x="2002.8" y="494.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="2015.0" y="522" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="2028.8" y="494.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2041.0" y="522" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<text x="55" y="558" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Linear attention · control B</text>
<text x="55" y="575" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">in_proj_b</text>
<g data-layer="model.language_model.layers.0.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.0.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="390.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="403.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.1.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.1.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="416.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="429.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.2.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.2.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="442.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="455.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="468.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="481.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.4.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.4.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="494.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="507.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.5.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.5.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="520.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="533.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.6.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.6.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="546.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="559.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="572.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="585.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.8.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.8.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="598.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="611.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.9.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.9.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="624.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="637.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.10.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.10.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="650.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="663.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="676.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="689.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.12.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.12.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="702.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="715.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.13.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.13.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="728.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="741.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.14.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.14.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="754.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="767.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="780.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="793.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.16.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.16.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="806.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="819.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.17.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.17.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="832.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="845.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.18.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.18.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="858.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="871.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="884.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="897.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.20.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.20.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="910.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="923.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.21.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.21.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="936.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="949.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.22.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.22.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="962.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="975.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="988.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1001.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.24.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.24.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1014.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1027.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.25.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.25.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1040.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1053.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.26.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.26.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1066.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1079.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1092.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1105.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.28.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.28.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1118.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1131.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.29.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.29.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1144.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1157.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.30.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.30.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1170.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1183.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1196.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1209.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.32.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.32.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1222.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1235.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.33.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.33.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1248.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1261.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.34.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.34.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1274.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1287.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1300.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1313.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.36.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.36.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1326.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1339.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.37.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.37.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1352.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1365.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.38.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.38.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1378.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1391.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1404.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1417.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.40.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.40.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1430.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1443.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.41.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.41.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1456.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1469.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.42.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.42.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1482.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1495.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1508.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1521.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.44.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.44.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1534.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1547.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.45.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.45.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1560.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1573.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.46.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.46.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1586.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1599.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1612.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1625.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.48.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.48.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1638.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1651.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.49.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.49.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1664.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1677.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.50.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.50.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1690.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1703.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1716.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1729.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.52.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.52.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1742.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1755.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.53.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.53.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1768.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1781.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.54.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.54.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1794.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1807.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1820.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1833.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.56.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.56.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1846.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1859.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.57.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.57.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1872.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1885.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.58.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.58.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1898.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1911.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="1924.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1937.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.60.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.60.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1950.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1963.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.61.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.61.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="1976.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="1989.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<g data-layer="model.language_model.layers.62.linear_attn.in_proj_b" data-stored-bits="16">
<title >model.language_model.layers.62.linear_attn.in_proj_b: 16-bit storage</title>
<rect x="2002.8" y="538.8" width="24.4" height="42.4" fill="#485365" rx="1.5"/>
<text x="2015.0" y="566" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">16</text>
</g>
<rect x="2028.8" y="538.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2041.0" y="566" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<text x="55" y="614" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Full attention · query</text>
<text x="55" y="631" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">q_proj</text>
<rect x="390.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="403.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="416.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="429.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="442.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="455.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.3.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.3.self_attn.q_proj: 8-bit storage</title>
<rect x="468.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="481.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="494.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="507.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="520.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="533.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="546.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="559.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.7.self_attn.q_proj" data-stored-bits="7">
<title >model.language_model.layers.7.self_attn.q_proj: 7-bit storage</title>
<rect x="572.8" y="594.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="585.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="598.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="611.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="624.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="637.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="650.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="663.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.11.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.11.self_attn.q_proj: 8-bit storage</title>
<rect x="676.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="689.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="702.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="715.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="728.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="741.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="754.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="767.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.15.self_attn.q_proj" data-stored-bits="7">
<title >model.language_model.layers.15.self_attn.q_proj: 7-bit storage</title>
<rect x="780.8" y="594.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="793.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="806.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="819.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="832.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="845.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="858.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="871.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.19.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.19.self_attn.q_proj: 8-bit storage</title>
<rect x="884.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="897.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="910.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="923.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="936.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="949.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="962.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="975.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.23.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.23.self_attn.q_proj: 8-bit storage</title>
<rect x="988.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1001.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1014.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1027.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1040.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1053.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1066.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1079.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.27.self_attn.q_proj" data-stored-bits="7">
<title >model.language_model.layers.27.self_attn.q_proj: 7-bit storage</title>
<rect x="1092.8" y="594.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1105.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1118.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1131.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1144.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1157.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1170.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1183.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.31.self_attn.q_proj" data-stored-bits="7">
<title >model.language_model.layers.31.self_attn.q_proj: 7-bit storage</title>
<rect x="1196.8" y="594.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1209.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1222.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1235.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1248.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1261.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1274.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1287.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.35.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.35.self_attn.q_proj: 8-bit storage</title>
<rect x="1300.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1313.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1326.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1339.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1352.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1365.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1378.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1391.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.39.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.39.self_attn.q_proj: 8-bit storage</title>
<rect x="1404.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1417.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1430.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1443.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1456.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1469.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1482.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1495.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.43.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.43.self_attn.q_proj: 8-bit storage</title>
<rect x="1508.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1521.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1534.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1547.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1560.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1573.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1586.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1599.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.47.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.47.self_attn.q_proj: 8-bit storage</title>
<rect x="1612.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1625.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1638.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1651.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1664.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1677.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1690.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1703.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.51.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.51.self_attn.q_proj: 8-bit storage</title>
<rect x="1716.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1729.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1742.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1755.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1768.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1781.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1794.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1807.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.55.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.55.self_attn.q_proj: 8-bit storage</title>
<rect x="1820.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1833.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1846.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1859.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1872.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1885.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1898.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1911.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.59.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.59.self_attn.q_proj: 8-bit storage</title>
<rect x="1924.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1937.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1950.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1963.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1976.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1989.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="2002.8" y="594.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2015.0" y="622" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.63.self_attn.q_proj" data-stored-bits="8">
<title >model.language_model.layers.63.self_attn.q_proj: 8-bit storage</title>
<rect x="2028.8" y="594.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2041.0" y="622" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<text x="55" y="658" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Full attention · key</text>
<text x="55" y="675" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">k_proj</text>
<rect x="390.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="403.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="416.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="429.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="442.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="455.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.3.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.3.self_attn.k_proj: 8-bit storage</title>
<rect x="468.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="481.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="494.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="507.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="520.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="533.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="546.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="559.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.7.self_attn.k_proj" data-stored-bits="7">
<title >model.language_model.layers.7.self_attn.k_proj: 7-bit storage</title>
<rect x="572.8" y="638.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="585.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="598.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="611.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="624.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="637.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="650.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="663.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.11.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.11.self_attn.k_proj: 8-bit storage</title>
<rect x="676.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="689.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="702.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="715.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="728.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="741.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="754.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="767.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.15.self_attn.k_proj" data-stored-bits="7">
<title >model.language_model.layers.15.self_attn.k_proj: 7-bit storage</title>
<rect x="780.8" y="638.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="793.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="806.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="819.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="832.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="845.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="858.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="871.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.19.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.19.self_attn.k_proj: 8-bit storage</title>
<rect x="884.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="897.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="910.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="923.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="936.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="949.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="962.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="975.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.23.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.23.self_attn.k_proj: 8-bit storage</title>
<rect x="988.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1001.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1014.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1027.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1040.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1053.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1066.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1079.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.27.self_attn.k_proj" data-stored-bits="7">
<title >model.language_model.layers.27.self_attn.k_proj: 7-bit storage</title>
<rect x="1092.8" y="638.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1105.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1118.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1131.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1144.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1157.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1170.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1183.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.31.self_attn.k_proj" data-stored-bits="7">
<title >model.language_model.layers.31.self_attn.k_proj: 7-bit storage</title>
<rect x="1196.8" y="638.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1209.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1222.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1235.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1248.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1261.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1274.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1287.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.35.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.35.self_attn.k_proj: 8-bit storage</title>
<rect x="1300.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1313.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1326.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1339.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1352.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1365.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1378.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1391.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.39.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.39.self_attn.k_proj: 8-bit storage</title>
<rect x="1404.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1417.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1430.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1443.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1456.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1469.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1482.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1495.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.43.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.43.self_attn.k_proj: 8-bit storage</title>
<rect x="1508.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1521.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1534.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1547.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1560.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1573.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1586.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1599.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.47.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.47.self_attn.k_proj: 8-bit storage</title>
<rect x="1612.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1625.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1638.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1651.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1664.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1677.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1690.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1703.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.51.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.51.self_attn.k_proj: 8-bit storage</title>
<rect x="1716.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1729.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1742.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1755.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1768.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1781.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1794.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1807.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.55.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.55.self_attn.k_proj: 8-bit storage</title>
<rect x="1820.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1833.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1846.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1859.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1872.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1885.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1898.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1911.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.59.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.59.self_attn.k_proj: 8-bit storage</title>
<rect x="1924.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1937.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1950.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1963.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1976.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1989.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="2002.8" y="638.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2015.0" y="666" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.63.self_attn.k_proj" data-stored-bits="8">
<title >model.language_model.layers.63.self_attn.k_proj: 8-bit storage</title>
<rect x="2028.8" y="638.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2041.0" y="666" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<text x="55" y="702" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Full attention · value</text>
<text x="55" y="719" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">v_proj</text>
<rect x="390.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="403.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="416.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="429.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="442.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="455.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.3.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.3.self_attn.v_proj: 8-bit storage</title>
<rect x="468.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="481.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="494.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="507.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="520.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="533.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="546.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="559.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.7.self_attn.v_proj" data-stored-bits="7">
<title >model.language_model.layers.7.self_attn.v_proj: 7-bit storage</title>
<rect x="572.8" y="682.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="585.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="598.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="611.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="624.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="637.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="650.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="663.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.11.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.11.self_attn.v_proj: 8-bit storage</title>
<rect x="676.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="689.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="702.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="715.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="728.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="741.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="754.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="767.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.15.self_attn.v_proj" data-stored-bits="7">
<title >model.language_model.layers.15.self_attn.v_proj: 7-bit storage</title>
<rect x="780.8" y="682.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="793.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="806.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="819.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="832.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="845.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="858.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="871.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.19.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.19.self_attn.v_proj: 8-bit storage</title>
<rect x="884.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="897.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="910.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="923.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="936.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="949.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="962.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="975.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.23.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.23.self_attn.v_proj: 8-bit storage</title>
<rect x="988.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1001.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1014.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1027.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1040.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1053.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1066.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1079.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.27.self_attn.v_proj" data-stored-bits="7">
<title >model.language_model.layers.27.self_attn.v_proj: 7-bit storage</title>
<rect x="1092.8" y="682.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1105.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1118.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1131.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1144.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1157.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1170.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1183.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.31.self_attn.v_proj" data-stored-bits="7">
<title >model.language_model.layers.31.self_attn.v_proj: 7-bit storage</title>
<rect x="1196.8" y="682.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1209.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1222.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1235.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1248.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1261.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1274.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1287.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.35.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.35.self_attn.v_proj: 8-bit storage</title>
<rect x="1300.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1313.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1326.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1339.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1352.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1365.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1378.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1391.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.39.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.39.self_attn.v_proj: 8-bit storage</title>
<rect x="1404.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1417.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1430.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1443.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1456.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1469.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1482.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1495.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.43.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.43.self_attn.v_proj: 8-bit storage</title>
<rect x="1508.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1521.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1534.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1547.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1560.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1573.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1586.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1599.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.47.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.47.self_attn.v_proj: 8-bit storage</title>
<rect x="1612.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1625.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1638.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1651.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1664.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1677.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1690.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1703.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.51.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.51.self_attn.v_proj: 8-bit storage</title>
<rect x="1716.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1729.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1742.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1755.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1768.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1781.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1794.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1807.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.55.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.55.self_attn.v_proj: 8-bit storage</title>
<rect x="1820.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1833.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1846.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1859.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1872.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1885.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1898.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1911.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.59.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.59.self_attn.v_proj: 8-bit storage</title>
<rect x="1924.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1937.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1950.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1963.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1976.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1989.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="2002.8" y="682.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2015.0" y="710" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.63.self_attn.v_proj" data-stored-bits="8">
<title >model.language_model.layers.63.self_attn.v_proj: 8-bit storage</title>
<rect x="2028.8" y="682.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2041.0" y="710" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<text x="55" y="746" font-size="19" fill="#183342" font-weight="650" text-anchor="start">Full attention · output</text>
<text x="55" y="763" font-size="12" fill="#536b78" font-weight="400" text-anchor="start">o_proj</text>
<rect x="390.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="403.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="416.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="429.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="442.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="455.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.3.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.3.self_attn.o_proj: 8-bit storage</title>
<rect x="468.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="481.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="494.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="507.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="520.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="533.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="546.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="559.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.7.self_attn.o_proj" data-stored-bits="7">
<title >model.language_model.layers.7.self_attn.o_proj: 7-bit storage</title>
<rect x="572.8" y="726.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="585.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="598.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="611.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="624.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="637.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="650.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="663.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.11.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.11.self_attn.o_proj: 8-bit storage</title>
<rect x="676.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="689.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="702.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="715.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="728.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="741.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="754.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="767.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.15.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.15.self_attn.o_proj: 8-bit storage</title>
<rect x="780.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="793.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="806.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="819.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="832.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="845.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="858.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="871.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.19.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.19.self_attn.o_proj: 8-bit storage</title>
<rect x="884.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="897.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="910.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="923.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="936.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="949.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="962.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="975.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.23.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.23.self_attn.o_proj: 8-bit storage</title>
<rect x="988.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1001.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1014.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1027.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1040.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1053.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1066.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1079.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.27.self_attn.o_proj" data-stored-bits="7">
<title >model.language_model.layers.27.self_attn.o_proj: 7-bit storage</title>
<rect x="1092.8" y="726.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1105.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1118.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1131.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1144.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1157.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1170.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1183.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.31.self_attn.o_proj" data-stored-bits="7">
<title >model.language_model.layers.31.self_attn.o_proj: 7-bit storage</title>
<rect x="1196.8" y="726.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1209.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1222.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1235.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1248.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1261.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1274.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1287.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.35.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.35.self_attn.o_proj: 8-bit storage</title>
<rect x="1300.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1313.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1326.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1339.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1352.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1365.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1378.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1391.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.39.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.39.self_attn.o_proj: 8-bit storage</title>
<rect x="1404.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1417.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1430.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1443.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1456.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1469.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1482.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1495.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.43.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.43.self_attn.o_proj: 8-bit storage</title>
<rect x="1508.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1521.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1534.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1547.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1560.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1573.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1586.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1599.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.47.self_attn.o_proj" data-stored-bits="7">
<title >model.language_model.layers.47.self_attn.o_proj: 7-bit storage</title>
<rect x="1612.8" y="726.8" width="24.4" height="42.4" fill="#2e7f83" rx="1.5"/>
<text x="1625.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">7</text>
</g>
<rect x="1638.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1651.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1664.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1677.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1690.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1703.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.51.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.51.self_attn.o_proj: 8-bit storage</title>
<rect x="1716.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1729.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1742.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1755.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1768.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1781.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1794.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1807.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.55.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.55.self_attn.o_proj: 8-bit storage</title>
<rect x="1820.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1833.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1846.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1859.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1872.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1885.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1898.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1911.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.59.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.59.self_attn.o_proj: 8-bit storage</title>
<rect x="1924.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="1937.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
<rect x="1950.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1963.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="1976.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="1989.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<rect x="2002.8" y="726.8" width="24.4" height="42.4" fill="#e8edf0" rx="1.5"/>
<text x="2015.0" y="754" font-size="13" fill="#97a7af" font-weight="400" text-anchor="middle">–</text>
<g data-layer="model.language_model.layers.63.self_attn.o_proj" data-stored-bits="8">
<title >model.language_model.layers.63.self_attn.o_proj: 8-bit storage</title>
<rect x="2028.8" y="726.8" width="24.4" height="42.4" fill="#185b69" rx="1.5"/>
<text x="2041.0" y="754" font-size="16" fill="#ffffff" font-weight="700" text-anchor="middle">8</text>
</g>
</svg>